From 280b6a4046ee0bb4f7f34fc7dfc744523d1488ee Mon Sep 17 00:00:00 2001 From: HiddenTrojan <93521146+Labeeb2339@users.noreply.github.com> Date: Sun, 2 Aug 2026 08:47:37 +0800 Subject: [PATCH 01/24] Freeze Experiment 013 static Q468 protocol --- ...EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md | 338 +++++++ scripts/resolve_static_q468_identity.py | 950 ++++++++++++++++++ tests/test_resolve_static_q468_identity.py | 395 ++++++++ 3 files changed, 1683 insertions(+) create mode 100644 research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md create mode 100644 scripts/resolve_static_q468_identity.py create mode 100644 tests/test_resolve_static_q468_identity.py diff --git a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md new file mode 100644 index 0000000..329eb06 --- /dev/null +++ b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md @@ -0,0 +1,338 @@ +# Experiment 013: static RHT-Q468 packed-native adoption protocol + +> **Status: frozen before Experiment 013 identity resolution, policy fitting, +> protected-set access, model-weight loading, or quality measurement.** +> +> Upstream revisions are frozen below. Canonical row identities, token spans, +> tokenizer-file hashes, and content hashes remain unresolved placeholders +> until a separate identity candidate is resolved and explicitly promoted. An +> identity candidate is not authorization to load model weights. + +Protocol frozen: 2026-08-02 + +## Question + +Can a calibration-frozen, static Q4/Q6/Q8 recurrent-state layout retain the +quality of an exact dynamic mixed-bit oracle while using one immutable code map +that a packed GPU kernel can execute efficiently? + +The candidate is: + +```text +rht_q468_static_k29334 +``` + +The short name is **static RHT-Q468**. `K29334` is the exact sum of precision +steps over all 36,864 recurrent-state groups, where Q4, Q6, and Q8 contribute +zero, one, and two steps respectively. This experiment is an adoption study, +not a novelty, state-of-the-art, deployment, or breakthrough claim. + +## Fixed model contracts + +The primary checkpoint and tokenizer are: + +```text +Qwen/Qwen3.5-0.8B-Base +revision dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68 +Transformers 5.14.1 +``` + +The conditional scale check is: + +```text +Qwen/Qwen3.5-2B-Base +revision b1485b2fa6dfa1287294f269f5fb618e03d52d7c +Transformers 5.14.1 +``` + +The 2B check is authorized only if a cold-start language-model load measures no +more than `7.5 GiB` peak device memory on the RTX 5070 8GB. Failure or inability +to satisfy that gate is recorded as a resource stop; it cannot be replaced by +an unreported offload configuration. + +Both checkpoints use batch one, eager evaluation, no sampling, BF16 model +weights, and FP32 reference recurrent state. Model architecture, recurrent +layer indices, state geometry, tokenizer class, tokenizer files, and every +runtime package are identity-bound before weights are opened. A mismatch stops +the run. + +The metadata-only upstream identities are frozen without opening example +contents: + +| Role | Source | Revision | +| --- | --- | --- | +| existing MBPP calibration | `google-research-datasets/mbpp` | `4bb6404fdc6cacfda99d4ac4205087b89d32030c` | +| PG19 | `emozilla/pg19` | `c021754c8e01c5b1cc83a1f549c1f97fbbb756b8` | +| RULER generator | `NVIDIA/RULER` | `c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a` | +| HumanEval+ dataset | `evalplus/humanevalplus` | `d32357cf319e50e9c8d8dab5ea876c72b0fd321b` | +| EvalPlus source | `evalplus/evalplus` | `26d6d00bb1fd0fa37f39c99d5290da67891d1c5e` | + +PG19 train, validation, and test parquet siblings were confirmed at the pinned +revision. That metadata check does not resolve, retain, format, or tokenize an +Experiment 013 row. + +## Candidate format and exact bytes + +Every recurrent group contains 128 FP32 values before quantization. The codec +applies the already specified deterministic right randomized Hadamard transform +within the value axis, followed by symmetric signed absmax Q4, Q6, or Q8 with +one FP16 scale per group. Nearest-even rounding is fixed. The sign schedule and +transform convention are inherited from Experiment 009; a source freeze must +bind their exact implementation and tests. + +The static precision map is fixed after calibration and does not change by +prompt, token, cache write, batch element, or runtime activation. Codes use two +bits per group. Uint16 offsets address the packed group pool. The primary +resident allocation is: + +| Component | Bytes | +| --- | ---: | +| packed Q4/Q6/Q8 payload | 3,297,984 | +| FP16 scales | 73,728 | +| two-bit precision codes | 9,216 | +| uint16 pool offsets | 73,728 | +| alignment padding | 8 | +| **resident recurrent state** | **3,454,664** | + +The FP32 recurrent-state reference is `18,874,368` bytes. Shared metadata, +codec constants, transient decode workspaces, allocator peaks, model weights, +convolution state, and attention KV caches are reported separately. None may +be hidden inside the recurrent-state number. No candidate or comparator may +retain an undisclosed persistent FP32 state or dequantized mirror. + +The exact-byte static Q4/Q8 comparator is `rht_q48_static_p14739`: 14,739 groups +use Q8 and the rest Q4. Its payload is `3,302,592` bytes, its one-bit code map +is `4,608` bytes, and its scales, uint16 offsets, and eight alignment bytes +make the same `3,454,664`-byte total. + +The `K27030` layout is a prespecified diagnostic budget used only to compare +static and dynamic selection without conflating that comparison with the +primary exact-byte contrast. Its physical allocation is reported exactly and +is never rounded up to the primary budget. + +## Calibration and static-policy freeze + +Policy fitting uses only these calibration sources: + +1. the existing frozen public-evaluation v0.2 MBPP calibration population of + 128 tasks; +2. 16 SHA-ranked eligible PG19 training books, with one non-overlapping + 2,304-token segment from each book; and +3. the four official NVIDIA RULER task families at lengths 2,048 and 4,096, + with generator seeds 12,339 and 12,340. + +Each workload family receives equal weight regardless of its number of tokens +or examples. Within a family, examples receive equal weight. The exact +configurations, canonical IDs, formatter identities, content hashes, +tokenizer-file hashes, and token spans must be resolved into an identity +amendment before fitting. The protocol does not guess unresolved row or token +identities. + +Eligible PG19 IDs are ranked by lowercase SHA-256 of the UTF-8 domain-separated +canonical ID. Training and validation use distinct domains. HumanEval+ uses a +different domain, and Stage C uses a separate confirmation domain. Ties break +by canonical ID. The exact domains are: + +```text +recurquant.experiment013.pg19.train.v1\0 +recurquant.experiment013.pg19.validation.v1\0 +recurquant.experiment013.pg19.test.v1\0 +recurquant.experiment013.humaneval-plus.stage-a-b.v1\0 +recurquant.experiment013.humaneval-plus.stage-c.v1\0 +``` + +Within each broad calibration family, and separately within each of RULER's +four official subfamilies, SHA-rank canonical sequence IDs and alternate even +and odd ranks into split halves A and B. Recompute the complete equation and +both exact-K allocations independently on each half. This produces +deterministic halves without observing a quality result. + +For every calibration sequence of `T` tokens, maintain the existing causal +normalized-query-energy EMA over every token, with decay `2^(-1/32)`, epsilon +`1e-6`, and a uniform `1/128` prior. Capture exactly the unique zero-based +post-token anchors + +```text +p_j = floor((j + 1) T / 16) - 1, j = 0,...,15. +``` + +If `T < 16`, capture all `T` positions. The 16-anchor rule is frozen to bound +calibration cost. An empty sequence, duplicate anchor after canonicalization, +non-finite state, energy, distortion, or aggregate fails closed. Every anchor +identity and its ordered manifest hash is recorded. + +At anchor `p`, for physical row +`r = (frozen recurrent-layer order, head, key-row)`, transform the FP32 state +with right-RHT seed 2,339. For `b` in `{4, 6, 8}`, use symmetric per-row +group-size-128 quantize/dequantize and accumulate in CPU FP64: + +```text +x[e,p,r,b] = EMA_query_energy[e,p,r] + * mean_value((Q_b(RHT(S[e,p,r])) - RHT(S[e,p,r]))^2) +``` + +Mean anchors within each sequence. Mean sequences within MBPP and PG19. For +RULER, mean sequences within each of its four official families, then mean the +four family means. The final score is + +```text +D_b(r) = (D_MBPP,b(r) + D_PG19,b(r) + D_RULER,b(r)) / 3. +``` + +The three broad calibration families therefore have equal coefficients +irrespective of example counts. There is no additional normalization, +clipping, rescaling, quota, or task-loss input. + +Flatten rows layer-major, then head-major, then key-row-major using the frozen +18-layer list. Feed `D4`, `D6`, and `D8` directly to the existing +`allocate_exact_multibit_codes_fast` allocator at `K=29334` and `K=27030`; +codes zero, one, and two mean Q4, Q6, and Q8. Its exact-equality tie rule is the +lexicographically greatest flattened code vector, so the lower flat row gets +higher precision first. + +No task loss, Stage-A value, Stage-B value, or Stage-C value participates in +the code-map fit. The primary map uses `K29334`. The diagnostic static and +dynamic layouts both use `K27030`. Once an identity and code map are committed, +no seed, score, quota, tie rule, token span, group size, or bit budget may be +changed under Experiment 013. + +## Policy-stability gates + +Before any Stage-A quality result is opened, independently fit K29334 maps on +split halves A and B. All three gates are conjunctive: + +1. split-half Spearman rank correlation is at least `0.70`; +2. Q8-set Jaccard similarity is at least `0.50`; and +3. every recurrent layer's absolute mean-code shift is at most `0.25` bits. + +Failure stops the static candidate. The map may not be stabilized by changing +the data, metric, seed, threshold, or aggregation after the failure is known. + +## Frozen evaluation identities + +Identity resolution is staged. A resolver may create only a quarantined +candidate. A separate explicit promotion, checked by candidate SHA-256, +creates the identity that must be committed before model weights are loaded. +Stage-B and Stage-C content is protected and requires separate authorization; +ordinary resolver tests and dry runs must not read it. + +### Stage A: multi-workload falsification + +Stage A contains exactly 12 examples: + +- the first four SHA-ranked eligible PG19 validation books, each using 4,096 + prefill tokens followed by 128 scored tokens; +- one official RULER configuration from each of the four families at length + 4,096 and seed 2,339, scoring only the identity-bound official answer span; + and +- the first four SHA-ranked HumanEval+ canonical IDs, scoring at most the first + 128 canonical-solution tokens after the identity-bound prompt. + +The exact canonical IDs, configurations, source revisions, formatter hashes, +content hashes, prompt-token hashes, target-token hashes, and half-open token +spans must be committed before weights are opened. No example may be replaced +because its result is inconvenient. + +The exact dynamic Q468 allocator is the Stage-A quality oracle. Static K27030 +must have an excess-NLL upper confidence bound no more than `0.01` nats/token +above dynamic K27030. No Stage-A workload may have a static-candidate +disadvantage above `0.015` nats/token. Static K29334 must beat exact-byte +`rht_q48_static_p14739` on every Stage-A workload. If the multi-workload advantage +does not reproduce, stop; do not proceed by reframing the oracle as optional. + +### Stage B: development + +Stage B remains closed until Stage A and every identity gate pass. It contains: + +- the remaining 28 eligible PG19 validation books after Stage A; +- the remaining 44 frozen RULER configurations; and +- the remaining 28 HumanEval+ tasks under the Stage-A/B ranking domain. + +Stage-B identities and token spans are unresolved protected placeholders in +this protocol. They must be promoted and committed in a separate amendment +before Stage-B model access. + +### Stage C: untouched confirmation + +Stage C remains closed until the complete Stage-B decision is committed. It +contains: + +- 32 SHA-ranked eligible PG19 test books; +- 48 frozen RULER configurations using seeds 3,339 through 3,342; and +- the next 32 HumanEval+ canonical IDs under the separate Stage-C confirmation + hash domain. + +The exact RULER configuration grid, IDs, and token spans remain unresolved +until a separate protected identity amendment is frozen. Stage C may not be +partially previewed. + +## Methods and measurements + +Every accepted quality run includes FP32 recurrent state, uniform RHT Q4 and +Q8 anchors, `rht_q48_static_p14739`, static K27030, exact dynamic K27030, and +`rht_q468_static_k29334`. A closest eligible published comparator is added only +through a pre-result identity amendment with its exact implementation and byte +accounting; an incompatible or unavailable comparator is documented rather +than imitated under its name. + +Primary quality is task-macro aligned excess next-token NLL relative to the +matched FP32 trajectory. Report task-macro and token-micro excess NLL, mean and +tail KL, top-1 agreement, local codec SSE, trajectory error, result by workload +family, resident bytes, transient bytes, peak HBM, and latency. Statistical +intervals are paired task bootstraps with 10,000 resamples and seed 2,339. + +For the full evaluation, the candidate must improve on the strongest eligible +comparator by both at least `10%` and at least `0.002` nats/token, with the +paired 95% lower confidence bound above zero. The point improvement must be +positive in each of PG19, RULER, and HumanEval+. Candidate top-1 agreement may +trail by at most `0.005`. A non-positive comparator excess NLL makes the +relative gate fail closed; it is not redefined. + +## Packed-native deployment gate + +The production path must consume the packed Q4/Q6/Q8 pools directly through a +fused Triton or CUDA implementation. A Python dequantization loop, persistent +FP32 mirror, or benchmark of an isolated helper cannot satisfy deployment. +Correctness must first match the frozen reference codec and recurrent update +within declared numerical tolerances across all three codes, offsets, boundary +groups, shapes, and deterministic fixtures. + +Under a pinned hardware/software protocol and identical model loop: + +- batch-1 p50 decode latency must be at most `1.05x` optimized FP32-cache; +- batch-1 p95 decode latency must be at most `1.10x` optimized FP32-cache; +- measured peak HBM must be lower than optimized FP32-cache; and +- either throughput at batch size at least eight improves by at least `10%`, + or the maximum batch before OOM improves by at least `1.25x`. + +Report warm-up, repeat count, synchronization, clock/power state, device, +driver, CUDA, kernel version, compiler flags, batch, prompt/decode lengths, and +all raw repeats. The existing uniform-kernel microbenchmark is an isolated +diagnostic only and cannot be used as end-to-end evidence. + +## Advancement and claim boundary + +Every integrity, stability, quality, and deployment gate is conjunctive for an +adoption-ready claim. A quality pass without the deployment gate supports only +a quality/storage result. A kernel pass without the protected quality stages +supports only an implementation result. + +The prior-art boundary is narrow. [RateQuant](https://arxiv.org/abs/2605.06675v2) +already fits calibration-based mixed-precision rate-distortion policies for KV +caches. [Q-Mamba](https://aclanthology.org/2025.findings-acl.551/) quantizes +Mamba state caches, while [Quamba2](https://arxiv.org/abs/2503.22879v4) provides +quantized SSM deployment and kernels. [Gated DeltaNet-2](https://arxiv.org/abs/2605.22791v1) +motivates the architecture family, but its public reproducibility remains +blocked. Therefore only a confirmed exact-byte static Q4/Q6/Q8 packed-native +Gated DeltaNet path plus end-to-end adoption benefit could be differentiated; +this protocol makes no novelty claim. + +Even a complete pass would establish only that the frozen static packed layout +was useful on the pinned checkpoints, workloads, budgets, and hardware. It +would not establish that RHT, mixed precision, loss sensitivity, Q4/Q6/Q8, +static allocation, or packed kernels are new. It would not make RecurQuant a +new base model, prove generated-code correctness, eliminate contamination, or +justify "breakthrough," "state of the art," "lossless," or universal language. + +Failure is a publishable result. Any change after a gate is observed creates a +new experiment number with new protected data. diff --git a/scripts/resolve_static_q468_identity.py b/scripts/resolve_static_q468_identity.py new file mode 100644 index 0000000..470c69b --- /dev/null +++ b/scripts/resolve_static_q468_identity.py @@ -0,0 +1,950 @@ +#!/usr/bin/env python3 +"""Build a fail-closed Experiment 013 identity without loading model weights. + +The resolver consumes an offline metadata manifest. Raw prompts, targets, +books, generated RULER examples, and token-ID arrays are forbidden: the input +contains only canonical identities, immutable revisions, spans, sizes, and +SHA-256 commitments produced by a separately audited extractor. + +Resolution writes a deterministic *candidate* under a quarantine directory. +Freezing is a separate, explicit promotion bound to the candidate file hash. +This version deliberately refuses Stage B and Stage C before reading an input +file; protected identities require a later protocol amendment. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import string +import sys +import tempfile +import unicodedata +from collections.abc import Mapping, Sequence +from copy import deepcopy +from pathlib import Path +from typing import Any, Final + +INPUT_SCHEMA: Final = "recurquant.experiment013.identity-input.v1" +CANDIDATE_SCHEMA: Final = "recurquant.experiment013.identity-candidate.v1" +FROZEN_SCHEMA: Final = "recurquant.experiment013.identity-frozen.v1" +ARTIFACT_KIND: Final = "recurquant_static_rht_q468_identity" +RESOLVER_VERSION: Final = 1 + +PRIMARY_MODEL_ID: Final = "Qwen/Qwen3.5-0.8B-Base" +PRIMARY_MODEL_REVISION: Final = "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68" +CONDITIONAL_MODEL_ID: Final = "Qwen/Qwen3.5-2B-Base" +CONDITIONAL_MODEL_REVISION: Final = "b1485b2fa6dfa1287294f269f5fb618e03d52d7c" +TRANSFORMERS_VERSION: Final = "5.14.1" + +MBPP_DATASET_ID: Final = "google-research-datasets/mbpp" +MBPP_CONFIG: Final = "full" +MBPP_REVISION: Final = "4bb6404fdc6cacfda99d4ac4205087b89d32030c" +MBPP_SELECTION_NAMESPACE: Final = "rq-v0.2" +PG19_DATASET_ID: Final = "emozilla/pg19" +PG19_REVISION: Final = "c021754c8e01c5b1cc83a1f549c1f97fbbb756b8" +RULER_SOURCE_ID: Final = "NVIDIA/RULER" +RULER_REVISION: Final = "c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a" +HUMANEVAL_PLUS_DATASET_ID: Final = "evalplus/humanevalplus" +HUMANEVAL_PLUS_REVISION: Final = "d32357cf319e50e9c8d8dab5ea876c72b0fd321b" +EVALPLUS_SOURCE_ID: Final = "evalplus/evalplus" +EVALPLUS_SOURCE_REVISION: Final = "26d6d00bb1fd0fa37f39c99d5290da67891d1c5e" +DATASET_KEYS: Final = ("mbpp", "pg19", "ruler", "humaneval_plus") +FAMILY_ORDER: Final = {key: index for index, key in enumerate(DATASET_KEYS)} +FROZEN_DATASET_REVISIONS: Final = { + "mbpp": MBPP_REVISION, + "pg19": PG19_REVISION, + "ruler": RULER_REVISION, + "humaneval_plus": HUMANEVAL_PLUS_REVISION, +} + +PG19_TRAIN_NAMESPACE: Final = "recurquant.experiment013.pg19.train.v1\0" +PG19_VALIDATION_NAMESPACE: Final = ( + "recurquant.experiment013.pg19.validation.v1\0" +) +PG19_TEST_NAMESPACE: Final = "recurquant.experiment013.pg19.test.v1\0" +HUMANEVAL_AB_NAMESPACE: Final = ( + "recurquant.experiment013.humaneval-plus.stage-a-b.v1\0" +) +HUMANEVAL_C_NAMESPACE: Final = ( + "recurquant.experiment013.humaneval-plus.stage-c.v1\0" +) +CALIBRATION_SPLIT_NAMESPACE: Final = ( + "recurquant.experiment013.calibration-split.v1\0" +) + +CLAIM_BOUNDARY: Final = ( + "This artifact freezes Experiment 013 data and tokenizer identity only. " + "It is not quality, latency, novelty, state-of-the-art, deployment, or " + "breakthrough evidence." +) +PROTECTED_STAGES: Final = frozenset({"stage_b", "stage_c"}) +ALLOWED_PHASES: Final = frozenset({"calibration", "stage_a"}) +HEX_LENGTHS: Final = frozenset({40, 64}) + +DATASET_FIELDS: Final = frozenset( + { + "key", + "dataset_id", + "config", + "revision", + "split", + "canonical_id_field", + "canonical_id_manifest_sha256", + "formatter_id", + "formatter_sha256", + } +) +TOKENIZER_FIELDS: Final = frozenset( + {"source_id", "revision", "class", "transformers_version", "files"} +) +TOKENIZER_FILE_FIELDS: Final = frozenset({"name", "sha256", "size_bytes"}) +RECORD_FIELDS: Final = frozenset( + { + "family", + "canonical_id", + "config", + "selection_rank", + "selection_sha256", + "seed", + "sequence_length", + "source_content_sha256", + "formatted_content_sha256", + "prompt_token_ids_sha256", + "target_token_ids_sha256", + "tokenizer_manifest_sha256", + "token_span", + "anchor_manifest_sha256", + } +) +TOKEN_SPAN_FIELDS: Final = frozenset( + {"prefill_start", "prefill_stop", "scored_start", "scored_stop"} +) +CALIBRATION_BINDING_FIELDS: Final = frozenset( + { + "identity_file_sha256", + "canonical_identity_sha256", + "static_k29334_code_map_sha256", + "static_k27030_code_map_sha256", + "split_half_policy_manifest_sha256", + } +) + + +def canonical_json_bytes(value: object) -> bytes: + """Return the repository's deterministic JSON representation.""" + + return ( + json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + b"\n" + ) + + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def is_sha256(value: object) -> bool: + return ( + isinstance(value, str) + and len(value) == 64 + and value == value.lower() + and all(character in string.hexdigits for character in value) + ) + + +def require_sha256(value: object, *, context: str) -> str: + if not is_sha256(value): + raise ValueError(f"{context} must be a lowercase SHA-256") + return str(value) + + +def require_exact_revision(value: object, *, context: str) -> str: + if ( + not isinstance(value, str) + or len(value) not in HEX_LENGTHS + or value != value.lower() + or not all(character in string.hexdigits for character in value) + ): + raise ValueError( + f"{context} must be an immutable lowercase 40- or 64-hex revision" + ) + return value + + +def require_int(value: object, *, context: str, minimum: int = 0) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise ValueError(f"{context} must be an integer >= {minimum}") + return value + + +def require_string(value: object, *, context: str, allow_empty: bool = False) -> str: + if not isinstance(value, str) or (not allow_empty and not value): + raise ValueError(f"{context} must be a non-empty string") + if value != unicodedata.normalize("NFC", value) or value != value.strip(): + raise ValueError(f"{context} must be stripped NFC text") + return value + + +def require_mapping(value: object, *, context: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError(f"{context} must be an object") + return value + + +def require_sequence(value: object, *, context: str) -> Sequence[Any]: + if isinstance(value, (str, bytes, bytearray)) or not isinstance(value, Sequence): + raise ValueError(f"{context} must be an array") + return value + + +def require_exact_fields( + value: Mapping[str, Any], expected: frozenset[str], *, context: str +) -> None: + actual = frozenset(value) + if actual != expected: + missing = sorted(expected - actual) + extra = sorted(actual - expected) + raise ValueError(f"{context} fields drifted; missing={missing}, extra={extra}") + + +def selection_sha256(namespace: str, canonical_id: str) -> str: + return sha256_bytes(namespace.encode("utf-8") + canonical_id.encode("utf-8")) + + +def mbpp_selection_sha256(canonical_id: str) -> str: + return sha256_bytes(f"{MBPP_SELECTION_NAMESPACE}|{canonical_id}".encode()) + + +def calibration_split_key(record: Mapping[str, Any]) -> str: + identity = "\0".join( + ( + str(record["family"]), + str(record["config"]), + str(record["canonical_id"]), + str(record["seed"]), + str(record["sequence_length"]), + ) + ) + return selection_sha256(CALIBRATION_SPLIT_NAMESPACE, identity) + + +def anchor_positions(token_count: int) -> tuple[int, ...]: + """Return the frozen unique post-token anchor positions.""" + + if token_count < 1: + raise ValueError("calibration sequence token count must be positive") + if token_count < 16: + return tuple(range(token_count)) + positions = tuple((j + 1) * token_count // 16 - 1 for j in range(16)) + if len(positions) != len(set(positions)): + raise RuntimeError("frozen anchor equation produced duplicate positions") + return positions + + +def mbpp_calibration_identity() -> tuple[tuple[str, ...], str]: + """Return the existing v0.2 ID-only calibration selection and pool hash.""" + + population = tuple(str(task_id) for task_id in range(601, 975)) + ranked = sorted(population, key=lambda task_id: (mbpp_selection_sha256(task_id), int(task_id))) + selected = tuple(ranked[:128]) + population_hash = sha256_bytes(canonical_json_bytes(population)) + return selected, population_hash + + +def _validate_sha_rank_order( + rows: Sequence[Mapping[str, Any]], *, context: str +) -> None: + ranked = sorted( + rows, + key=lambda row: (str(row["selection_sha256"]), str(row["canonical_id"])), + ) + if [int(row["selection_rank"]) for row in ranked] != list(range(len(rows))): + raise ValueError(f"{context} selection ranks do not match SHA-256 order") + + +def _json_without_duplicate_keys(raw: bytes, *, context: str) -> dict[str, Any]: + def pairs_hook(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"{context} contains duplicate key {key!r}") + result[key] = value + return result + + try: + value = json.loads(raw.decode("utf-8"), object_pairs_hook=pairs_hook) + except UnicodeDecodeError as error: + raise ValueError(f"{context} must be UTF-8") from error + except json.JSONDecodeError as error: + raise ValueError(f"{context} must be strict JSON") from error + if not isinstance(value, dict): + raise ValueError(f"{context} must contain one JSON object") + return value + + +def _validate_dataset_contracts( + value: object, *, expected_revisions: Mapping[str, str] +) -> tuple[dict[str, Any], ...]: + entries = require_sequence(value, context="datasets") + if len(entries) != len(DATASET_KEYS): + raise ValueError("datasets must contain exactly four contracts") + normalized: dict[str, dict[str, Any]] = {} + for index, raw in enumerate(entries): + item = require_mapping(raw, context=f"datasets[{index}]") + require_exact_fields(item, DATASET_FIELDS, context=f"datasets[{index}]") + key = require_string(item["key"], context=f"datasets[{index}].key") + if key not in DATASET_KEYS or key in normalized: + raise ValueError(f"dataset key is unknown or duplicated: {key}") + revision = require_exact_revision( + item["revision"], context=f"datasets[{index}].revision" + ) + if revision != expected_revisions[key]: + raise ValueError(f"{key} dataset revision does not match the CLI contract") + contract = { + "key": key, + "dataset_id": require_string( + item["dataset_id"], context=f"datasets[{index}].dataset_id" + ), + "config": require_string( + item["config"], context=f"datasets[{index}].config", allow_empty=True + ), + "revision": revision, + "split": require_string( + item["split"], context=f"datasets[{index}].split" + ), + "canonical_id_field": require_string( + item["canonical_id_field"], + context=f"datasets[{index}].canonical_id_field", + ), + "canonical_id_manifest_sha256": require_sha256( + item["canonical_id_manifest_sha256"], + context=f"datasets[{index}].canonical_id_manifest_sha256", + ), + "formatter_id": require_string( + item["formatter_id"], context=f"datasets[{index}].formatter_id" + ), + "formatter_sha256": require_sha256( + item["formatter_sha256"], + context=f"datasets[{index}].formatter_sha256", + ), + } + if key == "mbpp" and ( + contract["dataset_id"] != MBPP_DATASET_ID + or contract["config"] != MBPP_CONFIG + ): + raise ValueError("MBPP identity must match the frozen v0.2 source") + if key == "mbpp": + _selected_ids, expected_population_hash = mbpp_calibration_identity() + if contract["canonical_id_manifest_sha256"] != expected_population_hash: + raise ValueError("MBPP train ID manifest does not match the frozen v0.2 pool") + expected_source_id = { + "mbpp": MBPP_DATASET_ID, + "pg19": PG19_DATASET_ID, + "ruler": RULER_SOURCE_ID, + "humaneval_plus": HUMANEVAL_PLUS_DATASET_ID, + }[key] + if contract["dataset_id"] != expected_source_id: + raise ValueError(f"{key} source ID does not match the frozen upstream identity") + normalized[key] = contract + return tuple(normalized[key] for key in DATASET_KEYS) + + +def _validate_tokenizer(value: object) -> dict[str, Any]: + tokenizer = require_mapping(value, context="tokenizer") + require_exact_fields(tokenizer, TOKENIZER_FIELDS, context="tokenizer") + if tokenizer["source_id"] != PRIMARY_MODEL_ID: + raise ValueError("tokenizer source must equal the primary model ID") + if tokenizer["revision"] != PRIMARY_MODEL_REVISION: + raise ValueError("tokenizer revision must equal the pinned primary revision") + if tokenizer["transformers_version"] != TRANSFORMERS_VERSION: + raise ValueError("Transformers version drifted") + tokenizer_class = require_string(tokenizer["class"], context="tokenizer.class") + raw_files = require_sequence(tokenizer["files"], context="tokenizer.files") + if not raw_files: + raise ValueError("tokenizer.files cannot be empty") + files: list[dict[str, Any]] = [] + names: set[str] = set() + for index, raw in enumerate(raw_files): + item = require_mapping(raw, context=f"tokenizer.files[{index}]") + require_exact_fields( + item, TOKENIZER_FILE_FIELDS, context=f"tokenizer.files[{index}]" + ) + name = require_string(item["name"], context=f"tokenizer.files[{index}].name") + if Path(name).name != name or name in names: + raise ValueError("tokenizer file names must be unique basenames") + names.add(name) + files.append( + { + "name": name, + "sha256": require_sha256( + item["sha256"], context=f"tokenizer.files[{index}].sha256" + ), + "size_bytes": require_int( + item["size_bytes"], + context=f"tokenizer.files[{index}].size_bytes", + minimum=1, + ), + } + ) + files.sort(key=lambda item: item["name"]) + file_manifest_hash = sha256_bytes(canonical_json_bytes(files)) + return { + "source_id": PRIMARY_MODEL_ID, + "revision": PRIMARY_MODEL_REVISION, + "class": tokenizer_class, + "transformers_version": TRANSFORMERS_VERSION, + "files": files, + "file_manifest_sha256": file_manifest_hash, + } + + +def _selection_namespace(phase: str, family: str) -> str | None: + if phase == "calibration": + if family == "mbpp": + return None + if family == "pg19": + return PG19_TRAIN_NAMESPACE + return CALIBRATION_SPLIT_NAMESPACE + if phase == "stage_a": + if family == "pg19": + return PG19_VALIDATION_NAMESPACE + if family == "humaneval_plus": + return HUMANEVAL_AB_NAMESPACE + return CALIBRATION_SPLIT_NAMESPACE + raise ValueError(f"unsupported phase: {phase}") + + +def _normalize_record( + raw: object, *, index: int, phase: str, tokenizer_hash: str +) -> dict[str, Any]: + item = require_mapping(raw, context=f"records[{index}]") + require_exact_fields(item, RECORD_FIELDS, context=f"records[{index}]") + family = require_string(item["family"], context=f"records[{index}].family") + if family not in DATASET_KEYS: + raise ValueError(f"records[{index}].family is unknown") + canonical_id = require_string( + item["canonical_id"], context=f"records[{index}].canonical_id" + ) + config = require_string( + item["config"], context=f"records[{index}].config", allow_empty=True + ) + rank = require_int(item["selection_rank"], context=f"records[{index}].selection_rank") + seed_value = item["seed"] + seed = None if seed_value is None else require_int( + seed_value, context=f"records[{index}].seed" + ) + sequence_length = require_int( + item["sequence_length"], + context=f"records[{index}].sequence_length", + minimum=1, + ) + namespace = _selection_namespace(phase, family) + expected_selection = ( + mbpp_selection_sha256(canonical_id) + if family == "mbpp" + else selection_sha256(str(namespace), canonical_id) + ) + if item["selection_sha256"] != expected_selection: + raise ValueError(f"records[{index}] selection SHA-256 drifted") + if item["tokenizer_manifest_sha256"] != tokenizer_hash: + raise ValueError(f"records[{index}] tokenizer manifest binding drifted") + span = require_mapping(item["token_span"], context=f"records[{index}].token_span") + require_exact_fields(span, TOKEN_SPAN_FIELDS, context=f"records[{index}].token_span") + normalized_span = { + name: require_int(span[name], context=f"records[{index}].token_span.{name}") + for name in ("prefill_start", "prefill_stop", "scored_start", "scored_stop") + } + if ( + normalized_span["prefill_start"] != 0 + or normalized_span["prefill_stop"] != normalized_span["scored_start"] + or normalized_span["prefill_stop"] < 1 + or normalized_span["scored_stop"] < normalized_span["scored_start"] + ): + raise ValueError(f"records[{index}] token span is not contiguous and canonical") + positions = anchor_positions(sequence_length) + return { + "family": family, + "canonical_id": canonical_id, + "config": config, + "selection_rank": rank, + "selection_sha256": expected_selection, + "seed": seed, + "sequence_length": sequence_length, + "source_content_sha256": require_sha256( + item["source_content_sha256"], + context=f"records[{index}].source_content_sha256", + ), + "formatted_content_sha256": require_sha256( + item["formatted_content_sha256"], + context=f"records[{index}].formatted_content_sha256", + ), + "prompt_token_ids_sha256": require_sha256( + item["prompt_token_ids_sha256"], + context=f"records[{index}].prompt_token_ids_sha256", + ), + "target_token_ids_sha256": require_sha256( + item["target_token_ids_sha256"], + context=f"records[{index}].target_token_ids_sha256", + ), + "tokenizer_manifest_sha256": tokenizer_hash, + "token_span": normalized_span, + "anchor_manifest_sha256": require_sha256( + item["anchor_manifest_sha256"], + context=f"records[{index}].anchor_manifest_sha256", + ), + "anchor_positions": list(positions), + "anchor_positions_sha256": sha256_bytes(canonical_json_bytes(positions)), + } + + +def _record_sort_key(record: Mapping[str, Any]) -> tuple[Any, ...]: + return ( + FAMILY_ORDER[str(record["family"])], + int(record["selection_rank"]), + str(record["selection_sha256"]), + str(record["canonical_id"]), + str(record["config"]), + -1 if record["seed"] is None else int(record["seed"]), + int(record["sequence_length"]), + ) + + +def _validate_calibration_records(records: Sequence[Mapping[str, Any]]) -> None: + grouped = { + family: [record for record in records if record["family"] == family] + for family in DATASET_KEYS + } + expected_counts = {"mbpp": 128, "pg19": 16, "ruler": 16, "humaneval_plus": 0} + if {family: len(rows) for family, rows in grouped.items()} != expected_counts: + raise ValueError("calibration record counts must be MBPP=128, PG19=16, RULER=16") + expected_mbpp_ids, _population_hash = mbpp_calibration_identity() + actual_mbpp = sorted( + grouped["mbpp"], key=lambda row: int(row["selection_rank"]) + ) + if tuple(str(row["canonical_id"]) for row in actual_mbpp) != expected_mbpp_ids: + raise ValueError("MBPP rows do not match the frozen v0.2 calibration identity") + if sorted(int(row["selection_rank"]) for row in grouped["pg19"]) != list(range(16)): + raise ValueError("calibration PG19 ranks must be exactly 0..15") + _validate_sha_rank_order(grouped["pg19"], context="calibration PG19") + for row in grouped["pg19"]: + span = row["token_span"] + if row["seed"] is not None or row["sequence_length"] != 2_304: + raise ValueError("calibration PG19 must use one 2,304-token sequence per book") + if span != { + "prefill_start": 0, + "prefill_stop": 2_304, + "scored_start": 2_304, + "scored_stop": 2_304, + }: + raise ValueError("calibration PG19 span must cover exactly 2,304 tokens") + ruler_tuples = { + (str(row["config"]), int(row["sequence_length"]), int(row["seed"])) + for row in grouped["ruler"] + } + if len({row["config"] for row in grouped["ruler"]}) != 4 or len(ruler_tuples) != 16: + raise ValueError("calibration RULER must contain four unique official families") + expected_pairs = {(length, seed) for length in (2_048, 4_096) for seed in (12_339, 12_340)} + for config in {str(row["config"]) for row in grouped["ruler"]}: + actual_pairs = { + (int(row["sequence_length"]), int(row["seed"])) + for row in grouped["ruler"] + if row["config"] == config + } + if actual_pairs != expected_pairs: + raise ValueError(f"RULER family {config!r} does not have the frozen grid") + for row in grouped["mbpp"]: + if row["seed"] is not None: + raise ValueError("MBPP calibration records cannot have a generator seed") + identities = [(row["family"], row["canonical_id"], row["config"]) for row in records] + if len(identities) != len(set(identities)): + raise ValueError("calibration canonical identities are not unique") + + +def _validate_stage_a_records(records: Sequence[Mapping[str, Any]]) -> None: + grouped = { + family: [record for record in records if record["family"] == family] + for family in DATASET_KEYS + } + expected_counts = {"mbpp": 0, "pg19": 4, "ruler": 4, "humaneval_plus": 4} + if {family: len(rows) for family, rows in grouped.items()} != expected_counts: + raise ValueError("Stage A must contain exactly four PG19, RULER, and HumanEval+ rows") + for family in ("pg19", "humaneval_plus"): + if sorted(int(row["selection_rank"]) for row in grouped[family]) != list(range(4)): + raise ValueError(f"Stage-A {family} ranks must be exactly 0..3") + _validate_sha_rank_order(grouped[family], context=f"Stage-A {family}") + for row in grouped["pg19"]: + span = row["token_span"] + if row["seed"] is not None or row["sequence_length"] != 4_224: + raise ValueError("Stage-A PG19 must use 4,096 prefill plus 128 scored tokens") + if span != { + "prefill_start": 0, + "prefill_stop": 4_096, + "scored_start": 4_096, + "scored_stop": 4_224, + }: + raise ValueError("Stage-A PG19 token span drifted") + if len({row["config"] for row in grouped["ruler"]}) != 4: + raise ValueError("Stage-A RULER must contain four distinct official families") + for row in grouped["ruler"]: + span = row["token_span"] + if row["seed"] != 2_339 or row["sequence_length"] != 4_096: + raise ValueError("Stage-A RULER must use length 4,096 and seed 2,339") + if span["scored_stop"] <= span["scored_start"]: + raise ValueError("Stage-A RULER answer span cannot be empty") + for row in grouped["humaneval_plus"]: + span = row["token_span"] + scored = span["scored_stop"] - span["scored_start"] + if row["seed"] is not None or not 1 <= scored <= 128: + raise ValueError("Stage-A HumanEval+ must score 1..128 solution tokens") + if row["sequence_length"] != span["scored_stop"]: + raise ValueError("Stage-A HumanEval+ sequence length must equal span stop") + identities = [(row["family"], row["canonical_id"], row["config"]) for row in records] + if len(identities) != len(set(identities)): + raise ValueError("Stage-A canonical identities are not unique") + + +def _split_half_manifest(records: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + assignments: list[dict[str, Any]] = [] + groups: list[tuple[str, list[Mapping[str, Any]]]] = [] + groups.append(("mbpp", [row for row in records if row["family"] == "mbpp"])) + groups.append(("pg19", [row for row in records if row["family"] == "pg19"])) + ruler_rows = [row for row in records if row["family"] == "ruler"] + for config in sorted({str(row["config"]) for row in ruler_rows}): + groups.append((f"ruler:{config}", [row for row in ruler_rows if row["config"] == config])) + for group, rows in groups: + ranked = sorted(rows, key=lambda row: (calibration_split_key(row), _record_sort_key(row))) + for rank, row in enumerate(ranked): + assignments.append( + { + "group": group, + "canonical_id": row["canonical_id"], + "config": row["config"], + "rank": rank, + "half": "a" if rank % 2 == 0 else "b", + "rank_sha256": calibration_split_key(row), + } + ) + assignments.sort(key=lambda item: (item["group"], item["rank"])) + return { + "namespace": CALIBRATION_SPLIT_NAMESPACE, + "assignment": assignments, + "assignment_sha256": sha256_bytes(canonical_json_bytes(assignments)), + } + + +def _validate_calibration_binding(value: object) -> dict[str, str]: + binding = require_mapping(value, context="calibration_binding") + require_exact_fields( + binding, CALIBRATION_BINDING_FIELDS, context="calibration_binding" + ) + return { + key: require_sha256(binding[key], context=f"calibration_binding.{key}") + for key in sorted(CALIBRATION_BINDING_FIELDS) + } + + +def build_candidate( + source: Mapping[str, Any], *, expected_revisions: Mapping[str, str] +) -> dict[str, Any]: + """Validate metadata and return a deterministic candidate artifact.""" + + phase = source.get("phase") + expected_fields = { + "schema", + "phase", + "datasets", + "tokenizer", + "records", + "model_weights_loaded", + } + if phase == "stage_a": + expected_fields.add("calibration_binding") + require_exact_fields(source, frozenset(expected_fields), context="identity input") + if source["schema"] != INPUT_SCHEMA: + raise ValueError("identity input schema drifted") + if phase not in ALLOWED_PHASES: + if phase in PROTECTED_STAGES: + raise PermissionError(f"{phase} is protected and unavailable in resolver v1") + raise ValueError(f"unsupported identity phase: {phase!r}") + if source["model_weights_loaded"] is not False: + raise ValueError("identity resolution must occur before model weights") + if set(expected_revisions) != set(DATASET_KEYS): + raise ValueError("all four dataset revisions are mandatory") + revisions = { + key: require_exact_revision(value, context=f"CLI {key} revision") + for key, value in expected_revisions.items() + } + if revisions != FROZEN_DATASET_REVISIONS: + raise ValueError("CLI dataset revisions do not match the frozen upstream commits") + datasets = _validate_dataset_contracts(source["datasets"], expected_revisions=revisions) + tokenizer = _validate_tokenizer(source["tokenizer"]) + raw_records = require_sequence(source["records"], context="records") + records = [ + _normalize_record( + raw, + index=index, + phase=str(phase), + tokenizer_hash=tokenizer["file_manifest_sha256"], + ) + for index, raw in enumerate(raw_records) + ] + records.sort(key=_record_sort_key) + if phase == "calibration": + _validate_calibration_records(records) + split_half = _split_half_manifest(records) + calibration_binding = None + else: + _validate_stage_a_records(records) + split_half = None + calibration_binding = _validate_calibration_binding(source["calibration_binding"]) + + content_manifest_hash = sha256_bytes(canonical_json_bytes(records)) + source_hash = sha256_bytes(canonical_json_bytes(source)) + evidence: dict[str, Any] = { + "schema_version": 1, + "artifact_kind": ARTIFACT_KIND, + "identity_schema": CANDIDATE_SCHEMA, + "resolver_version": RESOLVER_VERSION, + "status": "candidate", + "phase": phase, + "identity_only": True, + "claim_boundary": CLAIM_BOUNDARY, + "source_manifest_sha256": source_hash, + "model_contracts": { + "primary": {"id": PRIMARY_MODEL_ID, "revision": PRIMARY_MODEL_REVISION}, + "conditional_scale_check": { + "id": CONDITIONAL_MODEL_ID, + "revision": CONDITIONAL_MODEL_REVISION, + "cold_start_peak_hbm_limit_bytes": 8_053_063_680, + }, + "weights_loaded": False, + }, + "datasets": list(datasets), + "upstream_tool_contracts": { + "ruler_generator": { + "id": RULER_SOURCE_ID, + "revision": RULER_REVISION, + }, + "evalplus_formatter": { + "id": EVALPLUS_SOURCE_ID, + "revision": EVALPLUS_SOURCE_REVISION, + }, + }, + "tokenizer": tokenizer, + "records": records, + "record_count": len(records), + "content_manifest_sha256": content_manifest_hash, + "selection": { + "pg19_train_namespace": PG19_TRAIN_NAMESPACE, + "pg19_validation_namespace": PG19_VALIDATION_NAMESPACE, + "pg19_test_namespace": PG19_TEST_NAMESPACE, + "humaneval_plus_stage_a_b_namespace": HUMANEVAL_AB_NAMESPACE, + "humaneval_plus_stage_c_namespace": HUMANEVAL_C_NAMESPACE, + }, + "calibration_split_half": split_half, + "calibration_binding": calibration_binding, + "protected_identity": { + "stage_b_read": False, + "stage_c_read": False, + "ordinary_tests_may_read_protected_content": False, + }, + "promotion_required": True, + } + artifact = { + "canonical_evidence_sha256": sha256_bytes(canonical_json_bytes(evidence)), + "evidence": evidence, + } + validate_candidate_artifact(artifact) + return artifact + + +def validate_candidate_artifact(artifact: Mapping[str, Any]) -> None: + if set(artifact) != {"canonical_evidence_sha256", "evidence"}: + raise ValueError("candidate wrapper fields drifted") + evidence = require_mapping(artifact.get("evidence"), context="candidate evidence") + if artifact.get("canonical_evidence_sha256") != sha256_bytes( + canonical_json_bytes(evidence) + ): + raise ValueError("candidate canonical evidence SHA-256 drifted") + if ( + evidence.get("identity_schema") != CANDIDATE_SCHEMA + or evidence.get("status") != "candidate" + or evidence.get("phase") not in ALLOWED_PHASES + or evidence.get("identity_only") is not True + or evidence.get("claim_boundary") != CLAIM_BOUNDARY + or evidence.get("promotion_required") is not True + ): + raise ValueError("candidate identity contract drifted") + models = require_mapping(evidence.get("model_contracts"), context="model contracts") + if models.get("weights_loaded") is not False: + raise ValueError("candidate claims model weights were loaded") + records = require_sequence(evidence.get("records"), context="candidate records") + if evidence.get("record_count") != len(records): + raise ValueError("candidate record count drifted") + if evidence.get("content_manifest_sha256") != sha256_bytes( + canonical_json_bytes(records) + ): + raise ValueError("candidate content manifest SHA-256 drifted") + protected = require_mapping( + evidence.get("protected_identity"), context="protected identity" + ) + if protected != { + "stage_b_read": False, + "stage_c_read": False, + "ordinary_tests_may_read_protected_content": False, + }: + raise ValueError("protected identity boundary drifted") + + +def promote_candidate( + candidate: Mapping[str, Any], *, candidate_file_sha256: str +) -> dict[str, Any]: + """Create a deterministic frozen identity from an authenticated candidate.""" + + validate_candidate_artifact(candidate) + require_sha256(candidate_file_sha256, context="candidate file SHA-256") + evidence = deepcopy(dict(candidate["evidence"])) + evidence["identity_schema"] = FROZEN_SCHEMA + evidence["status"] = "frozen" + evidence["promotion_required"] = False + evidence["promotion"] = { + "candidate_file_sha256": candidate_file_sha256, + "candidate_canonical_evidence_sha256": candidate[ + "canonical_evidence_sha256" + ], + "explicit": True, + } + return { + "canonical_evidence_sha256": sha256_bytes(canonical_json_bytes(evidence)), + "evidence": evidence, + } + + +def validate_quarantine_output(path: Path) -> None: + resolved = path.resolve() + if not any("quarantine" in part.lower() for part in resolved.parts[:-1]): + raise ValueError("candidate output must be inside a quarantine directory") + if resolved.exists(): + raise FileExistsError(f"refusing to overwrite existing candidate: {resolved}") + + +def validate_promotion_output(path: Path) -> None: + resolved = path.resolve() + if any("quarantine" in part.lower() for part in resolved.parts[:-1]): + raise ValueError("frozen identity output must be outside quarantine") + if resolved.exists(): + raise FileExistsError(f"refusing to overwrite existing identity: {resolved}") + + +def atomic_write(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", suffix=".tmp", dir=path.parent + ) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + finally: + if temporary.exists(): + temporary.unlink() + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Resolve or explicitly promote an Experiment 013 identity." + ) + parser.add_argument( + "--phase", + required=True, + choices=("calibration", "stage_a", "stage_b", "stage_c"), + ) + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--output", type=Path) + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--promote", action="store_true") + parser.add_argument("--expected-candidate-sha256") + parser.add_argument("--mbpp-revision") + parser.add_argument("--pg19-revision") + parser.add_argument("--ruler-revision") + parser.add_argument("--humaneval-plus-revision") + return parser.parse_args(argv) + + +def _reject_protected_before_input(phase: str) -> None: + if phase in PROTECTED_STAGES: + raise PermissionError( + f"{phase} is protected; resolver v1 refuses it before reading --input" + ) + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + _reject_protected_before_input(args.phase) + if args.promote: + if args.dry_run or args.output is None: + raise ValueError("promotion requires --output and forbids --dry-run") + expected_hash = require_sha256( + args.expected_candidate_sha256, + context="--expected-candidate-sha256", + ) + raw = args.input.read_bytes() + actual_hash = sha256_bytes(raw) + if actual_hash != expected_hash: + raise ValueError("candidate file SHA-256 does not match explicit promotion hash") + candidate = _json_without_duplicate_keys(raw, context="candidate artifact") + if candidate.get("evidence", {}).get("phase") != args.phase: + raise ValueError("candidate phase does not match --phase") + frozen = promote_candidate(candidate, candidate_file_sha256=actual_hash) + validate_promotion_output(args.output) + atomic_write(args.output, canonical_json_bytes(frozen)) + print(sha256_bytes(canonical_json_bytes(frozen))) + return 0 + + if args.expected_candidate_sha256 is not None: + raise ValueError("--expected-candidate-sha256 is valid only with --promote") + revisions = { + "mbpp": args.mbpp_revision, + "pg19": args.pg19_revision, + "ruler": args.ruler_revision, + "humaneval_plus": args.humaneval_plus_revision, + } + if any(value is None for value in revisions.values()): + raise ValueError("all four dataset revision arguments are mandatory") + source = _json_without_duplicate_keys(args.input.read_bytes(), context="identity input") + if source.get("phase") != args.phase: + raise ValueError("input phase does not match --phase") + candidate = build_candidate(source, expected_revisions=revisions) # type: ignore[arg-type] + payload = canonical_json_bytes(candidate) + digest = sha256_bytes(payload) + if args.dry_run: + if args.output is not None: + raise ValueError("--dry-run forbids --output") + print(digest) + return 0 + if args.output is None: + raise ValueError("candidate resolution requires --output or --dry-run") + validate_quarantine_output(args.output) + atomic_write(args.output, payload) + print(digest) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (FileExistsError, PermissionError, ValueError) as error: + print(f"error: {error}", file=sys.stderr) + raise SystemExit(2) from error diff --git a/tests/test_resolve_static_q468_identity.py b/tests/test_resolve_static_q468_identity.py new file mode 100644 index 0000000..7073b3a --- /dev/null +++ b/tests/test_resolve_static_q468_identity.py @@ -0,0 +1,395 @@ +from __future__ import annotations + +import copy +import importlib.util +import json +from pathlib import Path +from typing import Any + +import pytest + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +SCRIPT_PATH = REPOSITORY_ROOT / "scripts" / "resolve_static_q468_identity.py" +SPEC = importlib.util.spec_from_file_location("resolve_static_q468_identity", SCRIPT_PATH) +assert SPEC is not None and SPEC.loader is not None +resolver = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(resolver) + +REVISIONS = { + "mbpp": resolver.MBPP_REVISION, + "pg19": resolver.PG19_REVISION, + "ruler": resolver.RULER_REVISION, + "humaneval_plus": resolver.HUMANEVAL_PLUS_REVISION, +} + + +def _hash(label: str) -> str: + return resolver.sha256_bytes(label.encode()) + + +def _datasets() -> list[dict[str, Any]]: + return [ + { + "key": "mbpp", + "dataset_id": resolver.MBPP_DATASET_ID, + "config": resolver.MBPP_CONFIG, + "revision": REVISIONS["mbpp"], + "split": "train", + "canonical_id_field": "task_id", + "canonical_id_manifest_sha256": resolver.mbpp_calibration_identity()[1], + "formatter_id": "recurquant.mbpp.v0.2", + "formatter_sha256": _hash("mbpp-formatter"), + }, + { + "key": "pg19", + "dataset_id": resolver.PG19_DATASET_ID, + "config": "default", + "revision": REVISIONS["pg19"], + "split": "validation", + "canonical_id_field": "book_id", + "canonical_id_manifest_sha256": _hash("pg19-id-manifest"), + "formatter_id": "recurquant.pg19.contiguous.v1", + "formatter_sha256": _hash("pg19-formatter"), + }, + { + "key": "ruler", + "dataset_id": resolver.RULER_SOURCE_ID, + "config": "official-generator", + "revision": REVISIONS["ruler"], + "split": "generated", + "canonical_id_field": "configuration_id", + "canonical_id_manifest_sha256": _hash("ruler-id-manifest"), + "formatter_id": "recurquant.ruler.official.v1", + "formatter_sha256": _hash("ruler-formatter"), + }, + { + "key": "humaneval_plus", + "dataset_id": resolver.HUMANEVAL_PLUS_DATASET_ID, + "config": "default", + "revision": REVISIONS["humaneval_plus"], + "split": "test", + "canonical_id_field": "task_id", + "canonical_id_manifest_sha256": _hash("humaneval-id-manifest"), + "formatter_id": "recurquant.humaneval-plus.v1", + "formatter_sha256": _hash("humaneval-formatter"), + }, + ] + + +def _tokenizer() -> dict[str, Any]: + return { + "source_id": resolver.PRIMARY_MODEL_ID, + "revision": resolver.PRIMARY_MODEL_REVISION, + "class": "Qwen2Tokenizer", + "transformers_version": resolver.TRANSFORMERS_VERSION, + "files": [ + {"name": "tokenizer.json", "sha256": _hash("tokenizer"), "size_bytes": 100}, + { + "name": "tokenizer_config.json", + "sha256": _hash("tokenizer-config"), + "size_bytes": 20, + }, + ], + } + + +def _tokenizer_manifest_hash() -> str: + files = sorted(_tokenizer()["files"], key=lambda item: item["name"]) + return resolver.sha256_bytes(resolver.canonical_json_bytes(files)) + + +def _record( + *, + family: str, + canonical_id: str, + config: str, + rank: int, + seed: int | None, + sequence_length: int, + prefill_stop: int, + scored_stop: int, +) -> dict[str, Any]: + namespace = { + "pg19": resolver.PG19_VALIDATION_NAMESPACE, + "ruler": resolver.CALIBRATION_SPLIT_NAMESPACE, + "humaneval_plus": resolver.HUMANEVAL_AB_NAMESPACE, + }[family] + label = f"{family}-{canonical_id}-{config}-{seed}-{sequence_length}" + return { + "family": family, + "canonical_id": canonical_id, + "config": config, + "selection_rank": rank, + "selection_sha256": resolver.selection_sha256(namespace, canonical_id), + "seed": seed, + "sequence_length": sequence_length, + "source_content_sha256": _hash(f"source-{label}"), + "formatted_content_sha256": _hash(f"formatted-{label}"), + "prompt_token_ids_sha256": _hash(f"prompt-tokens-{label}"), + "target_token_ids_sha256": _hash(f"target-tokens-{label}"), + "tokenizer_manifest_sha256": _tokenizer_manifest_hash(), + "token_span": { + "prefill_start": 0, + "prefill_stop": prefill_stop, + "scored_start": prefill_stop, + "scored_stop": scored_stop, + }, + "anchor_manifest_sha256": _hash(f"anchors-{label}"), + } + + +def _stage_a_source() -> dict[str, Any]: + records: list[dict[str, Any]] = [] + for rank in range(4): + records.append( + _record( + family="pg19", + canonical_id=f"book-{rank}", + config="default", + rank=rank, + seed=None, + sequence_length=4_224, + prefill_stop=4_096, + scored_stop=4_224, + ) + ) + for rank, family in enumerate(("niah_single_1", "niah_multikey_1", "vt", "cwe")): + records.append( + _record( + family="ruler", + canonical_id=f"{family}-4096-2339", + config=family, + rank=rank, + seed=2_339, + sequence_length=4_096, + prefill_stop=4_096, + scored_stop=4_100, + ) + ) + for rank in range(4): + records.append( + _record( + family="humaneval_plus", + canonical_id=f"HumanEval/{rank}", + config="default", + rank=rank, + seed=None, + sequence_length=160 + rank, + prefill_stop=64, + scored_stop=160 + rank, + ) + ) + for selected_family in ("pg19", "humaneval_plus"): + ranked = sorted( + (row for row in records if row["family"] == selected_family), + key=lambda row: (row["selection_sha256"], row["canonical_id"]), + ) + for rank, row in enumerate(ranked): + row["selection_rank"] = rank + return { + "schema": resolver.INPUT_SCHEMA, + "phase": "stage_a", + "datasets": _datasets(), + "tokenizer": _tokenizer(), + "records": list(reversed(records)), + "model_weights_loaded": False, + "calibration_binding": { + "identity_file_sha256": _hash("calibration-file"), + "canonical_identity_sha256": _hash("calibration-identity"), + "static_k29334_code_map_sha256": _hash("k29334"), + "static_k27030_code_map_sha256": _hash("k27030"), + "split_half_policy_manifest_sha256": _hash("split-half-policies"), + }, + } + + +def _write_json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(resolver.canonical_json_bytes(value)) + + +def test_stage_a_candidate_is_deterministic_and_complete() -> None: + source = _stage_a_source() + first = resolver.build_candidate(source, expected_revisions=REVISIONS) + second = resolver.build_candidate(copy.deepcopy(source), expected_revisions=REVISIONS) + + assert first == second + resolver.validate_candidate_artifact(first) + evidence = first["evidence"] + assert evidence["record_count"] == 12 + assert evidence["model_contracts"]["weights_loaded"] is False + assert evidence["protected_identity"] == { + "stage_b_read": False, + "stage_c_read": False, + "ordinary_tests_may_read_protected_content": False, + } + assert [row["family"] for row in evidence["records"]] == [ + *(["pg19"] * 4), + *(["ruler"] * 4), + *(["humaneval_plus"] * 4), + ] + assert evidence["tokenizer"]["file_manifest_sha256"] == _tokenizer_manifest_hash() + assert evidence["content_manifest_sha256"] == resolver.sha256_bytes( + resolver.canonical_json_bytes(evidence["records"]) + ) + + +def test_raw_content_and_unknown_fields_fail_closed() -> None: + source = _stage_a_source() + source["records"][0]["prompt"] = "raw protected text" + + with pytest.raises(ValueError, match="fields drifted"): + resolver.build_candidate(source, expected_revisions=REVISIONS) + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + ( + lambda source: source["tokenizer"].update({"revision": "f" * 40}), + "tokenizer revision", + ), + ( + lambda source: source["records"][0].update( + {"tokenizer_manifest_sha256": "0" * 64} + ), + "tokenizer manifest binding", + ), + ( + lambda source: source["records"][0].update({"selection_sha256": "0" * 64}), + "selection SHA-256", + ), + ( + lambda source: source.update({"model_weights_loaded": True}), + "before model weights", + ), + ( + lambda source: source["records"][0]["token_span"].update( + {"scored_start": 4_095} + ), + "contiguous", + ), + ], +) +def test_identity_contract_drift_fails_closed(mutation: Any, message: str) -> None: + source = _stage_a_source() + mutation(source) + + with pytest.raises(ValueError, match=message): + resolver.build_candidate(source, expected_revisions=REVISIONS) + + +def test_dataset_revision_must_match_explicit_cli_contract() -> None: + source = _stage_a_source() + source["datasets"][1]["revision"] = "9" * 40 + + with pytest.raises(ValueError, match="does not match the CLI contract"): + resolver.build_candidate(source, expected_revisions=REVISIONS) + + +def test_dry_run_writes_nothing(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + source_path = tmp_path / "source.json" + _write_json(source_path, _stage_a_source()) + + result = resolver.main( + [ + "--phase", + "stage_a", + "--input", + str(source_path), + "--dry-run", + "--mbpp-revision", + REVISIONS["mbpp"], + "--pg19-revision", + REVISIONS["pg19"], + "--ruler-revision", + REVISIONS["ruler"], + "--humaneval-plus-revision", + REVISIONS["humaneval_plus"], + ] + ) + + assert result == 0 + assert len(capsys.readouterr().out.strip()) == 64 + assert sorted(path.name for path in tmp_path.iterdir()) == ["source.json"] + + +def test_candidate_requires_quarantine_then_exact_hash_promotion(tmp_path: Path) -> None: + source_path = tmp_path / "source.json" + _write_json(source_path, _stage_a_source()) + candidate_path = tmp_path / ".quarantine" / "stage-a-candidate.json" + base_args = [ + "--phase", + "stage_a", + "--input", + str(source_path), + "--mbpp-revision", + REVISIONS["mbpp"], + "--pg19-revision", + REVISIONS["pg19"], + "--ruler-revision", + REVISIONS["ruler"], + "--humaneval-plus-revision", + REVISIONS["humaneval_plus"], + ] + + assert resolver.main([*base_args, "--output", str(candidate_path)]) == 0 + candidate_hash = resolver.sha256_bytes(candidate_path.read_bytes()) + frozen_path = tmp_path / "frozen" / "stage-a-identity.json" + assert ( + resolver.main( + [ + "--phase", + "stage_a", + "--input", + str(candidate_path), + "--output", + str(frozen_path), + "--promote", + "--expected-candidate-sha256", + candidate_hash, + ] + ) + == 0 + ) + frozen = json.loads(frozen_path.read_text(encoding="utf-8")) + assert frozen["evidence"]["status"] == "frozen" + assert frozen["evidence"]["promotion"]["candidate_file_sha256"] == candidate_hash + assert frozen["evidence"]["model_contracts"]["weights_loaded"] is False + + +def test_candidate_wrong_hash_cannot_be_promoted(tmp_path: Path) -> None: + candidate = resolver.build_candidate(_stage_a_source(), expected_revisions=REVISIONS) + candidate_path = tmp_path / ".quarantine" / "candidate.json" + _write_json(candidate_path, candidate) + + with pytest.raises(ValueError, match="does not match explicit promotion hash"): + resolver.main( + [ + "--phase", + "stage_a", + "--input", + str(candidate_path), + "--output", + str(tmp_path / "frozen" / "identity.json"), + "--promote", + "--expected-candidate-sha256", + "0" * 64, + ] + ) + + +@pytest.mark.parametrize("phase", ["stage_b", "stage_c"]) +def test_protected_phase_is_rejected_before_input_read(tmp_path: Path, phase: str) -> None: + nonexistent = tmp_path / "protected-content-that-must-not-be-read.json" + + with pytest.raises(PermissionError, match="before reading --input"): + resolver.main(["--phase", phase, "--input", str(nonexistent), "--dry-run"]) + + +def test_candidate_tampering_is_detected() -> None: + candidate = resolver.build_candidate(_stage_a_source(), expected_revisions=REVISIONS) + candidate["evidence"]["records"][0]["source_content_sha256"] = "0" * 64 + + with pytest.raises(ValueError, match="canonical evidence SHA-256"): + resolver.validate_candidate_artifact(candidate) From 73ec814cda96bce126d1ad519e409958e2226a6e Mon Sep 17 00:00:00 2001 From: HiddenTrojan <93521146+Labeeb2339@users.noreply.github.com> Date: Sun, 2 Aug 2026 09:12:21 +0800 Subject: [PATCH 02/24] Clarify Experiment 013 identity contracts --- ...EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md | 29 +++++++++++++++++-- scripts/resolve_static_q468_identity.py | 11 +++++++ tests/test_resolve_static_q468_identity.py | 8 ++++- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md index 329eb06..6a0375a 100644 --- a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md +++ b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md @@ -116,7 +116,7 @@ Policy fitting uses only these calibration sources: 1. the existing frozen public-evaluation v0.2 MBPP calibration population of 128 tasks; -2. 16 SHA-ranked eligible PG19 training books, with one non-overlapping +2. 16 SHA-ranked eligible PG19 training books, with one deterministic 2,304-token segment from each book; and 3. the four official NVIDIA RULER task families at lengths 2,048 and 4,096, with generator seeds 12,339 and 12,340. @@ -141,6 +141,26 @@ recurquant.experiment013.humaneval-plus.stage-a-b.v1\0 recurquant.experiment013.humaneval-plus.stage-c.v1\0 ``` +The canonical PG19 ID is the exact UTF-8 `url` field; the pinned PG19 schema +does not contain a `book_id` field. A training book is eligible when the pinned +tokenizer produces at least 2,304 tokens. Rank all 13,684 training URLs before +opening text, then inspect them in that fixed order only until 16 eligible +books have been accepted. For an accepted book with `N` tokens, define + +```text +M = N - 2304 +u = unsigned big-endian integer from the first 8 bytes of + SHA256("recurquant.experiment013.pg19.segment.v1\0" || UTF8(url)) +segment_start = u mod (M + 1) +segment_stop = segment_start + 2304 +``` + +No tokenizer special tokens are added. The same URL identity and eligibility +rule applies to validation, except eligibility requires at least 4,224 tokens +for the frozen 4,096-token prefill and 128 scored tokens. HumanEval+ uses the +exact `task_id` field. RULER uses the complete domain-separated configuration +identity that the later generator amendment must freeze. + Within each broad calibration family, and separately within each of RULER's four official subfamilies, SHA-rank canonical sequence IDs and alternate even and odd ranks into split halves A and B. Recompute the complete equation and @@ -201,9 +221,12 @@ changed under Experiment 013. Before any Stage-A quality result is opened, independently fit K29334 maps on split halves A and B. All three gates are conjunctive: -1. split-half Spearman rank correlation is at least `0.70`; +1. Spearman rank correlation between the two flattened K29334 precision-code + vectors is at least `0.70`, using average ranks for tied codes; a constant + vector is undefined and fails closed; 2. Q8-set Jaccard similarity is at least `0.50`; and -3. every recurrent layer's absolute mean-code shift is at most `0.25` bits. +3. every recurrent layer's absolute mean assigned-bitwidth shift is at most + `0.25` bits, where codes zero, one, and two map to 4, 6, and 8 bits. Failure stops the static candidate. The map may not be stabilized by changing the data, metric, seed, threshold, or aggregation after the failure is known. diff --git a/scripts/resolve_static_q468_identity.py b/scripts/resolve_static_q468_identity.py index 470c69b..14282ce 100644 --- a/scripts/resolve_static_q468_identity.py +++ b/scripts/resolve_static_q468_identity.py @@ -59,6 +59,12 @@ "ruler": RULER_REVISION, "humaneval_plus": HUMANEVAL_PLUS_REVISION, } +FROZEN_CANONICAL_ID_FIELDS: Final = { + "mbpp": "task_id", + "pg19": "url", + "ruler": "configuration_id", + "humaneval_plus": "task_id", +} PG19_TRAIN_NAMESPACE: Final = "recurquant.experiment013.pg19.train.v1\0" PG19_VALIDATION_NAMESPACE: Final = ( @@ -337,6 +343,11 @@ def _validate_dataset_contracts( context=f"datasets[{index}].formatter_sha256", ), } + if contract["canonical_id_field"] != FROZEN_CANONICAL_ID_FIELDS[key]: + raise ValueError( + f"{key} canonical ID field must be " + f"{FROZEN_CANONICAL_ID_FIELDS[key]!r}" + ) if key == "mbpp" and ( contract["dataset_id"] != MBPP_DATASET_ID or contract["config"] != MBPP_CONFIG diff --git a/tests/test_resolve_static_q468_identity.py b/tests/test_resolve_static_q468_identity.py index 7073b3a..b02f857 100644 --- a/tests/test_resolve_static_q468_identity.py +++ b/tests/test_resolve_static_q468_identity.py @@ -46,7 +46,7 @@ def _datasets() -> list[dict[str, Any]]: "config": "default", "revision": REVISIONS["pg19"], "split": "validation", - "canonical_id_field": "book_id", + "canonical_id_field": "url", "canonical_id_manifest_sha256": _hash("pg19-id-manifest"), "formatter_id": "recurquant.pg19.contiguous.v1", "formatter_sha256": _hash("pg19-formatter"), @@ -259,6 +259,12 @@ def test_raw_content_and_unknown_fields_fail_closed() -> None: lambda source: source["records"][0].update({"selection_sha256": "0" * 64}), "selection SHA-256", ), + ( + lambda source: source["datasets"][1].update( + {"canonical_id_field": "book_id"} + ), + "pg19 canonical ID field", + ), ( lambda source: source.update({"model_weights_loaded": True}), "before model weights", From 482685736cef076e31f50e2fb21a31b2ba8a400b Mon Sep 17 00:00:00 2001 From: HiddenTrojan <93521146+Labeeb2339@users.noreply.github.com> Date: Sun, 2 Aug 2026 09:26:21 +0800 Subject: [PATCH 03/24] Separate RULER category and sequence identity --- ...EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md | 77 +++++-- scripts/resolve_static_q468_identity.py | 213 ++++++++++++++++-- tests/test_resolve_static_q468_identity.py | 49 +++- 3 files changed, 295 insertions(+), 44 deletions(-) diff --git a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md index 6a0375a..8a77066 100644 --- a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md +++ b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md @@ -118,8 +118,9 @@ Policy fitting uses only these calibration sources: 128 tasks; 2. 16 SHA-ranked eligible PG19 training books, with one deterministic 2,304-token segment from each book; and -3. the four official NVIDIA RULER task families at lengths 2,048 and 4,096, - with generator seeds 12,339 and 12,340. +3. four sequences from each of NVIDIA RULER's four official task categories, + using configured lengths 2,048 and 4,096 and generator seeds 12,339 and + 12,340 as frozen below. Each workload family receives equal weight regardless of its number of tokens or examples. Within a family, examples receive equal weight. The exact @@ -157,12 +158,53 @@ segment_stop = segment_start + 2304 No tokenizer special tokens are added. The same URL identity and eligibility rule applies to validation, except eligibility requires at least 4,224 tokens -for the frozen 4,096-token prefill and 128 scored tokens. HumanEval+ uses the -exact `task_id` field. RULER uses the complete domain-separated configuration -identity that the later generator amendment must freeze. +for the frozen 4,096-token prefill and 128 scored tokens. For an accepted +validation book, replace `2304` by `4224` in the equation above and use the +independent namespace +`recurquant.experiment013.pg19.validation-segment.v1\0`; the first 4,096 +tokens of that slice are prefill and the last 128 are scored. HumanEval+ uses +the exact `task_id` field. RULER uses the complete domain-separated +configuration identity that the later generator amendment must freeze. + +RULER category and exact configuration are separate identity fields. The +pinned `scripts/synthetic.yaml` contains these 13 configurations: + +| Category | Exact configuration IDs | +| --- | --- | +| retrieval | `niah_single_1`, `niah_single_2`, `niah_single_3`, `niah_multikey_1`, `niah_multikey_2`, `niah_multikey_3`, `niah_multivalue`, `niah_multiquery` | +| multi-hop tracing | `vt` | +| aggregation | `cwe`, `fwe` | +| question answering | `qa_1`, `qa_2` | + +For calibration, order `(configured_length, seed)` as `(2048,12339)`, +`(2048,12340)`, `(4096,12339)`, `(4096,12340)`. Within each category, rank +its exact config IDs by lowercase SHA-256 of + +```text +"recurquant.experiment013.ruler.calibration-config.v1\0" || UTF8(config_id) +``` + +and cycle through that ranked list across the four ordered pairs. The resolved +schedule is: + +| Category | Exact configs in pair order | +| --- | --- | +| retrieval | `niah_multiquery`, `niah_multikey_2`, `niah_single_1`, `niah_multivalue` | +| multi-hop tracing | `vt`, `vt`, `vt`, `vt` | +| aggregation | `fwe`, `cwe`, `fwe`, `cwe` | +| question answering | `qa_1`, `qa_2`, `qa_1`, `qa_2` | + +This is exactly 16 RULER calibration sequences. It is a compute-bounded, +category-balanced calibration sample, not the RULER evaluation grid. + +RULER's configured length is not assumed to equal the actual token count: its +official generators reserve answer tokens and may emit a shorter tokenized +sequence. Identity records therefore bind `configured_length`, actual +`sequence_length`, prompt/scored half-open spans, and the generator's own +length receipt separately. Anchors use the actual processed token count only. Within each broad calibration family, and separately within each of RULER's -four official subfamilies, SHA-rank canonical sequence IDs and alternate even +four official categories, SHA-rank canonical sequence IDs and alternate even and odd ranks into split halves A and B. Recompute the complete equation and both exact-K allocations independently on each half. This produces deterministic halves without observing a quality result. @@ -192,8 +234,8 @@ x[e,p,r,b] = EMA_query_energy[e,p,r] ``` Mean anchors within each sequence. Mean sequences within MBPP and PG19. For -RULER, mean sequences within each of its four official families, then mean the -four family means. The final score is +RULER, mean sequences within each of its four official categories, then mean +the four category means. The final score is ```text D_b(r) = (D_MBPP,b(r) + D_PG19,b(r) + D_RULER,b(r)) / 3. @@ -245,9 +287,9 @@ Stage A contains exactly 12 examples: - the first four SHA-ranked eligible PG19 validation books, each using 4,096 prefill tokens followed by 128 scored tokens; -- one official RULER configuration from each of the four families at length - 4,096 and seed 2,339, scoring only the identity-bound official answer span; - and +- four RULER category representatives at configured length 4,096 and seed + 2,339: `niah_multiquery`, `vt`, `fwe`, and `qa_1`, scoring only each + identity-bound official answer span; and - the first four SHA-ranked HumanEval+ canonical IDs, scoring at most the first 128 canonical-solution tokens after the identity-bound prompt. @@ -268,7 +310,9 @@ does not reproduce, stop; do not proceed by reframing the oracle as optional. Stage B remains closed until Stage A and every identity gate pass. It contains: - the remaining 28 eligible PG19 validation books after Stage A; -- the remaining 44 frozen RULER configurations; and +- the remaining 48 configurations in the complete development grid of all 13 + exact RULER configs at configured length 4,096 and seeds 2,339 through + 2,342; and - the remaining 28 HumanEval+ tasks under the Stage-A/B ranking domain. Stage-B identities and token spans are unresolved protected placeholders in @@ -281,13 +325,14 @@ Stage C remains closed until the complete Stage-B decision is committed. It contains: - 32 SHA-ranked eligible PG19 test books; -- 48 frozen RULER configurations using seeds 3,339 through 3,342; and +- all 52 combinations of the 13 exact RULER configs at configured length 4,096 + and seeds 3,339 through 3,342; and - the next 32 HumanEval+ canonical IDs under the separate Stage-C confirmation hash domain. -The exact RULER configuration grid, IDs, and token spans remain unresolved -until a separate protected identity amendment is frozen. Stage C may not be -partially previewed. +The generated RULER IDs, auxiliary-source hashes, formatter hashes, actual +lengths, and token spans remain unresolved until a separate protected identity +amendment is frozen. Stage C may not be partially previewed. ## Methods and measurements diff --git a/scripts/resolve_static_q468_identity.py b/scripts/resolve_static_q468_identity.py index 14282ce..aa65996 100644 --- a/scripts/resolve_static_q468_identity.py +++ b/scripts/resolve_static_q468_identity.py @@ -80,6 +80,57 @@ CALIBRATION_SPLIT_NAMESPACE: Final = ( "recurquant.experiment013.calibration-split.v1\0" ) +RULER_CALIBRATION_SELECTION_NAMESPACE: Final = ( + "recurquant.experiment013.ruler.calibration-sequence.v1\0" +) +RULER_STAGE_A_SELECTION_NAMESPACE: Final = ( + "recurquant.experiment013.ruler.stage-a-sequence.v1\0" +) +RULER_CATEGORIES: Final = ( + "retrieval", + "multi_hop_tracing", + "aggregation", + "question_answering", +) +RULER_CONFIG_CATEGORY: Final = { + "niah_single_1": "retrieval", + "niah_single_2": "retrieval", + "niah_single_3": "retrieval", + "niah_multikey_1": "retrieval", + "niah_multikey_2": "retrieval", + "niah_multikey_3": "retrieval", + "niah_multivalue": "retrieval", + "niah_multiquery": "retrieval", + "vt": "multi_hop_tracing", + "cwe": "aggregation", + "fwe": "aggregation", + "qa_1": "question_answering", + "qa_2": "question_answering", +} +RULER_CALIBRATION_SCHEDULE: Final = ( + ("retrieval", "niah_multiquery", 2_048, 12_339), + ("retrieval", "niah_multikey_2", 2_048, 12_340), + ("retrieval", "niah_single_1", 4_096, 12_339), + ("retrieval", "niah_multivalue", 4_096, 12_340), + ("multi_hop_tracing", "vt", 2_048, 12_339), + ("multi_hop_tracing", "vt", 2_048, 12_340), + ("multi_hop_tracing", "vt", 4_096, 12_339), + ("multi_hop_tracing", "vt", 4_096, 12_340), + ("aggregation", "fwe", 2_048, 12_339), + ("aggregation", "cwe", 2_048, 12_340), + ("aggregation", "fwe", 4_096, 12_339), + ("aggregation", "cwe", 4_096, 12_340), + ("question_answering", "qa_1", 2_048, 12_339), + ("question_answering", "qa_2", 2_048, 12_340), + ("question_answering", "qa_1", 4_096, 12_339), + ("question_answering", "qa_2", 4_096, 12_340), +) +RULER_STAGE_A_SCHEDULE: Final = ( + ("retrieval", "niah_multiquery", 4_096, 2_339), + ("multi_hop_tracing", "vt", 4_096, 2_339), + ("aggregation", "fwe", 4_096, 2_339), + ("question_answering", "qa_1", 4_096, 2_339), +) CLAIM_BOUNDARY: Final = ( "This artifact freezes Experiment 013 data and tokenizer identity only. " @@ -115,7 +166,10 @@ "selection_rank", "selection_sha256", "seed", + "configured_length", "sequence_length", + "ruler_category", + "generator_receipt_sha256", "source_content_sha256", "formatted_content_sha256", "prompt_token_ids_sha256", @@ -234,9 +288,11 @@ def calibration_split_key(record: Mapping[str, Any]) -> str: identity = "\0".join( ( str(record["family"]), + str(record["ruler_category"]), str(record["config"]), str(record["canonical_id"]), str(record["seed"]), + str(record["configured_length"]), str(record["sequence_length"]), ) ) @@ -424,12 +480,16 @@ def _selection_namespace(phase: str, family: str) -> str | None: return None if family == "pg19": return PG19_TRAIN_NAMESPACE + if family == "ruler": + return RULER_CALIBRATION_SELECTION_NAMESPACE return CALIBRATION_SPLIT_NAMESPACE if phase == "stage_a": if family == "pg19": return PG19_VALIDATION_NAMESPACE if family == "humaneval_plus": return HUMANEVAL_AB_NAMESPACE + if family == "ruler": + return RULER_STAGE_A_SELECTION_NAMESPACE return CALIBRATION_SPLIT_NAMESPACE raise ValueError(f"unsupported phase: {phase}") @@ -453,6 +513,51 @@ def _normalize_record( seed = None if seed_value is None else require_int( seed_value, context=f"records[{index}].seed" ) + configured_value = item["configured_length"] + configured_length = ( + None + if configured_value is None + else require_int( + configured_value, + context=f"records[{index}].configured_length", + minimum=1, + ) + ) + category_value = item["ruler_category"] + ruler_category = ( + None + if category_value is None + else require_string( + category_value, + context=f"records[{index}].ruler_category", + ) + ) + generator_receipt_value = item["generator_receipt_sha256"] + generator_receipt_sha256 = ( + None + if generator_receipt_value is None + else require_sha256( + generator_receipt_value, + context=f"records[{index}].generator_receipt_sha256", + ) + ) + if family == "ruler": + expected_category = RULER_CONFIG_CATEGORY.get(config) + if expected_category is None or ruler_category != expected_category: + raise ValueError( + f"records[{index}] RULER config/category binding drifted" + ) + if configured_length is None or generator_receipt_sha256 is None: + raise ValueError( + f"records[{index}] RULER configured length and generator receipt are required" + ) + elif any( + value is not None + for value in (configured_length, ruler_category, generator_receipt_sha256) + ): + raise ValueError( + f"records[{index}] non-RULER rows cannot carry RULER-only fields" + ) sequence_length = require_int( item["sequence_length"], context=f"records[{index}].sequence_length", @@ -479,8 +584,13 @@ def _normalize_record( or normalized_span["prefill_stop"] != normalized_span["scored_start"] or normalized_span["prefill_stop"] < 1 or normalized_span["scored_stop"] < normalized_span["scored_start"] + or normalized_span["scored_stop"] != sequence_length ): raise ValueError(f"records[{index}] token span is not contiguous and canonical") + if configured_length is not None and sequence_length > configured_length: + raise ValueError( + f"records[{index}] actual sequence exceeds the RULER configured length" + ) positions = anchor_positions(sequence_length) return { "family": family, @@ -489,7 +599,10 @@ def _normalize_record( "selection_rank": rank, "selection_sha256": expected_selection, "seed": seed, + "configured_length": configured_length, "sequence_length": sequence_length, + "ruler_category": ruler_category, + "generator_receipt_sha256": generator_receipt_sha256, "source_content_sha256": require_sha256( item["source_content_sha256"], context=f"records[{index}].source_content_sha256", @@ -523,8 +636,12 @@ def _record_sort_key(record: Mapping[str, Any]) -> tuple[Any, ...]: int(record["selection_rank"]), str(record["selection_sha256"]), str(record["canonical_id"]), + "" if record["ruler_category"] is None else str(record["ruler_category"]), str(record["config"]), -1 if record["seed"] is None else int(record["seed"]), + -1 + if record["configured_length"] is None + else int(record["configured_length"]), int(record["sequence_length"]), ) @@ -557,25 +674,50 @@ def _validate_calibration_records(records: Sequence[Mapping[str, Any]]) -> None: "scored_stop": 2_304, }: raise ValueError("calibration PG19 span must cover exactly 2,304 tokens") - ruler_tuples = { - (str(row["config"]), int(row["sequence_length"]), int(row["seed"])) + if sorted(int(row["selection_rank"]) for row in grouped["ruler"]) != list( + range(16) + ): + raise ValueError("calibration RULER ranks must be exactly 0..15") + _validate_sha_rank_order(grouped["ruler"], context="calibration RULER") + actual_ruler_schedule = { + ( + str(row["ruler_category"]), + str(row["config"]), + int(row["configured_length"]), + int(row["seed"]), + ) for row in grouped["ruler"] } - if len({row["config"] for row in grouped["ruler"]}) != 4 or len(ruler_tuples) != 16: - raise ValueError("calibration RULER must contain four unique official families") - expected_pairs = {(length, seed) for length in (2_048, 4_096) for seed in (12_339, 12_340)} - for config in {str(row["config"]) for row in grouped["ruler"]}: - actual_pairs = { - (int(row["sequence_length"]), int(row["seed"])) - for row in grouped["ruler"] - if row["config"] == config - } - if actual_pairs != expected_pairs: - raise ValueError(f"RULER family {config!r} does not have the frozen grid") + if actual_ruler_schedule != set(RULER_CALIBRATION_SCHEDULE): + raise ValueError("calibration RULER rows differ from the frozen 16-sequence schedule") + if { + category: sum(row["ruler_category"] == category for row in grouped["ruler"]) + for category in RULER_CATEGORIES + } != {category: 4 for category in RULER_CATEGORIES}: + raise ValueError("calibration RULER must contain four sequences per category") + for row in grouped["ruler"]: + span = row["token_span"] + if span != { + "prefill_start": 0, + "prefill_stop": row["sequence_length"], + "scored_start": row["sequence_length"], + "scored_stop": row["sequence_length"], + }: + raise ValueError("calibration RULER must anchor the actual prompt tokens only") for row in grouped["mbpp"]: if row["seed"] is not None: raise ValueError("MBPP calibration records cannot have a generator seed") - identities = [(row["family"], row["canonical_id"], row["config"]) for row in records] + identities = [ + ( + row["family"], + row["canonical_id"], + row["ruler_category"], + row["config"], + row["configured_length"], + row["seed"], + ) + for row in records + ] if len(identities) != len(set(identities)): raise ValueError("calibration canonical identities are not unique") @@ -603,12 +745,24 @@ def _validate_stage_a_records(records: Sequence[Mapping[str, Any]]) -> None: "scored_stop": 4_224, }: raise ValueError("Stage-A PG19 token span drifted") - if len({row["config"] for row in grouped["ruler"]}) != 4: - raise ValueError("Stage-A RULER must contain four distinct official families") + if sorted(int(row["selection_rank"]) for row in grouped["ruler"]) != list( + range(4) + ): + raise ValueError("Stage-A RULER ranks must be exactly 0..3") + _validate_sha_rank_order(grouped["ruler"], context="Stage-A RULER") + actual_ruler_schedule = { + ( + str(row["ruler_category"]), + str(row["config"]), + int(row["configured_length"]), + int(row["seed"]), + ) + for row in grouped["ruler"] + } + if actual_ruler_schedule != set(RULER_STAGE_A_SCHEDULE): + raise ValueError("Stage-A RULER rows differ from the frozen category representatives") for row in grouped["ruler"]: span = row["token_span"] - if row["seed"] != 2_339 or row["sequence_length"] != 4_096: - raise ValueError("Stage-A RULER must use length 4,096 and seed 2,339") if span["scored_stop"] <= span["scored_start"]: raise ValueError("Stage-A RULER answer span cannot be empty") for row in grouped["humaneval_plus"]: @@ -618,7 +772,15 @@ def _validate_stage_a_records(records: Sequence[Mapping[str, Any]]) -> None: raise ValueError("Stage-A HumanEval+ must score 1..128 solution tokens") if row["sequence_length"] != span["scored_stop"]: raise ValueError("Stage-A HumanEval+ sequence length must equal span stop") - identities = [(row["family"], row["canonical_id"], row["config"]) for row in records] + identities = [ + ( + row["family"], + row["canonical_id"], + row["ruler_category"], + row["config"], + ) + for row in records + ] if len(identities) != len(set(identities)): raise ValueError("Stage-A canonical identities are not unique") @@ -629,8 +791,13 @@ def _split_half_manifest(records: Sequence[Mapping[str, Any]]) -> dict[str, Any] groups.append(("mbpp", [row for row in records if row["family"] == "mbpp"])) groups.append(("pg19", [row for row in records if row["family"] == "pg19"])) ruler_rows = [row for row in records if row["family"] == "ruler"] - for config in sorted({str(row["config"]) for row in ruler_rows}): - groups.append((f"ruler:{config}", [row for row in ruler_rows if row["config"] == config])) + for category in RULER_CATEGORIES: + groups.append( + ( + f"ruler:{category}", + [row for row in ruler_rows if row["ruler_category"] == category], + ) + ) for group, rows in groups: ranked = sorted(rows, key=lambda row: (calibration_split_key(row), _record_sort_key(row))) for rank, row in enumerate(ranked): @@ -638,7 +805,11 @@ def _split_half_manifest(records: Sequence[Mapping[str, Any]]) -> dict[str, Any] { "group": group, "canonical_id": row["canonical_id"], + "ruler_category": row["ruler_category"], "config": row["config"], + "configured_length": row["configured_length"], + "sequence_length": row["sequence_length"], + "seed": row["seed"], "rank": rank, "half": "a" if rank % 2 == 0 else "b", "rank_sha256": calibration_split_key(row), diff --git a/tests/test_resolve_static_q468_identity.py b/tests/test_resolve_static_q468_identity.py index b02f857..700a368 100644 --- a/tests/test_resolve_static_q468_identity.py +++ b/tests/test_resolve_static_q468_identity.py @@ -108,10 +108,12 @@ def _record( sequence_length: int, prefill_stop: int, scored_stop: int, + configured_length: int | None = None, + ruler_category: str | None = None, ) -> dict[str, Any]: namespace = { "pg19": resolver.PG19_VALIDATION_NAMESPACE, - "ruler": resolver.CALIBRATION_SPLIT_NAMESPACE, + "ruler": resolver.RULER_STAGE_A_SELECTION_NAMESPACE, "humaneval_plus": resolver.HUMANEVAL_AB_NAMESPACE, }[family] label = f"{family}-{canonical_id}-{config}-{seed}-{sequence_length}" @@ -122,7 +124,12 @@ def _record( "selection_rank": rank, "selection_sha256": resolver.selection_sha256(namespace, canonical_id), "seed": seed, + "configured_length": configured_length, "sequence_length": sequence_length, + "ruler_category": ruler_category, + "generator_receipt_sha256": ( + _hash(f"generator-receipt-{label}") if family == "ruler" else None + ), "source_content_sha256": _hash(f"source-{label}"), "formatted_content_sha256": _hash(f"formatted-{label}"), "prompt_token_ids_sha256": _hash(f"prompt-tokens-{label}"), @@ -153,17 +160,25 @@ def _stage_a_source() -> dict[str, Any]: scored_stop=4_224, ) ) - for rank, family in enumerate(("niah_single_1", "niah_multikey_1", "vt", "cwe")): + ruler_rows = ( + ("retrieval", "niah_multiquery"), + ("multi_hop_tracing", "vt"), + ("aggregation", "fwe"), + ("question_answering", "qa_1"), + ) + for rank, (category, config) in enumerate(ruler_rows): records.append( _record( family="ruler", - canonical_id=f"{family}-4096-2339", - config=family, + canonical_id=f"{config}-4096-2339", + config=config, rank=rank, seed=2_339, sequence_length=4_096, - prefill_stop=4_096, - scored_stop=4_100, + prefill_stop=4_092, + scored_stop=4_096, + configured_length=4_096, + ruler_category=category, ) ) for rank in range(4): @@ -179,7 +194,7 @@ def _stage_a_source() -> dict[str, Any]: scored_stop=160 + rank, ) ) - for selected_family in ("pg19", "humaneval_plus"): + for selected_family in ("pg19", "ruler", "humaneval_plus"): ranked = sorted( (row for row in records if row["family"] == selected_family), key=lambda row: (row["selection_sha256"], row["canonical_id"]), @@ -293,6 +308,26 @@ def test_dataset_revision_must_match_explicit_cli_contract() -> None: resolver.build_candidate(source, expected_revisions=REVISIONS) +def test_ruler_category_config_and_actual_length_are_independently_bound() -> None: + source = _stage_a_source() + ruler = next(row for row in source["records"] if row["family"] == "ruler") + ruler["ruler_category"] = "aggregation" + with pytest.raises(ValueError, match="config/category binding"): + resolver.build_candidate(source, expected_revisions=REVISIONS) + + source = _stage_a_source() + ruler = next(row for row in source["records"] if row["family"] == "ruler") + ruler["configured_length"] = 4_095 + with pytest.raises(ValueError, match="exceeds the RULER configured length"): + resolver.build_candidate(source, expected_revisions=REVISIONS) + + source = _stage_a_source() + pg19 = next(row for row in source["records"] if row["family"] == "pg19") + pg19["ruler_category"] = "retrieval" + with pytest.raises(ValueError, match="non-RULER rows"): + resolver.build_candidate(source, expected_revisions=REVISIONS) + + def test_dry_run_writes_nothing(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: source_path = tmp_path / "source.json" _write_json(source_path, _stage_a_source()) From ea984104d8a1d7ef8b743c28d0ee12c019c539a8 Mon Sep 17 00:00:00 2001 From: HiddenTrojan <93521146+Labeeb2339@users.noreply.github.com> Date: Sun, 2 Aug 2026 09:46:17 +0800 Subject: [PATCH 04/24] Implement static Q468 and exact-byte Q48 codecs --- src/recurquant/static_q468.py | 2068 +++++++++++++++++++++++++++++++++ tests/test_static_q468.py | 790 +++++++++++++ 2 files changed, 2858 insertions(+) create mode 100644 src/recurquant/static_q468.py create mode 100644 tests/test_static_q468.py diff --git a/src/recurquant/static_q468.py b/src/recurquant/static_q468.py new file mode 100644 index 0000000..92fac72 --- /dev/null +++ b/src/recurquant/static_q468.py @@ -0,0 +1,2068 @@ +"""Static, packed RHT Q4/Q6/Q8 policies for Experiment 013. + +The policy artifact freezes one precision code for every recurrent-state row. +Codes are selected once from calibration distortions and are not recomputed on +the inference path. Packed states own three integer payload pools, one FP16 +scale per row, a two-bit code stream, and a uint16 offset into the selected +pool. No FP32 state mirror or runtime score tensor is resident. + +This is a correctness-first reference format. The explicit offsets are the +addressing contract for a packed-native kernel; this module does not claim that +the Python materialization path is an optimized runtime. +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import math +import os +import re +import tempfile +from collections.abc import Mapping +from dataclasses import dataclass +from fractions import Fraction +from pathlib import Path +from typing import Any, Literal, TypeAlias + +import numpy as np +import torch + +from .mixed_quantization import ( + PackedMixedQuantizedTensor, + _pack_precision_mask, + _unpack_precision_mask, + quantize_pack_mixed, +) +from .multibit_policy import allocate_exact_multibit_codes_fast +from .multibit_quantization import ( + INT4_PRECISION_CODE, + INT6_PRECISION_CODE, + INT8_PRECISION_CODE, + PackedMultiBitQuantizedTensor, + _pack_precision_codes, + _unpack_precision_codes, + quantize_pack_multibit, +) +from .quantization import QuantizationSpec +from .rht import RHT_SEED, right_rht_decode, right_rht_encode + +STATIC_Q468_POLICY_SCHEMA = "recurquant.static-rht-q468-policy.v1" +STATIC_Q48_POLICY_SCHEMA = "recurquant.static-rht-q48-policy.v1" +STATIC_Q468_POLICY_REVISION = "experiment-013-static-policy-v1" +STATIC_Q48_POLICY_REVISION = "experiment-013-static-q48-policy-v1" +STATIC_Q468_ALLOCATOR_REVISION = "exact-multibit-o-nlogn-v1" +STATIC_Q468_CODEC_REVISION = "rht-q468-pools-u16-offsets-v1" +STATIC_Q48_SELECTOR_REVISION = "exact-top-q4-to-q8-benefit-v1" +STATIC_Q48_CODEC_REVISION = "rht-q48-pools-u16-offsets-v1" + +STATIC_Q468_PRIMARY_METHOD = "rht_q468_static_k29334" +STATIC_Q468_ABLATION_METHOD = "rht_q468_static_k27030" +STATIC_Q48_COMPARATOR_METHOD = "rht_q48_static_p14739" + +PRIMARY_MODEL_ID = "Qwen/Qwen3.5-0.8B-Base" +PRIMARY_MODEL_REVISION = "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68" +PRIMARY_TOKENIZER_ID = PRIMARY_MODEL_ID +PRIMARY_TOKENIZER_REVISION = PRIMARY_MODEL_REVISION +FROZEN_TRANSFORMERS_VERSION = "5.14.1" + +FROZEN_STATIC_Q468_PRIMARY_STEPS = 29_334 +FROZEN_STATIC_Q468_ABLATION_STEPS = 27_030 +FROZEN_STATIC_Q48_PROMOTIONS = 14_739 +FROZEN_STATELEASE_RESIDENT_BYTES = 3_454_664 +FROZEN_RECURRENT_LAYER_INDICES = ( + 0, + 1, + 2, + 4, + 5, + 6, + 8, + 9, + 10, + 12, + 13, + 14, + 16, + 17, + 18, + 20, + 21, + 22, +) + +StaticCodec: TypeAlias = Literal["q468", "q48"] +_SHA256_RE = re.compile(r"[0-9a-f]{64}") +_GIT_REVISION_RE = re.compile(r"[0-9a-f]{40}") +_METHOD_RE = re.compile(r"[a-z0-9][a-z0-9_.-]{2,127}") +_VERSION_RE = re.compile(r"[0-9]+\.[0-9]+\.[0-9]+(?:[a-z0-9.+-]*)?") + + +def _canonical_json(value: object) -> bytes: + return json.dumps( + value, + ensure_ascii=True, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def _sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _validate_sha256(value: object, *, name: str) -> str: + if not isinstance(value, str) or _SHA256_RE.fullmatch(value) is None: + raise ValueError(f"{name} must be a lowercase 64-character SHA-256 hex digest") + return value + + +def _validate_revision(value: object, *, name: str) -> str: + if not isinstance(value, str) or not value or len(value) > 128: + raise ValueError(f"{name} must be a non-empty string of at most 128 characters") + if not value.isascii() or any(character.isspace() for character in value): + raise ValueError(f"{name} must be printable ASCII without whitespace") + return value + + +def _validate_identity(value: object, *, name: str) -> str: + if not isinstance(value, str) or not value or len(value) > 256: + raise ValueError(f"{name} must be a non-empty string of at most 256 characters") + if ( + value != value.strip() + or not value.isascii() + or any(ord(character) < 32 for character in value) + ): + raise ValueError(f"{name} must be stripped printable ASCII") + return value + + +def _validate_git_revision(value: object, *, name: str) -> str: + if not isinstance(value, str) or _GIT_REVISION_RE.fullmatch(value) is None: + raise ValueError(f"{name} must be an immutable lowercase 40-hex Git revision") + return value + + +def _validate_transformers_version(value: object) -> str: + if not isinstance(value, str) or _VERSION_RE.fullmatch(value) is None: + raise ValueError("transformers_version must be a pinned semantic version") + return value + + +def _validate_integer(value: object, *, name: str, minimum: int = 0) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{name} must be an integer") + if value < minimum: + raise ValueError(f"{name} must be at least {minimum}") + return value + + +def _storage_bytes(tensor: torch.Tensor) -> int: + return int(tensor.untyped_storage().nbytes()) + + +def _validate_owned_tensor(tensor: torch.Tensor, *, name: str) -> None: + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"{name} must be a torch.Tensor") + if tensor.device.type == "meta": + raise ValueError(f"{name} must be materialized") + if not tensor.is_contiguous(): + raise ValueError(f"{name} must be contiguous") + if tensor.storage_offset() != 0: + raise ValueError(f"{name} must have zero storage offset") + logical_bytes = tensor.numel() * tensor.element_size() + if _storage_bytes(tensor) != logical_bytes: + raise ValueError( + f"{name} owns {_storage_bytes(tensor)} bytes but exposes {logical_bytes} bytes" + ) + + +@dataclass(frozen=True, slots=True) +class StaticRhtQ468Geometry: + """Complete row geometry and resident-byte target for one static policy.""" + + layer_indices: tuple[int, ...] + heads: int + key_rows: int + value_width: int + target_resident_bytes: int + + def __post_init__(self) -> None: + if not isinstance(self.layer_indices, tuple) or not self.layer_indices: + raise ValueError("layer_indices must be a non-empty tuple") + if any( + isinstance(index, bool) or not isinstance(index, int) or index < 0 + for index in self.layer_indices + ): + raise ValueError("layer_indices must contain non-negative integers") + if len(set(self.layer_indices)) != len(self.layer_indices): + raise ValueError("layer_indices must be unique") + _validate_integer(self.heads, name="heads", minimum=1) + _validate_integer(self.key_rows, name="key_rows", minimum=1) + width = _validate_integer(self.value_width, name="value_width", minimum=1) + if width & (width - 1): + raise ValueError("value_width must be a power of two for the frozen RHT") + if width % 4: + raise ValueError("value_width must make Q4, Q6, and Q8 rows byte aligned") + _validate_integer( + self.target_resident_bytes, + name="target_resident_bytes", + minimum=1, + ) + if self.total_rows > 1 << 16: + raise ValueError("uint16 pool offsets support at most 65,536 rows") + + @property + def layers(self) -> int: + return len(self.layer_indices) + + @property + def rows_per_layer(self) -> int: + return self.heads * self.key_rows + + @property + def total_rows(self) -> int: + return self.layers * self.rows_per_layer + + @property + def state_elements(self) -> int: + return self.total_rows * self.value_width + + def canonical_dict(self) -> dict[str, object]: + return { + "heads": self.heads, + "key_rows": self.key_rows, + "layer_indices": list(self.layer_indices), + "target_resident_bytes": self.target_resident_bytes, + "value_width": self.value_width, + } + + @property + def geometry_sha256(self) -> str: + return _sha256_bytes(_canonical_json(self.canonical_dict())) + + +FROZEN_QWEN35_STATIC_Q468_GEOMETRY = StaticRhtQ468Geometry( + layer_indices=FROZEN_RECURRENT_LAYER_INDICES, + heads=16, + key_rows=128, + value_width=128, + target_resident_bytes=FROZEN_STATELEASE_RESIDENT_BYTES, +) + + +@dataclass(frozen=True, slots=True) +class StaticRhtByteLedger: + """Exact resident tensor arithmetic for one static packed format.""" + + method_id: str + codec: StaticCodec + selected_units: int + payload_bytes: int + scale_bytes: int + precision_code_bytes: int + pool_offset_bytes: int + data_bytes: int + alignment_bytes: int + resident_bytes: int + target_resident_bytes: int + budget_delta_bytes: int + exact_budget_eligible: bool + + def evidence_dict(self) -> dict[str, object]: + return { + "alignment_bytes": self.alignment_bytes, + "budget_delta_bytes": self.budget_delta_bytes, + "codec": self.codec, + "data_bytes": self.data_bytes, + "exact_budget_eligible": self.exact_budget_eligible, + "method_id": self.method_id, + "payload_bytes": self.payload_bytes, + "pool_offset_bytes": self.pool_offset_bytes, + "precision_code_bytes": self.precision_code_bytes, + "resident_bytes": self.resident_bytes, + "scale_bytes": self.scale_bytes, + "selected_units": self.selected_units, + "target_resident_bytes": self.target_resident_bytes, + } + + +def _finish_ledger( + *, + method_id: str, + codec: StaticCodec, + selected_units: int, + payload_bytes: int, + scale_bytes: int, + precision_code_bytes: int, + pool_offset_bytes: int, + target_resident_bytes: int, +) -> StaticRhtByteLedger: + data_bytes = payload_bytes + scale_bytes + precision_code_bytes + pool_offset_bytes + # The frozen exact-budget layouts reserve the remaining eight bytes. An + # under-budget ablation keeps its natural size instead of adding filler. + alignment_bytes = 8 if data_bytes + 8 == target_resident_bytes else 0 + resident_bytes = data_bytes + alignment_bytes + delta = target_resident_bytes - resident_bytes + return StaticRhtByteLedger( + method_id=method_id, + codec=codec, + selected_units=selected_units, + payload_bytes=payload_bytes, + scale_bytes=scale_bytes, + precision_code_bytes=precision_code_bytes, + pool_offset_bytes=pool_offset_bytes, + data_bytes=data_bytes, + alignment_bytes=alignment_bytes, + resident_bytes=resident_bytes, + target_resident_bytes=target_resident_bytes, + budget_delta_bytes=delta, + exact_budget_eligible=delta == 0, + ) + + +def static_q468_byte_ledger( + geometry: StaticRhtQ468Geometry, + marginal_steps: int, + *, + method_id: str | None = None, +) -> StaticRhtByteLedger: + """Return physical Q4/Q6/Q8 bytes for a frozen static code budget.""" + + if not isinstance(geometry, StaticRhtQ468Geometry): + raise TypeError("geometry must be a StaticRhtQ468Geometry") + steps = _validate_integer(marginal_steps, name="marginal_steps") + if steps > 2 * geometry.total_rows: + raise ValueError("marginal_steps exceeds two steps per state row") + selected_method = method_id or f"rht_q468_static_k{steps}" + if _METHOD_RE.fullmatch(selected_method) is None: + raise ValueError("method_id must use lowercase identifier characters") + + base_q4 = geometry.state_elements * 4 // 8 + marginal_payload = steps * geometry.value_width * 2 // 8 + return _finish_ledger( + method_id=selected_method, + codec="q468", + selected_units=steps, + payload_bytes=base_q4 + marginal_payload, + scale_bytes=geometry.total_rows * 2, + precision_code_bytes=math.ceil(geometry.total_rows * 2 / 8), + pool_offset_bytes=geometry.total_rows * 2, + target_resident_bytes=geometry.target_resident_bytes, + ) + + +def static_q48_byte_ledger( + geometry: StaticRhtQ468Geometry, + promoted_rows: int, + *, + method_id: str | None = None, +) -> StaticRhtByteLedger: + """Return physical Q4/Q8 comparator bytes with a one-bit precision map.""" + + if not isinstance(geometry, StaticRhtQ468Geometry): + raise TypeError("geometry must be a StaticRhtQ468Geometry") + promotions = _validate_integer(promoted_rows, name="promoted_rows") + if promotions > geometry.total_rows: + raise ValueError("promoted_rows exceeds the number of state rows") + selected_method = method_id or f"rht_q48_static_p{promotions}" + if _METHOD_RE.fullmatch(selected_method) is None: + raise ValueError("method_id must use lowercase identifier characters") + + base_q4 = geometry.state_elements * 4 // 8 + promoted_payload = promotions * geometry.value_width * 4 // 8 + return _finish_ledger( + method_id=selected_method, + codec="q48", + selected_units=promotions, + payload_bytes=base_q4 + promoted_payload, + scale_bytes=geometry.total_rows * 2, + precision_code_bytes=math.ceil(geometry.total_rows / 8), + pool_offset_bytes=geometry.total_rows * 2, + target_resident_bytes=geometry.target_resident_bytes, + ) + + +def frozen_static_byte_accounting() -> dict[str, dict[str, object]]: + """Return the three frozen Experiment 013 byte ledgers.""" + + geometry = FROZEN_QWEN35_STATIC_Q468_GEOMETRY + return { + STATIC_Q468_PRIMARY_METHOD: static_q468_byte_ledger( + geometry, + FROZEN_STATIC_Q468_PRIMARY_STEPS, + method_id=STATIC_Q468_PRIMARY_METHOD, + ).evidence_dict(), + STATIC_Q468_ABLATION_METHOD: static_q468_byte_ledger( + geometry, + FROZEN_STATIC_Q468_ABLATION_STEPS, + method_id=STATIC_Q468_ABLATION_METHOD, + ).evidence_dict(), + STATIC_Q48_COMPARATOR_METHOD: static_q48_byte_ledger( + geometry, + FROZEN_STATIC_Q48_PROMOTIONS, + method_id=STATIC_Q48_COMPARATOR_METHOD, + ).evidence_dict(), + } + + +def _expected_pool_offsets(codes: torch.Tensor) -> tuple[torch.Tensor, tuple[int, int, int]]: + flat = codes.detach().to(device="cpu", dtype=torch.uint8).reshape(-1) + offsets = torch.empty(flat.numel(), dtype=torch.int64) + counts: list[int] = [] + for code in (INT4_PRECISION_CODE, INT6_PRECISION_CODE, INT8_PRECISION_CODE): + mask = flat == code + count = int(mask.sum().item()) + counts.append(count) + if count: + offsets[mask] = torch.arange(count, dtype=torch.int64) + return offsets.to(torch.uint16).contiguous(), (counts[0], counts[1], counts[2]) + + +def _tensor_b64(tensor: torch.Tensor, *, dtype: np.dtype[Any]) -> str: + array = tensor.detach().to("cpu").contiguous().numpy().astype(dtype, copy=False) + return base64.b64encode(array.tobytes(order="C")).decode("ascii") + + +def _decode_b64(value: object, *, name: str) -> bytes: + if not isinstance(value, str): + raise TypeError(f"{name} must be a base64 string") + try: + return base64.b64decode(value.encode("ascii"), validate=True) + except (UnicodeEncodeError, ValueError) as error: + raise ValueError(f"{name} is not canonical base64") from error + + +def _atomic_publish_new(path: Path, payload: bytes) -> None: + """Atomically publish ``payload`` while refusing to replace any path entry.""" + + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", + suffix=".tmp", + dir=path.parent, + ) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + try: + os.link(temporary, path) + except FileExistsError as error: + raise FileExistsError(f"refusing to overwrite policy artifact: {path}") from error + finally: + if temporary.exists(): + temporary.unlink() + + +@dataclass(frozen=True, slots=True) +class StaticRhtQ468Policy: + """Canonical static precision map and packed-native addressing metadata.""" + + method_id: str + policy_revision: str + allocator_revision: str + codec_revision: str + model_id: str + model_revision: str + tokenizer_id: str + tokenizer_revision: str + tokenizer_manifest_sha256: str + transformers_version: str + identity_artifact_sha256: str + source_commit: str + geometry: StaticRhtQ468Geometry + calibration_manifest_sha256: str + calibration_scores_sha256: str + marginal_steps: int + rht_seed: int + packed_precision_codes: torch.Tensor + pool_offsets: torch.Tensor + pool_counts: tuple[int, int, int] + + def __post_init__(self) -> None: + if not isinstance(self.method_id, str) or _METHOD_RE.fullmatch(self.method_id) is None: + raise ValueError("method_id must use lowercase identifier characters") + _validate_revision(self.policy_revision, name="policy_revision") + _validate_revision(self.allocator_revision, name="allocator_revision") + _validate_revision(self.codec_revision, name="codec_revision") + if self.policy_revision != STATIC_Q468_POLICY_REVISION: + raise ValueError("unsupported static Q468 policy revision") + if self.allocator_revision != STATIC_Q468_ALLOCATOR_REVISION: + raise ValueError("unsupported static Q468 allocator revision") + if self.codec_revision != STATIC_Q468_CODEC_REVISION: + raise ValueError("unsupported static Q468 codec revision") + _validate_identity(self.model_id, name="model_id") + _validate_git_revision(self.model_revision, name="model_revision") + _validate_identity(self.tokenizer_id, name="tokenizer_id") + _validate_git_revision(self.tokenizer_revision, name="tokenizer_revision") + _validate_sha256( + self.tokenizer_manifest_sha256, + name="tokenizer_manifest_sha256", + ) + _validate_transformers_version(self.transformers_version) + _validate_sha256( + self.identity_artifact_sha256, + name="identity_artifact_sha256", + ) + _validate_git_revision(self.source_commit, name="source_commit") + if not isinstance(self.geometry, StaticRhtQ468Geometry): + raise TypeError("geometry must be a StaticRhtQ468Geometry") + _validate_sha256( + self.calibration_manifest_sha256, + name="calibration_manifest_sha256", + ) + _validate_sha256( + self.calibration_scores_sha256, + name="calibration_scores_sha256", + ) + steps = _validate_integer(self.marginal_steps, name="marginal_steps") + if steps > 2 * self.geometry.total_rows: + raise ValueError("marginal_steps exceeds two steps per state row") + frozen_steps = { + STATIC_Q468_PRIMARY_METHOD: FROZEN_STATIC_Q468_PRIMARY_STEPS, + STATIC_Q468_ABLATION_METHOD: FROZEN_STATIC_Q468_ABLATION_STEPS, + }.get(self.method_id) + if frozen_steps is not None: + if self.geometry != FROZEN_QWEN35_STATIC_Q468_GEOMETRY: + raise ValueError("reserved static Q468 method requires the frozen geometry") + if steps != frozen_steps: + raise ValueError("reserved static Q468 method has the wrong exact-K budget") + if ( + self.model_id != PRIMARY_MODEL_ID + or self.model_revision != PRIMARY_MODEL_REVISION + or self.tokenizer_id != PRIMARY_TOKENIZER_ID + or self.tokenizer_revision != PRIMARY_TOKENIZER_REVISION + or self.transformers_version != FROZEN_TRANSFORMERS_VERSION + ): + raise ValueError("reserved static Q468 method requires the frozen model identity") + if self.rht_seed != RHT_SEED: + raise ValueError(f"rht_seed must equal the codec seed {RHT_SEED}") + + _validate_owned_tensor(self.packed_precision_codes, name="packed_precision_codes") + _validate_owned_tensor(self.pool_offsets, name="pool_offsets") + if self.packed_precision_codes.device != self.pool_offsets.device: + raise ValueError("precision codes and pool offsets must share one device") + if ( + self.packed_precision_codes.dtype != torch.uint8 + or self.packed_precision_codes.ndim != 1 + ): + raise TypeError("packed_precision_codes must be one-dimensional torch.uint8") + if self.pool_offsets.dtype != torch.uint16 or self.pool_offsets.ndim != 1: + raise TypeError("pool_offsets must be one-dimensional torch.uint16") + if self.pool_offsets.numel() != self.geometry.total_rows: + raise ValueError("pool_offsets must contain one uint16 value per state row") + + codes = _unpack_precision_codes( + self.packed_precision_codes, + self.geometry.total_rows, + ).reshape(-1) + if int(codes.to(torch.int64).sum().item()) != steps: + raise ValueError("precision-code marginal sum does not match marginal_steps") + expected_offsets, expected_counts = _expected_pool_offsets(codes) + if not isinstance(self.pool_counts, tuple) or len(self.pool_counts) != 3: + raise TypeError("pool_counts must be a three-integer tuple") + for index, count in enumerate(self.pool_counts): + _validate_integer(count, name=f"pool_counts[{index}]") + if self.pool_counts != expected_counts: + raise ValueError( + f"pool_counts {self.pool_counts} do not match precision codes {expected_counts}" + ) + if sum(self.pool_counts) != self.geometry.total_rows: + raise ValueError("pool_counts do not cover every state row") + if not torch.equal(self.pool_offsets.to("cpu"), expected_offsets): + raise ValueError("pool_offsets are not canonical per-pool prefix offsets") + + def precision_codes(self) -> torch.Tensor: + return _unpack_precision_codes( + self.packed_precision_codes, + self.geometry.total_rows, + ).reshape(self.geometry.layers, self.geometry.heads, self.geometry.key_rows) + + def clone_to(self, device: torch.device | str) -> StaticRhtQ468Policy: + """Copy the physical policy tensors to ``device`` without aliasing input storage.""" + + return StaticRhtQ468Policy( + method_id=self.method_id, + policy_revision=self.policy_revision, + allocator_revision=self.allocator_revision, + codec_revision=self.codec_revision, + model_id=self.model_id, + model_revision=self.model_revision, + tokenizer_id=self.tokenizer_id, + tokenizer_revision=self.tokenizer_revision, + tokenizer_manifest_sha256=self.tokenizer_manifest_sha256, + transformers_version=self.transformers_version, + identity_artifact_sha256=self.identity_artifact_sha256, + source_commit=self.source_commit, + geometry=self.geometry, + calibration_manifest_sha256=self.calibration_manifest_sha256, + calibration_scores_sha256=self.calibration_scores_sha256, + marginal_steps=self.marginal_steps, + rht_seed=self.rht_seed, + packed_precision_codes=self.packed_precision_codes.detach().to(device).clone(), + pool_offsets=self.pool_offsets.detach().to(device).clone(), + pool_counts=self.pool_counts, + ) + + def _content_dict(self) -> dict[str, object]: + return { + "allocator_revision": self.allocator_revision, + "calibration_manifest_sha256": self.calibration_manifest_sha256, + "calibration_scores_sha256": self.calibration_scores_sha256, + "code_map_sha256": self.code_map_sha256, + "codec_revision": self.codec_revision, + "geometry": self.geometry.canonical_dict(), + "geometry_sha256": self.geometry.geometry_sha256, + "identity_artifact_sha256": self.identity_artifact_sha256, + "marginal_steps": self.marginal_steps, + "method_id": self.method_id, + "model_id": self.model_id, + "model_revision": self.model_revision, + "packed_precision_codes_b64": _tensor_b64( + self.packed_precision_codes, + dtype=np.dtype("u1"), + ), + "policy_revision": self.policy_revision, + "pool_counts": list(self.pool_counts), + "pool_offsets_le_b64": _tensor_b64( + self.pool_offsets, + dtype=np.dtype(" str: + digest = hashlib.sha256() + digest.update(b"recurquant.static-q468-code-map.v1\0") + digest.update(bytes.fromhex(self.geometry.geometry_sha256)) + digest.update(self.marginal_steps.to_bytes(8, "little", signed=False)) + digest.update( + self.packed_precision_codes.detach().to("cpu").contiguous().numpy().tobytes() + ) + return digest.hexdigest() + + @property + def pool_offsets_sha256(self) -> str: + offsets = ( + self.pool_offsets.detach() + .to("cpu") + .contiguous() + .numpy() + .astype(" str: + return _sha256_bytes(_canonical_json(self._content_dict())) + + def evidence_dict(self) -> dict[str, object]: + ledger = static_q468_byte_ledger( + self.geometry, + self.marginal_steps, + method_id=self.method_id, + ) + return { + "allocator_revision": self.allocator_revision, + "calibration_manifest_sha256": self.calibration_manifest_sha256, + "calibration_scores_sha256": self.calibration_scores_sha256, + "code_map_sha256": self.code_map_sha256, + "codec_revision": self.codec_revision, + "geometry_sha256": self.geometry.geometry_sha256, + "identity_artifact_sha256": self.identity_artifact_sha256, + "ledger": ledger.evidence_dict(), + "marginal_steps": self.marginal_steps, + "method_id": self.method_id, + "model_id": self.model_id, + "model_revision": self.model_revision, + "policy_revision": self.policy_revision, + "policy_sha256": self.policy_sha256, + "pool_counts": list(self.pool_counts), + "pool_offsets_sha256": self.pool_offsets_sha256, + "rht_seed": self.rht_seed, + "schema": STATIC_Q468_POLICY_SCHEMA, + "source_commit": self.source_commit, + "tokenizer_id": self.tokenizer_id, + "tokenizer_manifest_sha256": self.tokenizer_manifest_sha256, + "tokenizer_revision": self.tokenizer_revision, + "transformers_version": self.transformers_version, + } + + +def _normalize_static_distortions( + d4: torch.Tensor, + d6: torch.Tensor, + d8: torch.Tensor, + *, + geometry: StaticRhtQ468Geometry, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + normalized: list[torch.Tensor] = [] + expected_shape = tuple(d4.shape) if isinstance(d4, torch.Tensor) else None + for name, tensor in (("D4", d4), ("D6", d6), ("D8", d8)): + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"{name} must be a torch.Tensor") + if tensor.numel() != geometry.total_rows: + raise ValueError(f"{name} must contain exactly {geometry.total_rows} row distortions") + if tuple(tensor.shape) != expected_shape: + raise ValueError("D4, D6, and D8 must have identical shapes") + if not tensor.is_floating_point(): + raise TypeError(f"{name} must use a floating-point dtype") + if tensor.device.type == "meta": + raise ValueError(f"{name} must be materialized") + if not torch.isfinite(tensor).all().item() or (tensor < 0).any().item(): + raise ValueError(f"{name} must contain finite, non-negative values") + normalized.append(tensor.detach().reshape(1, geometry.total_rows)) + return normalized[0], normalized[1], normalized[2] + + +def static_q468_distortion_sha256( + d4: torch.Tensor, + d6: torch.Tensor, + d8: torch.Tensor, + *, + geometry: StaticRhtQ468Geometry, +) -> str: + """Hash canonical CPU-FP64 calibration distortions and their geometry.""" + + values = _normalize_static_distortions(d4, d6, d8, geometry=geometry) + digest = hashlib.sha256() + digest.update(b"recurquant.static-q468-distortions.v1\0") + digest.update(_canonical_json(geometry.canonical_dict())) + for label, tensor in zip((b"D4\0", b"D6\0", b"D8\0"), values, strict=True): + digest.update(label) + array = tensor.to(device="cpu", dtype=torch.float64).contiguous().numpy() + digest.update(array.astype(" StaticRhtQ468Policy: + """Build the deterministic exact-allocation policy from calibration losses.""" + + normalized = _normalize_static_distortions(d4, d6, d8, geometry=geometry) + scores_sha256 = static_q468_distortion_sha256(*normalized, geometry=geometry) + if calibration_scores_sha256 is not None: + _validate_sha256(calibration_scores_sha256, name="calibration_scores_sha256") + if calibration_scores_sha256 != scores_sha256: + raise ValueError("calibration_scores_sha256 does not match supplied distortions") + codes = allocate_exact_multibit_codes_fast( + *normalized, + marginal_steps=marginal_steps, + ).reshape(-1) + offsets, counts = _expected_pool_offsets(codes) + return StaticRhtQ468Policy( + method_id=method_id or f"rht_q468_static_k{marginal_steps}", + policy_revision=policy_revision, + allocator_revision=allocator_revision, + codec_revision=codec_revision, + model_id=model_id, + model_revision=model_revision, + tokenizer_id=tokenizer_id, + tokenizer_revision=tokenizer_revision, + tokenizer_manifest_sha256=tokenizer_manifest_sha256, + transformers_version=transformers_version, + identity_artifact_sha256=identity_artifact_sha256, + source_commit=source_commit, + geometry=geometry, + calibration_manifest_sha256=calibration_manifest_sha256, + calibration_scores_sha256=scores_sha256, + marginal_steps=marginal_steps, + rht_seed=rht_seed, + packed_precision_codes=_pack_precision_codes(codes), + pool_offsets=offsets, + pool_counts=counts, + ) + + +def serialize_static_rht_q468_policy(policy: StaticRhtQ468Policy) -> bytes: + """Serialize a policy as canonical, self-hashing JSON bytes.""" + + if not isinstance(policy, StaticRhtQ468Policy): + raise TypeError("policy must be a StaticRhtQ468Policy") + content = policy._content_dict() + envelope = { + "content": content, + "policy_sha256": _sha256_bytes(_canonical_json(content)), + "schema": STATIC_Q468_POLICY_SCHEMA, + } + return _canonical_json(envelope) + b"\n" + + +def _strict_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON key: {key}") + result[key] = value + return result + + +def _expect_keys(value: object, *, name: str, expected: set[str]) -> dict[str, object]: + if not isinstance(value, dict): + raise TypeError(f"{name} must be a JSON object") + actual = set(value) + if actual != expected: + missing = sorted(expected - actual) + extra = sorted(actual - expected) + raise ValueError(f"{name} keys differ; missing={missing}, extra={extra}") + return value + + +def deserialize_static_rht_q468_policy(data: bytes) -> StaticRhtQ468Policy: + """Load and independently verify canonical policy bytes.""" + + if not isinstance(data, bytes): + raise TypeError("serialized policy must be bytes") + try: + root = json.loads(data.decode("utf-8"), object_pairs_hook=_strict_object) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError("serialized policy is not valid UTF-8 JSON") from error + envelope = _expect_keys( + root, + name="policy envelope", + expected={"content", "policy_sha256", "schema"}, + ) + if envelope["schema"] != STATIC_Q468_POLICY_SCHEMA: + raise ValueError("unsupported static Q468 policy schema") + declared_digest = _validate_sha256(envelope["policy_sha256"], name="policy_sha256") + content = _expect_keys( + envelope["content"], + name="policy content", + expected={ + "allocator_revision", + "calibration_manifest_sha256", + "calibration_scores_sha256", + "code_map_sha256", + "codec_revision", + "geometry", + "geometry_sha256", + "identity_artifact_sha256", + "marginal_steps", + "method_id", + "model_id", + "model_revision", + "packed_precision_codes_b64", + "policy_revision", + "pool_counts", + "pool_offsets_le_b64", + "pool_offsets_sha256", + "rht_seed", + "source_commit", + "tokenizer_id", + "tokenizer_manifest_sha256", + "tokenizer_revision", + "transformers_version", + }, + ) + if _sha256_bytes(_canonical_json(content)) != declared_digest: + raise ValueError("policy_sha256 does not authenticate policy content") + + geometry_dict = _expect_keys( + content["geometry"], + name="geometry", + expected={ + "heads", + "key_rows", + "layer_indices", + "target_resident_bytes", + "value_width", + }, + ) + layer_indices = geometry_dict["layer_indices"] + if not isinstance(layer_indices, list): + raise TypeError("geometry.layer_indices must be a list") + geometry = StaticRhtQ468Geometry( + layer_indices=tuple(layer_indices), + heads=geometry_dict["heads"], + key_rows=geometry_dict["key_rows"], + value_width=geometry_dict["value_width"], + target_resident_bytes=geometry_dict["target_resident_bytes"], + ) + if content["geometry_sha256"] != geometry.geometry_sha256: + raise ValueError("geometry_sha256 does not authenticate geometry") + + code_bytes = _decode_b64( + content["packed_precision_codes_b64"], + name="packed_precision_codes_b64", + ) + offset_bytes = _decode_b64( + content["pool_offsets_le_b64"], + name="pool_offsets_le_b64", + ) + expected_code_bytes = math.ceil(geometry.total_rows * 2 / 8) + if len(code_bytes) != expected_code_bytes: + raise ValueError("packed precision code byte length does not match geometry") + if len(offset_bytes) != geometry.total_rows * 2: + raise ValueError("pool offset byte length does not match geometry") + packed_codes = torch.from_numpy(np.frombuffer(code_bytes, dtype="u1").copy()) + pool_offsets = torch.from_numpy( + np.frombuffer(offset_bytes, dtype=" None: + """Atomically publish canonical policy bytes without replacing ``path``.""" + + _atomic_publish_new(Path(path), serialize_static_rht_q468_policy(policy)) + + +def load_static_rht_q468_policy(path: str | Path) -> StaticRhtQ468Policy: + """Read and verify a canonical policy artifact from ``path``.""" + + return deserialize_static_rht_q468_policy(Path(path).read_bytes()) + + +def verify_static_rht_q468_policy( + policy: StaticRhtQ468Policy, + *, + expected_policy_sha256: str | None = None, +) -> dict[str, object]: + """Revalidate a policy and return its independently checkable evidence.""" + + if not isinstance(policy, StaticRhtQ468Policy): + raise TypeError("policy must be a StaticRhtQ468Policy") + # Reconstructing exercises every fail-closed dataclass invariant again. + verified = deserialize_static_rht_q468_policy(serialize_static_rht_q468_policy(policy)) + if expected_policy_sha256 is not None: + _validate_sha256(expected_policy_sha256, name="expected_policy_sha256") + if verified.policy_sha256 != expected_policy_sha256: + raise ValueError("verified policy SHA-256 does not match the expected digest") + return verified.evidence_dict() + + +def allocate_exact_q48_mask( + d4: torch.Tensor, + d8: torch.Tensor, + *, + promoted_rows: int, +) -> torch.Tensor: + """Select the exact fixed-cardinality Q8 set with a stable row-order tie rule.""" + + if not isinstance(d4, torch.Tensor) or not isinstance(d8, torch.Tensor): + raise TypeError("D4 and D8 must be torch.Tensor values") + if d4.numel() == 0 or tuple(d4.shape) != tuple(d8.shape): + raise ValueError("D4 and D8 must have one identical non-empty shape") + for name, tensor in (("D4", d4), ("D8", d8)): + if not tensor.is_floating_point(): + raise TypeError(f"{name} must use a floating-point dtype") + if tensor.device.type == "meta": + raise ValueError(f"{name} must be materialized") + if not torch.isfinite(tensor).all().item() or (tensor < 0).any().item(): + raise ValueError(f"{name} must contain finite, non-negative values") + promotions = _validate_integer(promoted_rows, name="promoted_rows") + if promotions > d4.numel(): + raise ValueError("promoted_rows exceeds the number of distortion rows") + + flat_d4 = d4.detach().to(device="cpu", dtype=torch.float64).reshape(-1).tolist() + flat_d8 = d8.detach().to(device="cpu", dtype=torch.float64).reshape(-1).tolist() + benefits = [ + Fraction.from_float(value4) - Fraction.from_float(value8) + for value4, value8 in zip(flat_d4, flat_d8, strict=True) + ] + ranked = sorted(range(len(benefits)), key=lambda index: (-benefits[index], index)) + mask = torch.zeros(len(benefits), dtype=torch.bool) + if promotions: + mask[ranked[:promotions]] = True + return mask.reshape(d4.shape) + + +def _expected_binary_pool_offsets( + mask: torch.Tensor, +) -> tuple[torch.Tensor, tuple[int, int]]: + flat = mask.detach().to(device="cpu", dtype=torch.bool).reshape(-1) + offsets = torch.empty(flat.numel(), dtype=torch.int64) + low_count = int((~flat).sum().item()) + high_count = int(flat.sum().item()) + if low_count: + offsets[~flat] = torch.arange(low_count, dtype=torch.int64) + if high_count: + offsets[flat] = torch.arange(high_count, dtype=torch.int64) + return offsets.to(torch.uint16).contiguous(), (low_count, high_count) + + +def _validate_packed_binary_mask(packed: torch.Tensor, *, total_rows: int) -> None: + _validate_owned_tensor(packed, name="packed_precision_mask") + if packed.dtype != torch.uint8 or packed.ndim != 1: + raise TypeError("packed_precision_mask must be one-dimensional torch.uint8") + expected_bytes = math.ceil(total_rows / 8) + if packed.numel() != expected_bytes: + raise ValueError( + f"packed_precision_mask must contain {expected_bytes} bytes, got {packed.numel()}" + ) + used_bits = total_rows % 8 + if used_bits and packed.numel(): + unused_mask = 0xFF ^ ((1 << used_bits) - 1) + if int(packed[-1].item()) & unused_mask: + raise ValueError("unused precision-mask padding bits must be zero") + + +def _normalize_static_q48_distortions( + d4: torch.Tensor, + d8: torch.Tensor, + *, + geometry: StaticRhtQ468Geometry, +) -> tuple[torch.Tensor, torch.Tensor]: + if not isinstance(geometry, StaticRhtQ468Geometry): + raise TypeError("geometry must be a StaticRhtQ468Geometry") + normalized: list[torch.Tensor] = [] + expected_shape = tuple(d4.shape) if isinstance(d4, torch.Tensor) else None + for name, tensor in (("D4", d4), ("D8", d8)): + if not isinstance(tensor, torch.Tensor): + raise TypeError(f"{name} must be a torch.Tensor") + if tensor.numel() != geometry.total_rows: + raise ValueError(f"{name} must contain exactly {geometry.total_rows} row distortions") + if tuple(tensor.shape) != expected_shape: + raise ValueError("D4 and D8 must have identical shapes") + if not tensor.is_floating_point(): + raise TypeError(f"{name} must use a floating-point dtype") + if tensor.device.type == "meta": + raise ValueError(f"{name} must be materialized") + if not torch.isfinite(tensor).all().item() or (tensor < 0).any().item(): + raise ValueError(f"{name} must contain finite, non-negative values") + normalized.append(tensor.detach().reshape(1, geometry.total_rows)) + return normalized[0], normalized[1] + + +def static_q48_distortion_sha256( + d4: torch.Tensor, + d8: torch.Tensor, + *, + geometry: StaticRhtQ468Geometry, +) -> str: + """Hash canonical CPU-FP64 Q4/Q8 calibration distortions and geometry.""" + + values = _normalize_static_q48_distortions(d4, d8, geometry=geometry) + digest = hashlib.sha256() + digest.update(b"recurquant.static-q48-distortions.v1\0") + digest.update(_canonical_json(geometry.canonical_dict())) + for label, tensor in zip((b"D4\0", b"D8\0"), values, strict=True): + digest.update(label) + array = tensor.to(device="cpu", dtype=torch.float64).contiguous().numpy() + digest.update(array.astype(" None: + if not isinstance(self.method_id, str) or _METHOD_RE.fullmatch(self.method_id) is None: + raise ValueError("method_id must use lowercase identifier characters") + _validate_revision(self.policy_revision, name="policy_revision") + _validate_revision(self.selector_revision, name="selector_revision") + _validate_revision(self.codec_revision, name="codec_revision") + if self.policy_revision != STATIC_Q48_POLICY_REVISION: + raise ValueError("unsupported static Q48 policy revision") + if self.selector_revision != STATIC_Q48_SELECTOR_REVISION: + raise ValueError("unsupported static Q48 selector revision") + if self.codec_revision != STATIC_Q48_CODEC_REVISION: + raise ValueError("unsupported static Q48 codec revision") + _validate_identity(self.model_id, name="model_id") + _validate_git_revision(self.model_revision, name="model_revision") + _validate_identity(self.tokenizer_id, name="tokenizer_id") + _validate_git_revision(self.tokenizer_revision, name="tokenizer_revision") + _validate_sha256( + self.tokenizer_manifest_sha256, + name="tokenizer_manifest_sha256", + ) + _validate_transformers_version(self.transformers_version) + _validate_sha256(self.identity_artifact_sha256, name="identity_artifact_sha256") + _validate_git_revision(self.source_commit, name="source_commit") + if not isinstance(self.geometry, StaticRhtQ468Geometry): + raise TypeError("geometry must be a StaticRhtQ468Geometry") + _validate_sha256( + self.calibration_manifest_sha256, + name="calibration_manifest_sha256", + ) + _validate_sha256(self.calibration_scores_sha256, name="calibration_scores_sha256") + promotions = _validate_integer(self.promoted_rows, name="promoted_rows") + if promotions > self.geometry.total_rows: + raise ValueError("promoted_rows exceeds the number of state rows") + if self.method_id == STATIC_Q48_COMPARATOR_METHOD: + if self.geometry != FROZEN_QWEN35_STATIC_Q468_GEOMETRY: + raise ValueError("reserved static Q48 method requires the frozen geometry") + if promotions != FROZEN_STATIC_Q48_PROMOTIONS: + raise ValueError("reserved static Q48 method has the wrong exact-P budget") + if ( + self.model_id != PRIMARY_MODEL_ID + or self.model_revision != PRIMARY_MODEL_REVISION + or self.tokenizer_id != PRIMARY_TOKENIZER_ID + or self.tokenizer_revision != PRIMARY_TOKENIZER_REVISION + or self.transformers_version != FROZEN_TRANSFORMERS_VERSION + ): + raise ValueError("reserved static Q48 method requires the frozen model identity") + if self.rht_seed != RHT_SEED: + raise ValueError(f"rht_seed must equal the codec seed {RHT_SEED}") + + _validate_packed_binary_mask( + self.packed_precision_mask, + total_rows=self.geometry.total_rows, + ) + _validate_owned_tensor(self.pool_offsets, name="pool_offsets") + if self.packed_precision_mask.device != self.pool_offsets.device: + raise ValueError("precision mask and pool offsets must share one device") + if self.pool_offsets.dtype != torch.uint16 or self.pool_offsets.ndim != 1: + raise TypeError("pool_offsets must be one-dimensional torch.uint16") + if self.pool_offsets.numel() != self.geometry.total_rows: + raise ValueError("pool_offsets must contain one uint16 value per state row") + + mask = self.high_precision_mask().reshape(-1) + if int(mask.sum().item()) != promotions: + raise ValueError("precision-mask population does not match promoted_rows") + expected_offsets, expected_counts = _expected_binary_pool_offsets(mask) + if not isinstance(self.pool_counts, tuple) or len(self.pool_counts) != 2: + raise TypeError("pool_counts must be a two-integer tuple") + for index, count in enumerate(self.pool_counts): + _validate_integer(count, name=f"pool_counts[{index}]") + if self.pool_counts != expected_counts: + raise ValueError( + f"pool_counts {self.pool_counts} do not match precision mask {expected_counts}" + ) + if sum(self.pool_counts) != self.geometry.total_rows: + raise ValueError("pool_counts do not cover every state row") + if not torch.equal(self.pool_offsets.to("cpu"), expected_offsets): + raise ValueError("pool_offsets are not canonical per-pool prefix offsets") + + def high_precision_mask(self) -> torch.Tensor: + return _unpack_precision_mask( + self.packed_precision_mask, + self.geometry.total_rows, + ).reshape(self.geometry.layers, self.geometry.heads, self.geometry.key_rows) + + def clone_to(self, device: torch.device | str) -> StaticRhtQ48Policy: + return StaticRhtQ48Policy( + method_id=self.method_id, + policy_revision=self.policy_revision, + selector_revision=self.selector_revision, + codec_revision=self.codec_revision, + model_id=self.model_id, + model_revision=self.model_revision, + tokenizer_id=self.tokenizer_id, + tokenizer_revision=self.tokenizer_revision, + tokenizer_manifest_sha256=self.tokenizer_manifest_sha256, + transformers_version=self.transformers_version, + identity_artifact_sha256=self.identity_artifact_sha256, + source_commit=self.source_commit, + geometry=self.geometry, + calibration_manifest_sha256=self.calibration_manifest_sha256, + calibration_scores_sha256=self.calibration_scores_sha256, + promoted_rows=self.promoted_rows, + rht_seed=self.rht_seed, + packed_precision_mask=self.packed_precision_mask.detach().to(device).clone(), + pool_offsets=self.pool_offsets.detach().to(device).clone(), + pool_counts=self.pool_counts, + ) + + @property + def mask_sha256(self) -> str: + digest = hashlib.sha256() + digest.update(b"recurquant.static-q48-mask.v1\0") + digest.update(bytes.fromhex(self.geometry.geometry_sha256)) + digest.update(self.promoted_rows.to_bytes(8, "little", signed=False)) + digest.update( + self.packed_precision_mask.detach().to("cpu").contiguous().numpy().tobytes() + ) + return digest.hexdigest() + + @property + def pool_offsets_sha256(self) -> str: + offsets = ( + self.pool_offsets.detach() + .to("cpu") + .contiguous() + .numpy() + .astype(" dict[str, object]: + return { + "calibration_manifest_sha256": self.calibration_manifest_sha256, + "calibration_scores_sha256": self.calibration_scores_sha256, + "codec_revision": self.codec_revision, + "geometry": self.geometry.canonical_dict(), + "geometry_sha256": self.geometry.geometry_sha256, + "identity_artifact_sha256": self.identity_artifact_sha256, + "mask_sha256": self.mask_sha256, + "method_id": self.method_id, + "model_id": self.model_id, + "model_revision": self.model_revision, + "packed_precision_mask_b64": _tensor_b64( + self.packed_precision_mask, + dtype=np.dtype("u1"), + ), + "policy_revision": self.policy_revision, + "pool_counts": list(self.pool_counts), + "pool_offsets_le_b64": _tensor_b64( + self.pool_offsets, + dtype=np.dtype(" str: + return _sha256_bytes(_canonical_json(self._content_dict())) + + def evidence_dict(self) -> dict[str, object]: + ledger = static_q48_byte_ledger( + self.geometry, + self.promoted_rows, + method_id=self.method_id, + ) + return { + "calibration_manifest_sha256": self.calibration_manifest_sha256, + "calibration_scores_sha256": self.calibration_scores_sha256, + "codec_revision": self.codec_revision, + "geometry_sha256": self.geometry.geometry_sha256, + "identity_artifact_sha256": self.identity_artifact_sha256, + "ledger": ledger.evidence_dict(), + "mask_sha256": self.mask_sha256, + "method_id": self.method_id, + "model_id": self.model_id, + "model_revision": self.model_revision, + "policy_revision": self.policy_revision, + "policy_sha256": self.policy_sha256, + "pool_counts": list(self.pool_counts), + "pool_offsets_sha256": self.pool_offsets_sha256, + "promoted_rows": self.promoted_rows, + "rht_seed": self.rht_seed, + "schema": STATIC_Q48_POLICY_SCHEMA, + "selector_revision": self.selector_revision, + "source_commit": self.source_commit, + "tokenizer_id": self.tokenizer_id, + "tokenizer_manifest_sha256": self.tokenizer_manifest_sha256, + "tokenizer_revision": self.tokenizer_revision, + "transformers_version": self.transformers_version, + } + + +def build_static_rht_q48_policy( + d4: torch.Tensor, + d8: torch.Tensor, + *, + geometry: StaticRhtQ468Geometry, + promoted_rows: int, + calibration_manifest_sha256: str, + identity_artifact_sha256: str, + tokenizer_manifest_sha256: str, + source_commit: str, + calibration_scores_sha256: str | None = None, + method_id: str | None = None, + policy_revision: str = STATIC_Q48_POLICY_REVISION, + selector_revision: str = STATIC_Q48_SELECTOR_REVISION, + codec_revision: str = STATIC_Q48_CODEC_REVISION, + model_id: str = PRIMARY_MODEL_ID, + model_revision: str = PRIMARY_MODEL_REVISION, + tokenizer_id: str = PRIMARY_TOKENIZER_ID, + tokenizer_revision: str = PRIMARY_TOKENIZER_REVISION, + transformers_version: str = FROZEN_TRANSFORMERS_VERSION, + rht_seed: int = RHT_SEED, +) -> StaticRhtQ48Policy: + """Build the deterministic exact-cardinality Q4/Q8 comparator policy.""" + + normalized = _normalize_static_q48_distortions(d4, d8, geometry=geometry) + scores_sha256 = static_q48_distortion_sha256(*normalized, geometry=geometry) + if calibration_scores_sha256 is not None: + _validate_sha256(calibration_scores_sha256, name="calibration_scores_sha256") + if calibration_scores_sha256 != scores_sha256: + raise ValueError("calibration_scores_sha256 does not match supplied distortions") + mask = allocate_exact_q48_mask( + *normalized, + promoted_rows=promoted_rows, + ).reshape(-1) + offsets, counts = _expected_binary_pool_offsets(mask) + return StaticRhtQ48Policy( + method_id=method_id or f"rht_q48_static_p{promoted_rows}", + policy_revision=policy_revision, + selector_revision=selector_revision, + codec_revision=codec_revision, + model_id=model_id, + model_revision=model_revision, + tokenizer_id=tokenizer_id, + tokenizer_revision=tokenizer_revision, + tokenizer_manifest_sha256=tokenizer_manifest_sha256, + transformers_version=transformers_version, + identity_artifact_sha256=identity_artifact_sha256, + source_commit=source_commit, + geometry=geometry, + calibration_manifest_sha256=calibration_manifest_sha256, + calibration_scores_sha256=scores_sha256, + promoted_rows=promoted_rows, + rht_seed=rht_seed, + packed_precision_mask=_pack_precision_mask(mask), + pool_offsets=offsets, + pool_counts=counts, + ) + + +def serialize_static_rht_q48_policy(policy: StaticRhtQ48Policy) -> bytes: + """Serialize a Q4/Q8 policy as canonical, self-hashing JSON bytes.""" + + if not isinstance(policy, StaticRhtQ48Policy): + raise TypeError("policy must be a StaticRhtQ48Policy") + content = policy._content_dict() + envelope = { + "content": content, + "policy_sha256": _sha256_bytes(_canonical_json(content)), + "schema": STATIC_Q48_POLICY_SCHEMA, + } + return _canonical_json(envelope) + b"\n" + + +def deserialize_static_rht_q48_policy(data: bytes) -> StaticRhtQ48Policy: + """Load and independently verify canonical Q4/Q8 policy bytes.""" + + if not isinstance(data, bytes): + raise TypeError("serialized policy must be bytes") + try: + root = json.loads(data.decode("utf-8"), object_pairs_hook=_strict_object) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError("serialized policy is not valid UTF-8 JSON") from error + envelope = _expect_keys( + root, + name="policy envelope", + expected={"content", "policy_sha256", "schema"}, + ) + if envelope["schema"] != STATIC_Q48_POLICY_SCHEMA: + raise ValueError("unsupported static Q48 policy schema") + declared_digest = _validate_sha256(envelope["policy_sha256"], name="policy_sha256") + content = _expect_keys( + envelope["content"], + name="policy content", + expected={ + "calibration_manifest_sha256", + "calibration_scores_sha256", + "codec_revision", + "geometry", + "geometry_sha256", + "identity_artifact_sha256", + "mask_sha256", + "method_id", + "model_id", + "model_revision", + "packed_precision_mask_b64", + "policy_revision", + "pool_counts", + "pool_offsets_le_b64", + "pool_offsets_sha256", + "promoted_rows", + "rht_seed", + "selector_revision", + "source_commit", + "tokenizer_id", + "tokenizer_manifest_sha256", + "tokenizer_revision", + "transformers_version", + }, + ) + if _sha256_bytes(_canonical_json(content)) != declared_digest: + raise ValueError("policy_sha256 does not authenticate policy content") + geometry_dict = _expect_keys( + content["geometry"], + name="geometry", + expected={ + "heads", + "key_rows", + "layer_indices", + "target_resident_bytes", + "value_width", + }, + ) + layer_indices = geometry_dict["layer_indices"] + if not isinstance(layer_indices, list): + raise TypeError("geometry.layer_indices must be a list") + geometry = StaticRhtQ468Geometry( + layer_indices=tuple(layer_indices), + heads=geometry_dict["heads"], + key_rows=geometry_dict["key_rows"], + value_width=geometry_dict["value_width"], + target_resident_bytes=geometry_dict["target_resident_bytes"], + ) + if content["geometry_sha256"] != geometry.geometry_sha256: + raise ValueError("geometry_sha256 does not authenticate geometry") + + mask_bytes = _decode_b64( + content["packed_precision_mask_b64"], + name="packed_precision_mask_b64", + ) + offset_bytes = _decode_b64( + content["pool_offsets_le_b64"], + name="pool_offsets_le_b64", + ) + if len(mask_bytes) != math.ceil(geometry.total_rows / 8): + raise ValueError("packed precision mask byte length does not match geometry") + if len(offset_bytes) != geometry.total_rows * 2: + raise ValueError("pool offset byte length does not match geometry") + packed_mask = torch.from_numpy(np.frombuffer(mask_bytes, dtype="u1").copy()) + pool_offsets = torch.from_numpy( + np.frombuffer(offset_bytes, dtype=" None: + """Atomically publish canonical Q4/Q8 policy bytes without replacement.""" + + _atomic_publish_new(Path(path), serialize_static_rht_q48_policy(policy)) + + +def load_static_rht_q48_policy(path: str | Path) -> StaticRhtQ48Policy: + """Read and verify a canonical Q4/Q8 policy artifact from ``path``.""" + + return deserialize_static_rht_q48_policy(Path(path).read_bytes()) + + +def verify_static_rht_q48_policy( + policy: StaticRhtQ48Policy, + *, + expected_policy_sha256: str | None = None, +) -> dict[str, object]: + """Revalidate a Q4/Q8 policy and return checkable evidence.""" + + if not isinstance(policy, StaticRhtQ48Policy): + raise TypeError("policy must be a StaticRhtQ48Policy") + verified = deserialize_static_rht_q48_policy(serialize_static_rht_q48_policy(policy)) + if expected_policy_sha256 is not None: + _validate_sha256(expected_policy_sha256, name="expected_policy_sha256") + if verified.policy_sha256 != expected_policy_sha256: + raise ValueError("verified policy SHA-256 does not match the expected digest") + return verified.evidence_dict() + + +def _q468_specs(geometry: StaticRhtQ468Geometry) -> tuple[QuantizationSpec, ...]: + common = { + "group_size": geometry.value_width, + "scale_bits": 16, + "flatten_last_dims": 1, + "rounding": "nearest", + "seed": RHT_SEED, + } + return ( + QuantizationSpec(bits=4, **common), + QuantizationSpec(bits=6, **common), + QuantizationSpec(bits=8, **common), + ) + + +def _q48_specs( + geometry: StaticRhtQ468Geometry, +) -> tuple[QuantizationSpec, QuantizationSpec]: + common = { + "group_size": geometry.value_width, + "scale_bits": 16, + "flatten_last_dims": 1, + "rounding": "nearest", + "seed": RHT_SEED, + } + return QuantizationSpec(bits=4, **common), QuantizationSpec(bits=8, **common) + + +def _validate_states( + states: Mapping[int, torch.Tensor], + *, + geometry: StaticRhtQ468Geometry, +) -> torch.device: + if not isinstance(states, Mapping): + raise TypeError("states must be a mapping from model-layer index to tensor") + if set(states) != set(geometry.layer_indices): + raise ValueError("states must contain exactly the policy recurrent-layer indices") + expected_shape = (1, geometry.heads, geometry.key_rows, geometry.value_width) + devices: set[torch.device] = set() + for layer_index in geometry.layer_indices: + state = states[layer_index] + if not isinstance(state, torch.Tensor): + raise TypeError(f"state for layer {layer_index} must be a torch.Tensor") + if state.dtype != torch.float32 or tuple(state.shape) != expected_shape: + raise TypeError( + f"state for layer {layer_index} must have shape {expected_shape} " + "and dtype torch.float32" + ) + if state.device.type == "meta": + raise ValueError("state tensors must be materialized") + if not torch.isfinite(state).all().item(): + raise ValueError(f"state for layer {layer_index} contains non-finite values") + if torch.is_grad_enabled() and state.requires_grad: + raise RuntimeError("static packed recurrent states are inference-only") + devices.add(state.device) + if len(devices) != 1: + raise ValueError("all state tensors must share one device") + return next(iter(devices)) + + +@dataclass(frozen=True, slots=True) +class StaticPackedRhtQ468State: + """Physically packed complete recurrent state under one static policy.""" + + policy: StaticRhtQ468Policy + int4_payload: torch.Tensor + int6_payload: torch.Tensor + int8_payload: torch.Tensor + scales: torch.Tensor + padding: torch.Tensor + + def __post_init__(self) -> None: + if not isinstance(self.policy, StaticRhtQ468Policy): + raise TypeError("policy must be a StaticRhtQ468Policy") + tensors = self.persistent_tensors() + devices: set[torch.device] = set() + owners: dict[tuple[str, int], str] = {} + for name, tensor in tensors: + _validate_owned_tensor(tensor, name=name) + if tensor.dtype in (torch.float32, torch.float64): + raise ValueError(f"persistent tensor {name} may not use FP32 or FP64") + devices.add(tensor.device) + if tensor.numel(): + identity = (str(tensor.device), tensor.untyped_storage().data_ptr()) + previous = owners.get(identity) + if previous is not None: + raise ValueError(f"persistent tensor {name} aliases {previous}") + owners[identity] = name + if len(devices) != 1: + raise ValueError("all persistent packed-state tensors must share one device") + if self.padding.dtype != torch.uint8 or self.padding.ndim != 1: + raise TypeError("padding must be a one-dimensional torch.uint8 tensor") + if self.padding.numel() and self.padding.any().item(): + raise ValueError("reserved alignment bytes must be zero") + + ledger = self.ledger + if self.padding.numel() != ledger.alignment_bytes: + raise ValueError("padding length does not match the frozen byte ledger") + q4_spec, q6_spec, q8_spec = _q468_specs(self.policy.geometry) + packed = PackedMultiBitQuantizedTensor( + int4_payload=self.int4_payload, + int6_payload=self.int6_payload, + int8_payload=self.int8_payload, + scales=self.scales, + packed_precision_codes=self.policy.packed_precision_codes, + int4_spec=q4_spec, + int6_spec=q6_spec, + int8_spec=q8_spec, + original_shape=(self.policy.geometry.total_rows, self.policy.geometry.value_width), + original_dtype=torch.float32, + flattened_size=self.policy.geometry.value_width, + padded_size=self.policy.geometry.value_width, + rows=self.policy.geometry.total_rows, + groups_per_row=1, + ) + if ( + packed.int4_groups, + packed.int6_groups, + packed.int8_groups, + ) != self.policy.pool_counts: + raise ValueError("physical payload pool counts do not match policy pool_counts") + if self.resident_bytes != ledger.resident_bytes: + raise ValueError("physical resident bytes do not match the static byte ledger") + + @property + def ledger(self) -> StaticRhtByteLedger: + return static_q468_byte_ledger( + self.policy.geometry, + self.policy.marginal_steps, + method_id=self.policy.method_id, + ) + + def persistent_tensors(self) -> tuple[tuple[str, torch.Tensor], ...]: + return ( + ("int4_payload", self.int4_payload), + ("int6_payload", self.int6_payload), + ("int8_payload", self.int8_payload), + ("scales", self.scales), + ("packed_precision_codes", self.policy.packed_precision_codes), + ("pool_offsets", self.policy.pool_offsets), + ("padding", self.padding), + ) + + @property + def data_bytes(self) -> int: + return sum( + _storage_bytes(tensor) + for name, tensor in self.persistent_tensors() + if name != "padding" + ) + + @property + def resident_bytes(self) -> int: + return sum(_storage_bytes(tensor) for _, tensor in self.persistent_tensors()) + + def _packed(self) -> PackedMultiBitQuantizedTensor: + q4_spec, q6_spec, q8_spec = _q468_specs(self.policy.geometry) + geometry = self.policy.geometry + return PackedMultiBitQuantizedTensor( + int4_payload=self.int4_payload, + int6_payload=self.int6_payload, + int8_payload=self.int8_payload, + scales=self.scales, + packed_precision_codes=self.policy.packed_precision_codes, + int4_spec=q4_spec, + int6_spec=q6_spec, + int8_spec=q8_spec, + original_shape=(geometry.total_rows, geometry.value_width), + original_dtype=torch.float32, + flattened_size=geometry.value_width, + padded_size=geometry.value_width, + rows=geometry.total_rows, + groups_per_row=1, + ) + + def materialize(self) -> dict[int, torch.Tensor]: + """Materialize decoded FP32 states without retaining an FP32 mirror.""" + + geometry = self.policy.geometry + encoded = self._packed().dequantize() + result: dict[int, torch.Tensor] = {} + for position, layer_index in enumerate(geometry.layer_indices): + start = position * geometry.rows_per_layer + stop = start + geometry.rows_per_layer + layer = encoded[start:stop].reshape( + 1, + geometry.heads, + geometry.key_rows, + geometry.value_width, + ) + result[layer_index] = right_rht_decode( + layer, + layer_index=layer_index, + expected_heads=geometry.heads, + output_dtype=torch.float32, + ) + return result + + def clone_to(self, device: torch.device | str) -> StaticPackedRhtQ468State: + return StaticPackedRhtQ468State( + policy=self.policy.clone_to(device), + int4_payload=self.int4_payload.detach().to(device).clone(), + int6_payload=self.int6_payload.detach().to(device).clone(), + int8_payload=self.int8_payload.detach().to(device).clone(), + scales=self.scales.detach().to(device).clone(), + padding=self.padding.detach().to(device).clone(), + ) + + +def pack_static_rht_q468( + states: Mapping[int, torch.Tensor], + policy: StaticRhtQ468Policy, +) -> StaticPackedRhtQ468State: + """RHT-encode and physically pack all recurrent rows under ``policy``.""" + + if not isinstance(policy, StaticRhtQ468Policy): + raise TypeError("policy must be a StaticRhtQ468Policy") + geometry = policy.geometry + device = _validate_states(states, geometry=geometry) + encoded_layers = [ + right_rht_encode( + states[layer_index], + layer_index=layer_index, + expected_heads=geometry.heads, + output_dtype=torch.float32, + ).reshape(geometry.rows_per_layer, geometry.value_width) + for layer_index in geometry.layer_indices + ] + encoded = torch.cat(encoded_layers, dim=0) + codes = policy.precision_codes().reshape(-1).to(device) + q4_spec, q6_spec, q8_spec = _q468_specs(geometry) + packed = quantize_pack_multibit( + encoded, + codes, + int4_spec=q4_spec, + int6_spec=q6_spec, + int8_spec=q8_spec, + ) + physical_policy = policy.clone_to(device) + if not torch.equal(packed.packed_precision_codes, physical_policy.packed_precision_codes): + raise RuntimeError("physical packer changed the canonical precision-code stream") + ledger = static_q468_byte_ledger( + geometry, + policy.marginal_steps, + method_id=policy.method_id, + ) + return StaticPackedRhtQ468State( + policy=physical_policy, + int4_payload=packed.int4_payload, + int6_payload=packed.int6_payload, + int8_payload=packed.int8_payload, + scales=packed.scales, + padding=torch.zeros(ledger.alignment_bytes, dtype=torch.uint8, device=device), + ) + + +def verify_static_packed_rht_q468( + state: StaticPackedRhtQ468State, +) -> dict[str, object]: + """Revalidate a packed state and return storage-only evidence.""" + + if not isinstance(state, StaticPackedRhtQ468State): + raise TypeError("state must be a StaticPackedRhtQ468State") + # Constructor replay validates payload shapes, offsets, counts, aliases, + # storage ownership, and the absence of persistent FP32/FP64 tensors. + verified = StaticPackedRhtQ468State( + policy=state.policy, + int4_payload=state.int4_payload, + int6_payload=state.int6_payload, + int8_payload=state.int8_payload, + scales=state.scales, + padding=state.padding, + ) + evidence = verified.policy.evidence_dict() + evidence["physical_data_bytes"] = verified.data_bytes + evidence["physical_resident_bytes"] = verified.resident_bytes + evidence["persistent_tensor_dtypes"] = { + name: str(tensor.dtype) for name, tensor in verified.persistent_tensors() + } + return evidence + + +@dataclass(frozen=True, slots=True) +class StaticPackedRhtQ48State: + """Physically packed complete recurrent state under a static Q4/Q8 policy.""" + + policy: StaticRhtQ48Policy + low_payload: torch.Tensor + high_payload: torch.Tensor + scales: torch.Tensor + padding: torch.Tensor + + def __post_init__(self) -> None: + if not isinstance(self.policy, StaticRhtQ48Policy): + raise TypeError("policy must be a StaticRhtQ48Policy") + tensors = self.persistent_tensors() + devices: set[torch.device] = set() + owners: dict[tuple[str, int], str] = {} + for name, tensor in tensors: + _validate_owned_tensor(tensor, name=name) + if tensor.dtype in (torch.float32, torch.float64): + raise ValueError(f"persistent tensor {name} may not use FP32 or FP64") + devices.add(tensor.device) + if tensor.numel(): + identity = (str(tensor.device), tensor.untyped_storage().data_ptr()) + previous = owners.get(identity) + if previous is not None: + raise ValueError(f"persistent tensor {name} aliases {previous}") + owners[identity] = name + if len(devices) != 1: + raise ValueError("all persistent packed-state tensors must share one device") + + geometry = self.policy.geometry + low_count, high_count = self.policy.pool_counts + expected_low_shape = (low_count, geometry.value_width * 4 // 8) + expected_high_shape = (high_count, geometry.value_width) + if ( + self.low_payload.dtype != torch.uint8 + or tuple(self.low_payload.shape) != expected_low_shape + ): + raise TypeError( + f"low_payload must have shape {expected_low_shape} and dtype torch.uint8" + ) + if ( + self.high_payload.dtype != torch.int8 + or tuple(self.high_payload.shape) != expected_high_shape + ): + raise TypeError( + f"high_payload must have shape {expected_high_shape} and dtype torch.int8" + ) + if self.low_payload.numel(): + low = torch.bitwise_and(self.low_payload, 0x0F) + high = torch.bitwise_right_shift(self.low_payload, 4) + if ((low == 8) | (high == 8)).any().item(): + raise ValueError("low_payload contains the reserved symmetric INT4 code -8") + if self.high_payload.numel() and (self.high_payload == -128).any().item(): + raise ValueError("high_payload contains the reserved symmetric INT8 code -128") + if self.scales.dtype != torch.float16 or tuple(self.scales.shape) != ( + geometry.total_rows, + ): + raise TypeError( + f"scales must have shape {(geometry.total_rows,)} and dtype torch.float16" + ) + if not torch.isfinite(self.scales).all().item() or (self.scales <= 0).any().item(): + raise ValueError("scales must contain finite, strictly positive values") + if self.padding.dtype != torch.uint8 or self.padding.ndim != 1: + raise TypeError("padding must be a one-dimensional torch.uint8 tensor") + if self.padding.numel() and self.padding.any().item(): + raise ValueError("reserved alignment bytes must be zero") + + ledger = self.ledger + if self.padding.numel() != ledger.alignment_bytes: + raise ValueError("padding length does not match the frozen byte ledger") + if self.data_bytes != ledger.data_bytes or self.resident_bytes != ledger.resident_bytes: + raise ValueError("physical resident bytes do not match the static Q4/Q8 byte ledger") + # Construct the shared packed representation as a final metadata and + # dequantization compatibility check. + self._packed() + + @property + def ledger(self) -> StaticRhtByteLedger: + return static_q48_byte_ledger( + self.policy.geometry, + self.policy.promoted_rows, + method_id=self.policy.method_id, + ) + + def persistent_tensors(self) -> tuple[tuple[str, torch.Tensor], ...]: + return ( + ("low_payload", self.low_payload), + ("high_payload", self.high_payload), + ("scales", self.scales), + ("packed_precision_mask", self.policy.packed_precision_mask), + ("pool_offsets", self.policy.pool_offsets), + ("padding", self.padding), + ) + + @property + def data_bytes(self) -> int: + return sum( + _storage_bytes(tensor) + for name, tensor in self.persistent_tensors() + if name != "padding" + ) + + @property + def resident_bytes(self) -> int: + return sum(_storage_bytes(tensor) for _, tensor in self.persistent_tensors()) + + def _packed(self) -> PackedMixedQuantizedTensor: + geometry = self.policy.geometry + low_spec, high_spec = _q48_specs(geometry) + return PackedMixedQuantizedTensor( + low_payload=self.low_payload, + high_payload=self.high_payload, + scales=self.scales, + precision_mask=self.policy.packed_precision_mask, + low_spec=low_spec, + high_spec=high_spec, + original_shape=(geometry.total_rows, geometry.value_width), + original_dtype=torch.float32, + flattened_size=geometry.value_width, + padded_size=geometry.value_width, + rows=geometry.total_rows, + groups_per_row=1, + ) + + def materialize(self) -> dict[int, torch.Tensor]: + """Materialize decoded FP32 states without retaining an FP32 mirror.""" + + geometry = self.policy.geometry + encoded = self._packed().dequantize() + result: dict[int, torch.Tensor] = {} + for position, layer_index in enumerate(geometry.layer_indices): + start = position * geometry.rows_per_layer + stop = start + geometry.rows_per_layer + layer = encoded[start:stop].reshape( + 1, + geometry.heads, + geometry.key_rows, + geometry.value_width, + ) + result[layer_index] = right_rht_decode( + layer, + layer_index=layer_index, + expected_heads=geometry.heads, + output_dtype=torch.float32, + ) + return result + + def clone_to(self, device: torch.device | str) -> StaticPackedRhtQ48State: + return StaticPackedRhtQ48State( + policy=self.policy.clone_to(device), + low_payload=self.low_payload.detach().to(device).clone(), + high_payload=self.high_payload.detach().to(device).clone(), + scales=self.scales.detach().to(device).clone(), + padding=self.padding.detach().to(device).clone(), + ) + + +def pack_static_rht_q48( + states: Mapping[int, torch.Tensor], + policy: StaticRhtQ48Policy, +) -> StaticPackedRhtQ48State: + """RHT-encode and physically pack all recurrent rows under a Q4/Q8 policy.""" + + if not isinstance(policy, StaticRhtQ48Policy): + raise TypeError("policy must be a StaticRhtQ48Policy") + geometry = policy.geometry + device = _validate_states(states, geometry=geometry) + encoded_layers = [ + right_rht_encode( + states[layer_index], + layer_index=layer_index, + expected_heads=geometry.heads, + output_dtype=torch.float32, + ).reshape(geometry.rows_per_layer, geometry.value_width) + for layer_index in geometry.layer_indices + ] + encoded = torch.cat(encoded_layers, dim=0) + mask = policy.high_precision_mask().reshape(-1).to(device) + low_spec, high_spec = _q48_specs(geometry) + packed = quantize_pack_mixed( + encoded, + mask, + low_spec=low_spec, + high_spec=high_spec, + ) + physical_policy = policy.clone_to(device) + if not torch.equal(packed.precision_mask, physical_policy.packed_precision_mask): + raise RuntimeError("physical packer changed the canonical precision-mask stream") + ledger = static_q48_byte_ledger( + geometry, + policy.promoted_rows, + method_id=policy.method_id, + ) + return StaticPackedRhtQ48State( + policy=physical_policy, + low_payload=packed.low_payload, + high_payload=packed.high_payload, + scales=packed.scales, + padding=torch.zeros(ledger.alignment_bytes, dtype=torch.uint8, device=device), + ) + + +def verify_static_packed_rht_q48( + state: StaticPackedRhtQ48State, +) -> dict[str, object]: + """Revalidate a packed Q4/Q8 state and return storage-only evidence.""" + + if not isinstance(state, StaticPackedRhtQ48State): + raise TypeError("state must be a StaticPackedRhtQ48State") + verified = StaticPackedRhtQ48State( + policy=state.policy, + low_payload=state.low_payload, + high_payload=state.high_payload, + scales=state.scales, + padding=state.padding, + ) + evidence = verified.policy.evidence_dict() + evidence["physical_data_bytes"] = verified.data_bytes + evidence["physical_resident_bytes"] = verified.resident_bytes + evidence["persistent_tensor_dtypes"] = { + name: str(tensor.dtype) for name, tensor in verified.persistent_tensors() + } + return evidence diff --git a/tests/test_static_q468.py b/tests/test_static_q468.py new file mode 100644 index 0000000..266625b --- /dev/null +++ b/tests/test_static_q468.py @@ -0,0 +1,790 @@ +from __future__ import annotations + +import itertools +import json +from dataclasses import replace +from fractions import Fraction + +import pytest +import torch + +from recurquant.rht import right_rht_decode, right_rht_encode +from recurquant.static_q468 import ( + FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + FROZEN_STATIC_Q48_PROMOTIONS, + FROZEN_STATIC_Q468_ABLATION_STEPS, + FROZEN_STATIC_Q468_PRIMARY_STEPS, + STATIC_Q48_COMPARATOR_METHOD, + STATIC_Q468_ABLATION_METHOD, + STATIC_Q468_PRIMARY_METHOD, + StaticPackedRhtQ48State, + StaticRhtQ468Geometry, + allocate_exact_q48_mask, + build_static_rht_q48_policy, + build_static_rht_q468_policy, + deserialize_static_rht_q48_policy, + deserialize_static_rht_q468_policy, + frozen_static_byte_accounting, + load_static_rht_q48_policy, + load_static_rht_q468_policy, + pack_static_rht_q48, + pack_static_rht_q468, + save_static_rht_q48_policy, + save_static_rht_q468_policy, + serialize_static_rht_q48_policy, + serialize_static_rht_q468_policy, + static_q48_distortion_sha256, + static_q468_distortion_sha256, + verify_static_packed_rht_q48, + verify_static_packed_rht_q468, + verify_static_rht_q48_policy, + verify_static_rht_q468_policy, +) + +MANIFEST_SHA256 = "23" * 32 +IDENTITY_SHA256 = "45" * 32 +TOKENIZER_MANIFEST_SHA256 = "67" * 32 +SOURCE_COMMIT = "89" * 20 +BINDINGS = { + "identity_artifact_sha256": IDENTITY_SHA256, + "tokenizer_manifest_sha256": TOKENIZER_MANIFEST_SHA256, + "source_commit": SOURCE_COMMIT, +} +TINY_GEOMETRY = StaticRhtQ468Geometry( + layer_indices=(0,), + heads=1, + key_rows=4, + value_width=8, + # K=3 data bytes are 39. The exact tiny layout owns eight reserved bytes. + target_resident_bytes=47, +) +TINY_Q48_GEOMETRY = StaticRhtQ468Geometry( + layer_indices=(0,), + heads=1, + key_rows=4, + value_width=8, + # P=2 data bytes are 41. The exact tiny layout owns eight reserved bytes. + target_resident_bytes=49, +) + + +def _tiny_distortions() -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + return ( + torch.tensor([[8.0, 2.0, 5.0, 9.0]], dtype=torch.float64), + torch.tensor([[7.0, 3.0, 1.0, 4.0]], dtype=torch.float64), + torch.tensor([[0.0, 1.0, 2.0, 6.0]], dtype=torch.float64), + ) + + +def _tiny_q48_distortions() -> tuple[torch.Tensor, torch.Tensor]: + return ( + torch.tensor([[5.0, 10.0, 4.0, 4.0]], dtype=torch.float64), + torch.tensor([[0.0, 9.0, 1.0, 1.0]], dtype=torch.float64), + ) + + +def _brute_force_codes( + distortions: tuple[torch.Tensor, torch.Tensor, torch.Tensor], + marginal_steps: int, +) -> torch.Tensor: + rows = distortions[0].numel() + stacked = torch.stack(distortions, dim=-1).reshape(rows, 3) + exact = [tuple(Fraction.from_float(float(value)) for value in row) for row in stacked] + candidates: list[tuple[Fraction, tuple[int, ...]]] = [] + for codes in itertools.product((0, 1, 2), repeat=rows): + if sum(codes) != marginal_steps: + continue + objective = sum( + (exact[row][code] for row, code in enumerate(codes)), + start=Fraction(), + ) + candidates.append((objective, codes)) + _, best = min(candidates, key=lambda item: (item[0], tuple(-code for code in item[1]))) + return torch.tensor(best, dtype=torch.uint8) + + +def _brute_force_q48_mask( + distortions: tuple[torch.Tensor, torch.Tensor], + promoted_rows: int, +) -> torch.Tensor: + rows = distortions[0].numel() + stacked = torch.stack(distortions, dim=-1).reshape(rows, 2) + exact = [tuple(Fraction.from_float(float(value)) for value in row) for row in stacked] + candidates: list[tuple[Fraction, tuple[int, ...]]] = [] + for mask in itertools.product((0, 1), repeat=rows): + if sum(mask) != promoted_rows: + continue + objective = sum( + (exact[row][selected] for row, selected in enumerate(mask)), + start=Fraction(), + ) + candidates.append((objective, mask)) + _, best = min(candidates, key=lambda item: (item[0], tuple(-bit for bit in item[1]))) + return torch.tensor(best, dtype=torch.bool) + + +def _decode_int4_pool_independently(payload: torch.Tensor) -> torch.Tensor: + octets = payload.to(torch.int16) + low = torch.bitwise_and(octets, 0x0F) + high = torch.bitwise_right_shift(octets, 4) + low = torch.where(low >= 8, low - 16, low) + high = torch.where(high >= 8, high - 16, high) + decoded = torch.empty( + (payload.shape[0], payload.shape[1] * 2), + dtype=torch.int16, + ) + decoded[:, 0::2] = low + decoded[:, 1::2] = high + return decoded + + +def _decode_int6_pool_independently(payload: torch.Tensor) -> torch.Tensor: + triples = payload.reshape(payload.shape[0], payload.shape[1] // 3, 3).to(torch.int16) + byte0, byte1, byte2 = triples.unbind(dim=-1) + unsigned = torch.stack( + ( + torch.bitwise_and(byte0, 0x3F), + torch.bitwise_right_shift(byte0, 6) + | torch.bitwise_left_shift(torch.bitwise_and(byte1, 0x0F), 2), + torch.bitwise_right_shift(byte1, 4) + | torch.bitwise_left_shift(torch.bitwise_and(byte2, 0x03), 4), + torch.bitwise_right_shift(byte2, 2), + ), + dim=-1, + ).reshape(payload.shape[0], -1) + return torch.where(unsigned >= 32, unsigned - 64, unsigned).to(torch.int16) + + +def _reference_quantize_rows( + encoded: torch.Tensor, + qmax: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + absmax = encoded.abs().amax(dim=1, keepdim=True) + ideal_scales = torch.where( + absmax > 1.0e-8, + absmax / qmax.to(torch.float32).unsqueeze(1), + torch.ones_like(absmax), + ).clamp(min=2.0**-24, max=torch.finfo(torch.float16).max) + scales = ideal_scales.to(torch.float16).squeeze(1) + normalized = encoded / scales.to(torch.float32).unsqueeze(1) + limit = qmax.to(torch.float32).unsqueeze(1) + codes = torch.minimum(torch.maximum(torch.round(normalized), -limit), limit) + return codes.to(torch.int16), scales + + +def _tiny_policy(*, marginal_steps: int = 3): + return build_static_rht_q468_policy( + *_tiny_distortions(), + geometry=TINY_GEOMETRY, + marginal_steps=marginal_steps, + calibration_manifest_sha256=MANIFEST_SHA256, + **BINDINGS, + ) + + +def _tiny_q48_policy(*, promoted_rows: int = 2): + return build_static_rht_q48_policy( + *_tiny_q48_distortions(), + geometry=TINY_Q48_GEOMETRY, + promoted_rows=promoted_rows, + calibration_manifest_sha256=MANIFEST_SHA256, + **BINDINGS, + ) + + +def test_frozen_static_ledgers_distinguish_data_alignment_and_budget_eligibility() -> None: + accounting = frozen_static_byte_accounting() + + assert accounting[STATIC_Q468_PRIMARY_METHOD] == { + "alignment_bytes": 8, + "budget_delta_bytes": 0, + "codec": "q468", + "data_bytes": 3_454_656, + "exact_budget_eligible": True, + "method_id": STATIC_Q468_PRIMARY_METHOD, + "payload_bytes": 3_297_984, + "pool_offset_bytes": 73_728, + "precision_code_bytes": 9_216, + "resident_bytes": 3_454_664, + "scale_bytes": 73_728, + "selected_units": FROZEN_STATIC_Q468_PRIMARY_STEPS, + "target_resident_bytes": 3_454_664, + } + assert accounting[STATIC_Q468_ABLATION_METHOD] == { + "alignment_bytes": 0, + "budget_delta_bytes": 73_736, + "codec": "q468", + "data_bytes": 3_380_928, + "exact_budget_eligible": False, + "method_id": STATIC_Q468_ABLATION_METHOD, + "payload_bytes": 3_224_256, + "pool_offset_bytes": 73_728, + "precision_code_bytes": 9_216, + "resident_bytes": 3_380_928, + "scale_bytes": 73_728, + "selected_units": FROZEN_STATIC_Q468_ABLATION_STEPS, + "target_resident_bytes": 3_454_664, + } + assert accounting[STATIC_Q48_COMPARATOR_METHOD] == { + "alignment_bytes": 8, + "budget_delta_bytes": 0, + "codec": "q48", + "data_bytes": 3_454_656, + "exact_budget_eligible": True, + "method_id": STATIC_Q48_COMPARATOR_METHOD, + "payload_bytes": 3_302_592, + "pool_offset_bytes": 73_728, + "precision_code_bytes": 4_608, + "resident_bytes": 3_454_664, + "scale_bytes": 73_728, + "selected_units": FROZEN_STATIC_Q48_PROMOTIONS, + "target_resident_bytes": 3_454_664, + } + + +@pytest.mark.parametrize("promoted_rows", range(5)) +def test_static_q48_selector_matches_exhaustive_search_for_every_tiny_budget( + promoted_rows: int, +) -> None: + distortions = _tiny_q48_distortions() + + actual = allocate_exact_q48_mask(*distortions, promoted_rows=promoted_rows) + expected = _brute_force_q48_mask(distortions, promoted_rows) + policy = _tiny_q48_policy(promoted_rows=promoted_rows) + + assert torch.equal(actual.reshape(-1), expected) + assert torch.equal(policy.high_precision_mask().reshape(-1), expected) + assert int(actual.sum().item()) == promoted_rows + + +def test_static_q48_selector_uses_earlier_flattened_row_on_exact_tie() -> None: + d4 = torch.tensor([[4.0, 4.0, 4.0]]) + d8 = torch.tensor([[1.0, 1.0, 1.0]]) + + mask = allocate_exact_q48_mask(d4, d8, promoted_rows=2) + + assert torch.equal(mask, torch.tensor([[True, True, False]])) + + +@pytest.mark.parametrize("marginal_steps", range(9)) +def test_static_policy_matches_exhaustive_allocation_for_every_tiny_budget( + marginal_steps: int, +) -> None: + distortions = _tiny_distortions() + + policy = build_static_rht_q468_policy( + *distortions, + geometry=TINY_GEOMETRY, + marginal_steps=marginal_steps, + calibration_manifest_sha256=MANIFEST_SHA256, + **BINDINGS, + ) + + assert torch.equal( + policy.precision_codes().reshape(-1), + _brute_force_codes(distortions, marginal_steps), + ) + assert sum(policy.pool_counts) == TINY_GEOMETRY.total_rows + assert int(policy.precision_codes().to(torch.int64).sum().item()) == marginal_steps + + +def test_policy_codes_offsets_hashes_and_serialization_are_deterministic(tmp_path) -> None: + first = _tiny_policy() + second = _tiny_policy() + + assert torch.equal(first.packed_precision_codes, second.packed_precision_codes) + assert torch.equal(first.pool_offsets, second.pool_offsets) + assert first.pool_offsets.dtype == torch.uint16 + assert first.pool_counts == second.pool_counts + assert first.calibration_scores_sha256 == static_q468_distortion_sha256( + *_tiny_distortions(), + geometry=TINY_GEOMETRY, + ) + assert first.model_id == "Qwen/Qwen3.5-0.8B-Base" + assert first.model_revision == "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68" + assert first.tokenizer_id == first.model_id + assert first.tokenizer_revision == first.model_revision + assert first.tokenizer_manifest_sha256 == TOKENIZER_MANIFEST_SHA256 + assert first.identity_artifact_sha256 == IDENTITY_SHA256 + assert first.source_commit == SOURCE_COMMIT + assert len(first.code_map_sha256) == 64 + assert len(first.pool_offsets_sha256) == 64 + assert first.policy_sha256 == second.policy_sha256 + assert serialize_static_rht_q468_policy(first) == serialize_static_rht_q468_policy(second) + + path = tmp_path / "policy.json" + save_static_rht_q468_policy(first, path) + original_bytes = path.read_bytes() + with pytest.raises(FileExistsError, match="refusing to overwrite"): + save_static_rht_q468_policy(second, path) + assert path.read_bytes() == original_bytes + assert list(tmp_path.glob(".policy.json.*.tmp")) == [] + loaded = load_static_rht_q468_policy(path) + assert loaded.policy_sha256 == first.policy_sha256 + assert loaded.evidence_dict() == first.evidence_dict() + assert torch.equal(loaded.precision_codes(), first.precision_codes()) + assert torch.equal(loaded.pool_offsets, first.pool_offsets) + assert verify_static_rht_q468_policy( + loaded, + expected_policy_sha256=first.policy_sha256, + )["policy_sha256"] == first.policy_sha256 + + +def test_q48_policy_artifact_is_deterministic_strict_and_atomically_published( + tmp_path, +) -> None: + first = _tiny_q48_policy() + second = _tiny_q48_policy() + + assert torch.equal( + first.high_precision_mask().reshape(-1), + torch.tensor([True, False, True, False]), + ) + assert first.pool_counts == (2, 2) + assert torch.equal(first.pool_offsets, torch.tensor([0, 0, 1, 1], dtype=torch.uint16)) + assert first.calibration_scores_sha256 == static_q48_distortion_sha256( + *_tiny_q48_distortions(), + geometry=TINY_Q48_GEOMETRY, + ) + assert first.model_revision == "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68" + assert first.tokenizer_manifest_sha256 == TOKENIZER_MANIFEST_SHA256 + assert first.identity_artifact_sha256 == IDENTITY_SHA256 + assert first.source_commit == SOURCE_COMMIT + assert first.mask_sha256 == second.mask_sha256 + assert first.policy_sha256 == second.policy_sha256 + + serialized = serialize_static_rht_q48_policy(first) + assert serialized == serialize_static_rht_q48_policy(second) + loaded = deserialize_static_rht_q48_policy(serialized) + assert loaded.evidence_dict() == first.evidence_dict() + assert verify_static_rht_q48_policy( + loaded, + expected_policy_sha256=first.policy_sha256, + )["policy_sha256"] == first.policy_sha256 + + path = tmp_path / "nested" / "q48-policy.json" + save_static_rht_q48_policy(first, path) + original_bytes = path.read_bytes() + with pytest.raises(FileExistsError, match="refusing to overwrite"): + save_static_rht_q48_policy(second, path) + assert path.read_bytes() == original_bytes + assert list(path.parent.glob(".q48-policy.json.*.tmp")) == [] + assert load_static_rht_q48_policy(path).policy_sha256 == first.policy_sha256 + + +def test_q48_policy_tamper_invalid_offsets_and_identity_fail_closed() -> None: + policy = _tiny_q48_policy() + offsets = policy.pool_offsets.clone() + offsets[-1] = 17 + with pytest.raises(ValueError, match="canonical per-pool prefix"): + replace(policy, pool_offsets=offsets) + with pytest.raises(ValueError, match="pool_counts"): + replace(policy, pool_counts=(3, 2)) + with pytest.raises(ValueError, match="40-hex"): + replace(policy, source_commit="main") + invalid_mask = policy.packed_precision_mask.clone() + invalid_mask[-1] = int(invalid_mask[-1].item()) | 0x80 + with pytest.raises(ValueError, match="unused precision-mask padding bits"): + replace(policy, packed_precision_mask=invalid_mask) + + serialized = serialize_static_rht_q48_policy(policy) + decoded = json.loads(serialized) + decoded["content"]["tokenizer_revision"] = "a" * 40 + tampered = json.dumps(decoded, sort_keys=True, separators=(",", ":")).encode() + b"\n" + with pytest.raises(ValueError, match="does not authenticate"): + deserialize_static_rht_q48_policy(tampered) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("policy_revision", "incompatible-policy-v999", "policy revision"), + ("selector_revision", "incompatible-selector-v999", "selector revision"), + ("codec_revision", "incompatible-codec-v999", "codec revision"), + ], +) +def test_q48_policy_rejects_unsupported_runtime_contracts( + field: str, + value: str, + message: str, +) -> None: + with pytest.raises(ValueError, match=message): + replace(_tiny_q48_policy(), **{field: value}) + + with pytest.raises(ValueError, match="frozen geometry"): + replace(_tiny_q48_policy(), method_id=STATIC_Q48_COMPARATOR_METHOD) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("model_id", "", "model_id"), + ("model_revision", "main", "40-hex"), + ("tokenizer_id", " tokenizer", "stripped printable"), + ("tokenizer_revision", "f" * 64, "40-hex"), + ("tokenizer_manifest_sha256", "f" * 40, "64-character"), + ("transformers_version", "5.14.1 dev", "semantic version"), + ("identity_artifact_sha256", "z" * 64, "64-character"), + ("source_commit", "ABCDEF" * 6 + "abcd", "40-hex"), + ], +) +def test_policy_fails_closed_on_invalid_identity_bindings( + field: str, + value: str, + message: str, +) -> None: + with pytest.raises(ValueError, match=message): + replace(_tiny_policy(), **{field: value}) + + +@pytest.mark.parametrize( + ("field", "value", "message"), + [ + ("policy_revision", "incompatible-policy-v999", "policy revision"), + ("allocator_revision", "incompatible-allocator-v999", "allocator revision"), + ("codec_revision", "incompatible-codec-v999", "codec revision"), + ], +) +def test_q468_policy_rejects_unsupported_runtime_contracts( + field: str, + value: str, + message: str, +) -> None: + with pytest.raises(ValueError, match=message): + replace(_tiny_policy(), **{field: value}) + + with pytest.raises(ValueError, match="frozen geometry"): + replace(_tiny_policy(), method_id=STATIC_Q468_PRIMARY_METHOD) + + +def test_pool_offsets_are_canonical_prefix_indices_within_each_pool() -> None: + policy = _tiny_policy() + codes = policy.precision_codes().reshape(-1) + + for code, count in enumerate(policy.pool_counts): + observed = policy.pool_offsets.to(torch.int64)[codes == code] + assert torch.equal(observed, torch.arange(count, dtype=torch.int64)) + + +def test_policy_rejects_invalid_offsets_counts_and_reserved_codes() -> None: + policy = _tiny_policy() + invalid_offsets = policy.pool_offsets.clone() + invalid_offsets[-1] = 99 + with pytest.raises(ValueError, match="canonical per-pool prefix"): + replace(policy, pool_offsets=invalid_offsets) + + with pytest.raises(ValueError, match="pool_counts"): + replace(policy, pool_counts=(policy.pool_counts[0] + 1, *policy.pool_counts[1:])) + + invalid_codes = policy.packed_precision_codes.clone() + invalid_codes[0] = int(invalid_codes[0].item()) | 0x03 + with pytest.raises(ValueError, match="reserved precision code 3"): + replace(policy, packed_precision_codes=invalid_codes) + + +def test_policy_rejects_mismatched_calibration_hash_and_noncanonical_artifact() -> None: + distortions = _tiny_distortions() + with pytest.raises(ValueError, match="does not match supplied distortions"): + build_static_rht_q468_policy( + *distortions, + geometry=TINY_GEOMETRY, + marginal_steps=3, + calibration_manifest_sha256=MANIFEST_SHA256, + calibration_scores_sha256="ab" * 32, + **BINDINGS, + ) + + serialized = serialize_static_rht_q468_policy(_tiny_policy()) + decoded = json.loads(serialized) + decoded["content"]["model_revision"] = "a" * 40 + tampered = json.dumps(decoded, sort_keys=True, separators=(",", ":")).encode() + b"\n" + with pytest.raises(ValueError, match="does not authenticate"): + deserialize_static_rht_q468_policy(tampered) + + with pytest.raises(ValueError, match="not in canonical form"): + deserialize_static_rht_q468_policy(serialized + b"\n") + + +def test_tiny_static_pack_owns_exact_bytes_without_persistent_fp32() -> None: + generator = torch.Generator().manual_seed(2339) + source = {0: torch.randn((1, 1, 4, 8), generator=generator)} + policy = _tiny_policy() + + packed = pack_static_rht_q468(source, policy) + + assert packed.data_bytes == 39 + assert packed.resident_bytes == 47 + assert packed.ledger.exact_budget_eligible is True + assert ( + packed.policy.packed_precision_codes.data_ptr() + != policy.packed_precision_codes.data_ptr() + ) + assert packed.policy.pool_offsets.data_ptr() != policy.pool_offsets.data_ptr() + assert all( + tensor.dtype not in (torch.float32, torch.float64) + for _, tensor in packed.persistent_tensors() + ) + assert [name for name, _ in packed.persistent_tensors()] == [ + "int4_payload", + "int6_payload", + "int8_payload", + "scales", + "packed_precision_codes", + "pool_offsets", + "padding", + ] + restored = packed.materialize() + assert set(restored) == {0} + assert restored[0].shape == source[0].shape + assert restored[0].dtype == torch.float32 + assert torch.isfinite(restored[0]).all().item() + assert restored[0].data_ptr() != source[0].data_ptr() + + evidence = verify_static_packed_rht_q468(packed) + assert evidence["physical_data_bytes"] == 39 + assert evidence["physical_resident_bytes"] == 47 + assert "torch.float32" not in evidence["persistent_tensor_dtypes"].values() + + +def test_tiny_q48_pack_materializes_and_owns_exact_physical_bytes() -> None: + generator = torch.Generator().manual_seed(2340) + source = {0: torch.randn((1, 1, 4, 8), generator=generator)} + policy = _tiny_q48_policy() + + packed = pack_static_rht_q48(source, policy) + + assert packed.data_bytes == 41 + assert packed.resident_bytes == 49 + assert packed.ledger.exact_budget_eligible is True + assert packed.policy.packed_precision_mask.data_ptr() != policy.packed_precision_mask.data_ptr() + assert packed.policy.pool_offsets.data_ptr() != policy.pool_offsets.data_ptr() + assert [name for name, _ in packed.persistent_tensors()] == [ + "low_payload", + "high_payload", + "scales", + "packed_precision_mask", + "pool_offsets", + "padding", + ] + assert all( + tensor.dtype not in (torch.float32, torch.float64) + for _, tensor in packed.persistent_tensors() + ) + restored = packed.materialize() + assert set(restored) == {0} + assert restored[0].shape == source[0].shape + assert restored[0].dtype == torch.float32 + assert torch.isfinite(restored[0]).all().item() + assert restored[0].data_ptr() != source[0].data_ptr() + + evidence = verify_static_packed_rht_q48(packed) + assert evidence["physical_data_bytes"] == 41 + assert evidence["physical_resident_bytes"] == 49 + assert "torch.float32" not in evidence["persistent_tensor_dtypes"].values() + + +def test_pool_offsets_independently_reconstruct_q468_and_q48_payloads() -> None: + generator = torch.Generator().manual_seed(2341) + source = {0: torch.randn((1, 1, 4, 8), generator=generator)} + encoded = right_rht_encode( + source[0], + layer_index=0, + expected_heads=1, + output_dtype=torch.float32, + ).reshape(TINY_GEOMETRY.total_rows, TINY_GEOMETRY.value_width) + + q468 = pack_static_rht_q468(source, _tiny_policy()) + q468_code_bytes = q468.policy.packed_precision_codes + q468_codes = torch.tensor( + [ + (int(q468_code_bytes[row // 4].item()) >> (2 * (row % 4))) & 0x03 + for row in range(TINY_GEOMETRY.total_rows) + ], + dtype=torch.uint8, + ) + q468_qmax = torch.tensor((7, 31, 127), dtype=torch.int16)[q468_codes.to(torch.long)] + expected_q468_codes, expected_q468_scales = _reference_quantize_rows( + encoded, + q468_qmax, + ) + q468_pools = ( + _decode_int4_pool_independently(q468.int4_payload), + _decode_int6_pool_independently(q468.int6_payload), + q468.int8_payload.to(torch.int16), + ) + observed_q468_codes = torch.empty_like(expected_q468_codes) + q468_offsets = q468.policy.pool_offsets.to(torch.int64) + for precision_code, pool in enumerate(q468_pools): + selected = q468_codes == precision_code + observed_q468_codes[selected] = pool.index_select(0, q468_offsets[selected]) + + assert torch.equal(q468.scales, expected_q468_scales) + assert torch.equal(observed_q468_codes, expected_q468_codes) + + q48 = pack_static_rht_q48(source, _tiny_q48_policy()) + q48_mask_bytes = q48.policy.packed_precision_mask + q48_mask = torch.tensor( + [ + bool((int(q48_mask_bytes[row // 8].item()) >> (row % 8)) & 0x01) + for row in range(TINY_Q48_GEOMETRY.total_rows) + ], + dtype=torch.bool, + ) + q48_qmax = torch.where(q48_mask, 127, 7).to(torch.int16) + expected_q48_codes, expected_q48_scales = _reference_quantize_rows(encoded, q48_qmax) + q48_pools = ( + _decode_int4_pool_independently(q48.low_payload), + q48.high_payload.to(torch.int16), + ) + observed_q48_codes = torch.empty_like(expected_q48_codes) + q48_offsets = q48.policy.pool_offsets.to(torch.int64) + for high_precision, pool in enumerate(q48_pools): + selected = q48_mask == bool(high_precision) + observed_q48_codes[selected] = pool.index_select(0, q48_offsets[selected]) + + assert torch.equal(q48.scales, expected_q48_scales) + assert torch.equal(observed_q48_codes, expected_q48_codes) + + for packed, integer_codes, scales in ( + (q468, observed_q468_codes, expected_q468_scales), + (q48, observed_q48_codes, expected_q48_scales), + ): + manually_dequantized = integer_codes.to(torch.float32) * scales.to( + torch.float32 + ).unsqueeze(1) + manually_restored = right_rht_decode( + manually_dequantized.reshape(1, 1, 4, 8), + layer_index=0, + expected_heads=1, + output_dtype=torch.float32, + ) + torch.testing.assert_close( + packed.materialize()[0], + manually_restored, + rtol=0.0, + atol=0.0, + ) + + +def test_q48_packed_state_rejects_alias_hidden_fp32_and_bad_payload() -> None: + source = {0: torch.ones((1, 1, 4, 8), dtype=torch.float32)} + packed = pack_static_rht_q48(source, _tiny_q48_policy()) + + with pytest.raises(ValueError, match="may not use FP32"): + replace(packed, scales=packed.scales.to(torch.float32)) + aliased_scales = packed.low_payload.view(torch.float16).reshape(-1) + with pytest.raises(ValueError, match="aliases low_payload"): + replace(packed, scales=aliased_scales) + with pytest.raises(TypeError, match="low_payload must have shape"): + replace(packed, low_payload=packed.low_payload[:-1].clone()) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA device is unavailable") +def test_static_q468_and_q48_reference_codecs_round_trip_on_cuda() -> None: + generator = torch.Generator(device="cuda").manual_seed(2339) + source = { + 0: torch.randn((1, 1, 4, 8), generator=generator, device="cuda") + } + + q468 = pack_static_rht_q468(source, _tiny_policy()) + q48 = pack_static_rht_q48(source, _tiny_q48_policy()) + + for packed, verifier in ( + (q468, verify_static_packed_rht_q468), + (q48, verify_static_packed_rht_q48), + ): + restored = packed.materialize()[0] + torch.cuda.synchronize() + assert restored.device.type == "cuda" + assert torch.isfinite(restored).all().item() + assert verifier(packed)["physical_resident_bytes"] == packed.resident_bytes + + +def test_packed_state_fails_closed_on_hidden_fp32_and_bad_pool_shape() -> None: + source = {0: torch.ones((1, 1, 4, 8), dtype=torch.float32)} + packed = pack_static_rht_q468(source, _tiny_policy()) + + with pytest.raises(ValueError, match="may not use FP32"): + replace(packed, scales=packed.scales.to(torch.float32)) + + with pytest.raises(ValueError, match="int4_payload must have shape"): + replace(packed, int4_payload=packed.int4_payload[:-1].clone()) + + +def test_real_geometry_primary_policy_has_exact_k_counts_and_uint16_offsets() -> None: + geometry = FROZEN_QWEN35_STATIC_Q468_GEOMETRY + row = torch.arange(geometry.total_rows, dtype=torch.float64) + distortions = ( + (((17 * row + 13) % 1009) / 1009).reshape(geometry.layers, -1), + (((29 * row + 7) % 1013) / 1013).reshape(geometry.layers, -1), + (((43 * row + 3) % 1019) / 1019).reshape(geometry.layers, -1), + ) + + policy = build_static_rht_q468_policy( + *distortions, + geometry=geometry, + marginal_steps=FROZEN_STATIC_Q468_PRIMARY_STEPS, + method_id=STATIC_Q468_PRIMARY_METHOD, + calibration_manifest_sha256=MANIFEST_SHA256, + **BINDINGS, + ) + + codes = policy.precision_codes().reshape(-1) + assert codes.shape == (36_864,) + assert int(codes.to(torch.int64).sum().item()) == FROZEN_STATIC_Q468_PRIMARY_STEPS + assert sum(policy.pool_counts) == 36_864 + assert policy.packed_precision_codes.numel() == 9_216 + assert policy.pool_offsets.dtype == torch.uint16 + assert policy.pool_offsets.numel() == 36_864 + for code, count in enumerate(policy.pool_counts): + observed = policy.pool_offsets.to(torch.int64)[codes == code] + assert torch.equal(observed, torch.arange(count, dtype=torch.int64)) + assert policy.evidence_dict()["ledger"]["resident_bytes"] == 3_454_664 + with pytest.raises(ValueError, match="wrong exact-K budget"): + replace(policy, marginal_steps=0) + with pytest.raises(ValueError, match="frozen model identity"): + replace(policy, model_id="Qwen/other-model") + + +def test_real_q48_policy_has_a_physical_exact_3454664_byte_state() -> None: + geometry = FROZEN_QWEN35_STATIC_Q468_GEOMETRY + row = torch.arange(geometry.total_rows, dtype=torch.float64) + d4 = (((19 * row + 11) % 1021) / 1021).reshape(geometry.layers, -1) + d8 = (((31 * row + 5) % 1031) / 1031).reshape(geometry.layers, -1) + policy = build_static_rht_q48_policy( + d4, + d8, + geometry=geometry, + promoted_rows=FROZEN_STATIC_Q48_PROMOTIONS, + method_id=STATIC_Q48_COMPARATOR_METHOD, + calibration_manifest_sha256=MANIFEST_SHA256, + **BINDINGS, + ) + low_count, high_count = policy.pool_counts + + packed = StaticPackedRhtQ48State( + policy=policy, + low_payload=torch.zeros( + (low_count, geometry.value_width * 4 // 8), + dtype=torch.uint8, + ), + high_payload=torch.zeros( + (high_count, geometry.value_width), + dtype=torch.int8, + ), + scales=torch.ones(geometry.total_rows, dtype=torch.float16), + padding=torch.zeros(8, dtype=torch.uint8), + ) + + assert policy.packed_precision_mask.numel() == 4_608 + assert policy.pool_offsets.numel() * policy.pool_offsets.element_size() == 73_728 + assert policy.pool_counts == (36_864 - 14_739, 14_739) + assert packed.low_payload.numel() + packed.high_payload.numel() == 3_302_592 + assert packed.scales.numel() * packed.scales.element_size() == 73_728 + assert packed.data_bytes == 3_454_656 + assert packed.resident_bytes == 3_454_664 + assert packed.ledger.exact_budget_eligible is True + with pytest.raises(ValueError, match="wrong exact-P budget"): + replace(policy, promoted_rows=0) + with pytest.raises(ValueError, match="frozen model identity"): + replace(policy, tokenizer_id="Qwen/other-tokenizer") From 0d879d021704336545ec7d93d0ff91ea14a0c3df Mon Sep 17 00:00:00 2001 From: HiddenTrojan <93521146+Labeeb2339@users.noreply.github.com> Date: Sun, 2 Aug 2026 09:47:15 +0800 Subject: [PATCH 05/24] Bind PG19 capture to active parquet manifest --- .gitignore | 1 + research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md | 13 +++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 723634e..e351b1e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ __pycache__/ .pytest-of-*/ .pytest/ .ruff_cache/ +.cache/ .coverage htmlcov/ dist/ diff --git a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md index 8a77066..36755ee 100644 --- a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md +++ b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md @@ -143,10 +143,15 @@ recurquant.experiment013.humaneval-plus.stage-c.v1\0 ``` The canonical PG19 ID is the exact UTF-8 `url` field; the pinned PG19 schema -does not contain a `book_id` field. A training book is eligible when the pinned -tokenizer produces at least 2,304 tokens. Rank all 13,684 training URLs before -opening text, then inspect them in that fixed order only until 16 eligible -books have been accepted. For an accepted book with `N` tokens, define +does not contain a `book_id` field. The population is the ordered `url` +projection from the active Hugging Face Dataset Viewer `/parquet` manifest for +the `default` configuration and pinned split. The capture must authenticate the +manifest's `x-revision`, hash that complete ordered projection, and exclude +repository-tree parquet files that the active manifest does not reference. A +training book is eligible when the pinned tokenizer produces at least 2,304 +tokens. Rank all 13,684 training URLs before opening text, then inspect them in +that fixed order only until 16 eligible books have been accepted. For an +accepted book with `N` tokens, define ```text M = N - 2304 From 3e4725291d92f0e0a75cdee5f023808c47bc21ae Mon Sep 17 00:00:00 2001 From: HiddenTrojan <93521146+Labeeb2339@users.noreply.github.com> Date: Sun, 2 Aug 2026 09:55:15 +0800 Subject: [PATCH 06/24] Authenticate the static codec source module --- scripts/capture_statelease_stage0.py | 1 + scripts/screen_statelease_stage_a.py | 1 + scripts/verify_statelease_stage0.py | 1 + 3 files changed, 3 insertions(+) diff --git a/scripts/capture_statelease_stage0.py b/scripts/capture_statelease_stage0.py index 2857545..158fdad 100644 --- a/scripts/capture_statelease_stage0.py +++ b/scripts/capture_statelease_stage0.py @@ -132,6 +132,7 @@ "src/recurquant/rht.py", "src/recurquant/row_policy.py", "src/recurquant/signals.py", + "src/recurquant/static_q468.py", "src/recurquant/statelease.py", "src/recurquant/statelease_artifact.py", "src/recurquant/statelease_baselines.py", diff --git a/scripts/screen_statelease_stage_a.py b/scripts/screen_statelease_stage_a.py index 5634b93..262182b 100644 --- a/scripts/screen_statelease_stage_a.py +++ b/scripts/screen_statelease_stage_a.py @@ -401,6 +401,7 @@ "src/recurquant/rht.py", "src/recurquant/row_policy.py", "src/recurquant/signals.py", + "src/recurquant/static_q468.py", "src/recurquant/statelease.py", "src/recurquant/statelease_artifact.py", "src/recurquant/statelease_baselines.py", diff --git a/scripts/verify_statelease_stage0.py b/scripts/verify_statelease_stage0.py index d132810..d8452c4 100644 --- a/scripts/verify_statelease_stage0.py +++ b/scripts/verify_statelease_stage0.py @@ -177,6 +177,7 @@ "src/recurquant/rht.py", "src/recurquant/row_policy.py", "src/recurquant/signals.py", + "src/recurquant/static_q468.py", "src/recurquant/statelease.py", "src/recurquant/statelease_artifact.py", "src/recurquant/statelease_baselines.py", From 242c2a3c61798592a278b9c5d60402f11fc64cf7 Mon Sep 17 00:00:00 2001 From: HiddenTrojan <93521146+Labeeb2339@users.noreply.github.com> Date: Sun, 2 Aug 2026 11:40:21 +0800 Subject: [PATCH 07/24] Implement static Q468 runtime and calibration chain --- src/recurquant/__init__.py | 16 + src/recurquant/static_q468.py | 7 + src/recurquant/static_q468_cache.py | 521 ++++ src/recurquant/static_q468_calibration.py | 2830 +++++++++++++++++++++ tests/test_static_q468.py | 6 + tests/test_static_q468_cache.py | 595 +++++ tests/test_static_q468_calibration.py | 1138 +++++++++ 7 files changed, 5113 insertions(+) create mode 100644 src/recurquant/static_q468_cache.py create mode 100644 src/recurquant/static_q468_calibration.py create mode 100644 tests/test_static_q468_cache.py create mode 100644 tests/test_static_q468_calibration.py diff --git a/src/recurquant/__init__.py b/src/recurquant/__init__.py index 2384130..5776189 100644 --- a/src/recurquant/__init__.py +++ b/src/recurquant/__init__.py @@ -173,6 +173,15 @@ reference_aligned_trajectory_nmse, ) from .statelease_observer import Qwen35StateLeaseObserver +from .static_q468_cache import ( + DYNAMIC_Q468_ORACLE_METHOD, + FROZEN_STATIC_RUNTIME_METHODS, + StaticRhtCacheUpdateEvidence, + StaticRhtQwen35Cache, + StaticRhtRuntimeCheckpoint, + create_qwen35_dynamic_q468_oracle_cache, + create_qwen35_static_rht_cache, +) from .transition_observer import Qwen35TransitionObserver __all__ = [ @@ -220,6 +229,8 @@ "EqualByteLinearAttentionLayer", "EqualByteNoReplayCache", "EqualByteQwen35Cache", + "DYNAMIC_Q468_ORACLE_METHOD", + "FROZEN_STATIC_RUNTIME_METHODS", "QueryEmaMixedPackedLinearAttentionLayer", "QueryEmaMixedPackedRecurrentStateCache", "Qwen35QueryEnergyObserver", @@ -244,6 +255,9 @@ "StateLeaseLinearAttentionLayer", "StateLeaseRecurrentStateCache", "StateLeaseUpdateEvidence", + "StaticRhtCacheUpdateEvidence", + "StaticRhtQwen35Cache", + "StaticRhtRuntimeCheckpoint", "PhysicalMetricRun", "PhysicalRowPromotionOracleResult", "QuantizationResult", @@ -268,6 +282,7 @@ "create_qwen35_cora_exact_budget_cache", "create_qwen35_exact_budget_cache", "create_qwen35_equal_byte_cache", + "create_qwen35_dynamic_q468_oracle_cache", "create_qwen35_experiment010_fixed_replay_cache", "create_qwen35_experiment010_statelease_cache", "create_qwen35_packed_cache", @@ -275,6 +290,7 @@ "create_qwen35_rank_fused_exact_budget_cache", "create_qwen35_right_rht_query_ema_exact_budget_cache", "create_qwen35_statelease_cache", + "create_qwen35_static_rht_cache", "create_qwen35_v02_mixed_cache", "experiment010_statelease_effective_plan_sha256", "create_fixed_replay_cache", diff --git a/src/recurquant/static_q468.py b/src/recurquant/static_q468.py index 92fac72..a8dcba8 100644 --- a/src/recurquant/static_q468.py +++ b/src/recurquant/static_q468.py @@ -523,6 +523,8 @@ def __post_init__(self) -> None: steps = _validate_integer(self.marginal_steps, name="marginal_steps") if steps > 2 * self.geometry.total_rows: raise ValueError("marginal_steps exceeds two steps per state row") + if self.method_id == STATIC_Q48_COMPARATOR_METHOD: + raise ValueError("reserved static Q48 method cannot identify a Q468 policy") frozen_steps = { STATIC_Q468_PRIMARY_METHOD: FROZEN_STATIC_Q468_PRIMARY_STEPS, STATIC_Q468_ABLATION_METHOD: FROZEN_STATIC_Q468_ABLATION_STEPS, @@ -1167,6 +1169,11 @@ def __post_init__(self) -> None: promotions = _validate_integer(self.promoted_rows, name="promoted_rows") if promotions > self.geometry.total_rows: raise ValueError("promoted_rows exceeds the number of state rows") + if self.method_id in { + STATIC_Q468_PRIMARY_METHOD, + STATIC_Q468_ABLATION_METHOD, + }: + raise ValueError("reserved static Q468 method cannot identify a Q48 policy") if self.method_id == STATIC_Q48_COMPARATOR_METHOD: if self.geometry != FROZEN_QWEN35_STATIC_Q468_GEOMETRY: raise ValueError("reserved static Q48 method requires the frozen geometry") diff --git a/src/recurquant/static_q468_cache.py b/src/recurquant/static_q468_cache.py new file mode 100644 index 0000000..068204c --- /dev/null +++ b/src/recurquant/static_q468_cache.py @@ -0,0 +1,521 @@ +"""Transactional Qwen3.5 runtime for frozen static RHT policies. + +The static policies choose every recurrent-row precision during calibration. +This runtime deliberately reuses the audited equal-byte cache's root-model +transaction, layer receipts, rollback, and batch-one lifecycle. It changes +only the complete-state packer and the evidence needed for a static policy. + +The implementation is correctness-first. Every model forward materializes +the previous complete FP32 state as transient workspace and repacks the new +complete state after the LM head succeeds. It is suitable for quality +experiments, but it is not the packed-native deployment path. +""" + +from __future__ import annotations + +import re +from dataclasses import asdict, dataclass +from typing import Literal, TypeAlias + +import torch + +from .qwen35 import _validate_transformers_compatibility, _validated_text_config +from .statelease_equal_byte_baselines import ( + FROZEN_QWEN35_EQUAL_BYTE_LAYOUT, + RHT_Q4_Q6_Q8, + EqualByteLayout, +) +from .statelease_equal_byte_cache import EqualByteQwen35Cache, create_qwen35_equal_byte_cache +from .static_q468 import ( + FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + STATIC_Q48_COMPARATOR_METHOD, + STATIC_Q468_ABLATION_METHOD, + STATIC_Q468_PRIMARY_METHOD, + StaticPackedRhtQ48State, + StaticPackedRhtQ468State, + StaticRhtByteLedger, + StaticRhtQ48Policy, + StaticRhtQ468Geometry, + StaticRhtQ468Policy, + deserialize_static_rht_q48_policy, + deserialize_static_rht_q468_policy, + pack_static_rht_q48, + pack_static_rht_q468, + serialize_static_rht_q48_policy, + serialize_static_rht_q468_policy, + static_q48_byte_ledger, + static_q468_byte_ledger, + verify_static_packed_rht_q48, + verify_static_packed_rht_q468, + verify_static_rht_q48_policy, + verify_static_rht_q468_policy, +) + +StaticRhtPolicy: TypeAlias = StaticRhtQ468Policy | StaticRhtQ48Policy +StaticPackedRhtState: TypeAlias = StaticPackedRhtQ468State | StaticPackedRhtQ48State +StaticPolicyKind: TypeAlias = Literal["q468", "q48"] + +DYNAMIC_Q468_ORACLE_METHOD = "rht_q468_dynamic_k27030" +FROZEN_STATIC_RUNTIME_METHODS = frozenset( + ( + STATIC_Q468_PRIMARY_METHOD, + STATIC_Q468_ABLATION_METHOD, + STATIC_Q48_COMPARATOR_METHOD, + ) +) + +_SHA256_RE = re.compile(r"[0-9a-f]{64}") + + +def _require_sha256(value: object, *, name: str) -> str: + if not isinstance(value, str) or _SHA256_RE.fullmatch(value) is None: + raise ValueError(f"{name} must be a lowercase 64-character SHA-256 hex digest") + return value + + +def _structural_geometry(geometry: StaticRhtQ468Geometry) -> tuple[object, ...]: + return ( + geometry.layer_indices, + geometry.heads, + geometry.key_rows, + geometry.value_width, + ) + + +def _layout_geometry(layout: EqualByteLayout) -> tuple[object, ...]: + return ( + layout.layer_indices, + layout.heads, + layout.key_rows, + layout.value_width, + ) + + +@dataclass(frozen=True, slots=True) +class StaticRhtRuntimeCheckpoint: + """One complete static packed checkpoint and its authenticated identity.""" + + state: StaticPackedRhtState + expected_policy_sha256: str + expected_method_id: str + + def __post_init__(self) -> None: + self.validate() + + @property + def policy(self) -> StaticRhtPolicy: + return self.state.policy + + @property + def ledger(self) -> StaticRhtByteLedger: + return self.state.ledger + + @property + def resident_bytes(self) -> int: + return self.state.resident_bytes + + def persistent_tensors(self) -> tuple[tuple[str, torch.Tensor], ...]: + return self.state.persistent_tensors() + + def materialize(self) -> dict[int, torch.Tensor]: + # Policy tensors are ordinary mutable torch storage even though the + # dataclass is frozen. Re-authenticate immediately before every decode + # so an accidental in-place edit cannot silently change the reserved + # method while retaining its old expected digest. + self.validate() + return self.state.materialize() + + def validate(self) -> None: + _require_sha256(self.expected_policy_sha256, name="expected_policy_sha256") + if self.policy.policy_sha256 != self.expected_policy_sha256: + raise ValueError("static checkpoint policy SHA-256 does not match the expected digest") + if self.policy.method_id != self.expected_method_id: + raise ValueError("static checkpoint method identity drifted") + if isinstance(self.state, StaticPackedRhtQ468State): + verify_static_packed_rht_q468(self.state) + elif isinstance(self.state, StaticPackedRhtQ48State): + verify_static_packed_rht_q48(self.state) + else: + raise TypeError("state must be a supported static packed RHT state") + if self.resident_bytes != self.ledger.resident_bytes: + raise ValueError("static checkpoint resident storage differs from its byte ledger") + + def to(self, device: torch.device | str) -> StaticRhtRuntimeCheckpoint: + return StaticRhtRuntimeCheckpoint( + state=self.state.clone_to(device), + expected_policy_sha256=self.expected_policy_sha256, + expected_method_id=self.expected_method_id, + ) + + +@dataclass(frozen=True, slots=True) +class StaticRhtCacheUpdateEvidence: + """Evidence for one successful static all-layer checkpoint transaction.""" + + update_index: int + method_id: str + policy_sha256: str + selection_sha256: str + pool_offsets_sha256: str + token_count: int + layer_indices: tuple[int, ...] + query_input_dtypes: tuple[str, ...] + previous_checkpoint_present: bool + retained_raw_state_workspace_peak_bytes: int + retained_query_workspace_peak_bytes: int + workspace_measurement_scope: str + cuda_allocator_peak_measured: bool + cuda_allocator_peak_bytes: int | None + logical_fp32_state_bytes: int + payload_bytes: int + scale_bytes: int + precision_code_bytes: int + pool_offset_bytes: int + alignment_bytes: int + resident_tensor_storage_bytes: int + target_resident_bytes: int + budget_delta_bytes: int + exact_budget_eligible: bool + selected_units: int + mean_squared_error: float + relative_l2_error: float + max_absolute_error: float + + def evidence_dict(self) -> dict[str, object]: + return asdict(self) + + +def _error_metrics( + source: dict[int, torch.Tensor], + materialized: dict[int, torch.Tensor], + *, + geometry: StaticRhtQ468Geometry, +) -> tuple[float, float, float]: + if set(source) != set(geometry.layer_indices) or set(materialized) != set( + geometry.layer_indices + ): + raise RuntimeError("static codec error measurement omitted a recurrent layer") + squared_error = torch.zeros((), dtype=torch.float64) + squared_source = torch.zeros((), dtype=torch.float64) + maximum = 0.0 + for layer_index in geometry.layer_indices: + reference = source[layer_index].detach().to(device="cpu", dtype=torch.float64) + restored = materialized[layer_index].detach().to(device="cpu", dtype=torch.float64) + error = restored - reference + squared_error += error.square().sum() + squared_source += reference.square().sum() + maximum = max(maximum, float(error.abs().max().item())) + mse = float((squared_error / geometry.state_elements).item()) + relative = float((squared_error.sqrt() / squared_source.sqrt().clamp_min(1e-12)).item()) + return mse, relative, maximum + + +class StaticRhtQwen35Cache(EqualByteQwen35Cache): + """Batch-one transactional cache for one immutable static RHT policy. + + The cache stores only canonical policy bytes and scalar/hash metadata before + its first checkpoint. It never retains a second tensor policy beside the + policy owned by the installed checkpoint. + """ + + def __init__( + self, + config: object, + *, + policy: StaticRhtPolicy, + expected_policy_sha256: str, + layout: EqualByteLayout = FROZEN_QWEN35_EQUAL_BYTE_LAYOUT, + record_evidence: bool = False, + ) -> None: + if not isinstance(layout, EqualByteLayout): + raise TypeError("layout must be an EqualByteLayout") + if not isinstance(record_evidence, bool): + raise TypeError("record_evidence must be a bool") + + if isinstance(policy, StaticRhtQ468Policy): + policy_kind: StaticPolicyKind = "q468" + serialized = serialize_static_rht_q468_policy(policy) + canonical = deserialize_static_rht_q468_policy(serialized) + verify_static_rht_q468_policy( + canonical, + expected_policy_sha256=expected_policy_sha256, + ) + selection_sha256 = canonical.code_map_sha256 + ledger = static_q468_byte_ledger( + canonical.geometry, + canonical.marginal_steps, + method_id=canonical.method_id, + ) + elif isinstance(policy, StaticRhtQ48Policy): + policy_kind = "q48" + serialized = serialize_static_rht_q48_policy(policy) + canonical = deserialize_static_rht_q48_policy(serialized) + verify_static_rht_q48_policy( + canonical, + expected_policy_sha256=expected_policy_sha256, + ) + selection_sha256 = canonical.mask_sha256 + ledger = static_q48_byte_ledger( + canonical.geometry, + canonical.promoted_rows, + method_id=canonical.method_id, + ) + else: + raise TypeError("policy must be a StaticRhtQ468Policy or StaticRhtQ48Policy") + + expected_policy_sha256 = _require_sha256( + expected_policy_sha256, + name="expected_policy_sha256", + ) + if _structural_geometry(canonical.geometry) != _layout_geometry(layout): + raise ValueError("static policy geometry does not match the Qwen3.5 cache layout") + if canonical.geometry.target_resident_bytes != layout.expected_resident_bytes: + raise ValueError("static policy target bytes do not match the cache layout target") + + # The superclass establishes the audited layer objects and transaction + # state. Its dynamic codec is never called by this subclass. + super().__init__( + config, + codec=RHT_Q4_Q6_Q8, + layout=layout, + record_evidence=record_evidence, + ) + + # Retain immutable host bytes and scalars only. In particular, do not + # assign ``canonical`` or the caller's ``policy`` to this object. + self.codec = canonical.method_id # type: ignore[assignment] + self.method_id = canonical.method_id + self.policy_kind = policy_kind + self.expected_policy_sha256 = expected_policy_sha256 + self.selection_sha256 = selection_sha256 + self.pool_offsets_sha256 = canonical.pool_offsets_sha256 + self._serialized_policy = serialized + self._static_target_resident_bytes = canonical.geometry.target_resident_bytes + self._static_expected_resident_bytes = ledger.resident_bytes + self._static_budget_delta_bytes = ledger.budget_delta_bytes + self._static_exact_budget_eligible = ledger.exact_budget_eligible + self.checkpoint: StaticRhtRuntimeCheckpoint | None = None # type: ignore[assignment] + self.update_evidence: list[StaticRhtCacheUpdateEvidence] = [] # type: ignore[assignment] + self.last_evidence: StaticRhtCacheUpdateEvidence | None = None # type: ignore[assignment] + + def _load_policy(self) -> StaticRhtPolicy: + if self.policy_kind == "q468": + policy = deserialize_static_rht_q468_policy(self._serialized_policy) + verify_static_rht_q468_policy( + policy, + expected_policy_sha256=self.expected_policy_sha256, + ) + else: + policy = deserialize_static_rht_q48_policy(self._serialized_policy) + verify_static_rht_q48_policy( + policy, + expected_policy_sha256=self.expected_policy_sha256, + ) + if policy.method_id != self.method_id: + raise RuntimeError("serialized static policy method identity drifted") + return policy + + def _pack_static_candidate( + self, + states: dict[int, torch.Tensor], + ) -> StaticRhtRuntimeCheckpoint: + policy = self._load_policy() + if isinstance(policy, StaticRhtQ468Policy): + state: StaticPackedRhtState = pack_static_rht_q468(states, policy) + else: + state = pack_static_rht_q48(states, policy) + return StaticRhtRuntimeCheckpoint( + state=state, + expected_policy_sha256=self.expected_policy_sha256, + expected_method_id=self.method_id, + ) + + def _pack_candidate( + self, + states: dict[int, torch.Tensor], + query_ema: torch.Tensor, + ) -> None: + del states, query_ema + raise RuntimeError( + "static RHT caches must use the immutable-policy packer, not the dynamic codec" + ) + + def commit_statelease_forward_transaction(self, transaction: object) -> None: + """Pack and install one static checkpoint only after complete model success.""" + + active = self._require_active_transaction(transaction) + if self._pending_observations: + raise RuntimeError( + "model forward returned with unconsumed static observations " + f"on layers {sorted(self._pending_observations)}" + ) + if tuple(self._receipt_order) != self.layout.layer_indices: + raise RuntimeError( + "model forward did not produce exactly one recurrent receipt for every frozen layer" + ) + if set(self._final_states) != set(self.layout.layer_indices): + raise RuntimeError("global recurrent-state staging is incomplete") + if set(self._queries) != set(self.layout.layer_indices): + raise RuntimeError("global causal-query staging is incomplete") + token_counts = {int(query.shape[1]) for query in self._queries.values()} + if len(token_counts) != 1: + raise RuntimeError("recurrent layers observed inconsistent token counts") + token_count = next(iter(token_counts)) + + candidate = self._pack_static_candidate(self._final_states) + candidate.validate() + ledger = candidate.ledger + if candidate.resident_bytes != ledger.resident_bytes: + raise RuntimeError("static candidate checkpoint violates its physical byte ledger") + if candidate.resident_bytes != self._static_expected_resident_bytes: + raise RuntimeError("static candidate checkpoint violates its frozen method bytes") + if ledger.target_resident_bytes != self._static_target_resident_bytes: + raise RuntimeError("static candidate target byte identity drifted") + if ledger.budget_delta_bytes != self._static_budget_delta_bytes: + raise RuntimeError("static candidate budget delta drifted") + if ledger.exact_budget_eligible is not self._static_exact_budget_eligible: + raise RuntimeError("static candidate exact-budget eligibility drifted") + + materialized = candidate.materialize() + mse, relative, maximum = _error_metrics( + self._final_states, + materialized, + geometry=candidate.policy.geometry, + ) + layers = tuple(self.equal_byte_layers()) + if tuple(index for index, _layer in layers) != self.layout.layer_indices: + raise RuntimeError("static cache recurrent layer identity drifted before commit") + + evidence = StaticRhtCacheUpdateEvidence( + update_index=self.update_count, + method_id=self.method_id, + policy_sha256=self.expected_policy_sha256, + selection_sha256=self.selection_sha256, + pool_offsets_sha256=self.pool_offsets_sha256, + token_count=token_count, + layer_indices=tuple(self._receipt_order), + query_input_dtypes=tuple( + str(self._queries[index].dtype) for index in self.layout.layer_indices + ), + previous_checkpoint_present=active.checkpoint is not None, + retained_raw_state_workspace_peak_bytes=self._forward_state_workspace_peak_bytes, + retained_query_workspace_peak_bytes=self._forward_query_workspace_peak_bytes, + workspace_measurement_scope="cache_retained_forward_tensors_only", + cuda_allocator_peak_measured=False, + cuda_allocator_peak_bytes=None, + logical_fp32_state_bytes=self.layout.fp32_state_bytes, + payload_bytes=ledger.payload_bytes, + scale_bytes=ledger.scale_bytes, + precision_code_bytes=ledger.precision_code_bytes, + pool_offset_bytes=ledger.pool_offset_bytes, + alignment_bytes=ledger.alignment_bytes, + resident_tensor_storage_bytes=ledger.resident_bytes, + target_resident_bytes=ledger.target_resident_bytes, + budget_delta_bytes=ledger.budget_delta_bytes, + exact_budget_eligible=ledger.exact_budget_eligible, + selected_units=ledger.selected_units, + mean_squared_error=mse, + relative_l2_error=relative, + max_absolute_error=maximum, + ) + next_evidence = ( + [*self.update_evidence, evidence] if self.record_evidence else self.update_evidence + ) + + # Every fallible pack, materialization, metric, ledger, and layer check + # is complete. Replace references only at this final commit point. + self.checkpoint = candidate + self.update_count += 1 + self.successful_tokens += token_count + self.last_evidence = evidence + if self.record_evidence: + self.update_evidence = next_evidence + for _, layer in layers: + layer.is_recurrent_states_initialized[0] = True + layer.has_previous_state[0] = True + + self._clear_forward_workspace() + active.active = False + self._active_equal_byte_transaction = None + + def storage_summary(self) -> dict[str, object]: + summary: dict[str, object] = dict(super().storage_summary()) + resident = 0 if self.checkpoint is None else self.checkpoint.resident_bytes + summary.update( + { + "codec": f"static_rht_{self.policy_kind}", + "method_id": self.method_id, + "policy_sha256": self.expected_policy_sha256, + "selection_sha256": self.selection_sha256, + "pool_offsets_sha256": self.pool_offsets_sha256, + "resident_bytes": resident, + "resident_tensor_storage_bytes": resident, + "expected_resident_bytes": self._static_expected_resident_bytes, + "target_resident_bytes": self._static_target_resident_bytes, + "budget_delta_bytes": self._static_budget_delta_bytes, + "exact_budget_eligible": self._static_exact_budget_eligible, + "policy_tensor_storage_bytes_outside_checkpoint": 0, + "serialized_policy_host_bytes": len(self._serialized_policy), + "workspace_measurement_scope": "cache_retained_forward_tensors_only", + "cuda_allocator_peak_measured": False, + "cuda_allocator_peak_bytes": None, + } + ) + return summary + + +def create_qwen35_static_rht_cache( + model_or_config: object, + *, + policy: StaticRhtPolicy, + expected_policy_sha256: str, + record_evidence: bool = False, +) -> StaticRhtQwen35Cache: + """Create a policy-locked Experiment 013 static Qwen3.5 cache. + + The expected policy SHA must come from the separately authenticated and + promoted experiment identity. This factory validates model structure and, + when given a model, evaluation mode, eager attention, and single-device + placement. It does not authenticate local model files or a Hub revision. + """ + + _validate_transformers_compatibility() + config = _validated_text_config(model_or_config) + if not isinstance(policy, (StaticRhtQ468Policy, StaticRhtQ48Policy)): + raise TypeError("policy must be a StaticRhtQ468Policy or StaticRhtQ48Policy") + if policy.method_id not in FROZEN_STATIC_RUNTIME_METHODS: + raise ValueError("public Experiment 013 runtime accepts only the three frozen methods") + if policy.geometry != FROZEN_QWEN35_STATIC_Q468_GEOMETRY: + raise ValueError("frozen Experiment 013 method requires the frozen Qwen3.5 geometry") + return StaticRhtQwen35Cache( + config, + policy=policy, + expected_policy_sha256=expected_policy_sha256, + record_evidence=record_evidence, + ) + + +def create_qwen35_dynamic_q468_oracle_cache( + model_or_config: object, + *, + record_evidence: bool = False, +) -> EqualByteQwen35Cache: + """Create the named exact dynamic K27030 quality oracle. + + This is the existing global RHT Q4/Q6/Q8 allocator with its frozen 27,030 + marginal steps and declared persistent query-energy EMA. The wrapper adds + an experiment method identity; it does not change the codec. + """ + + _validate_transformers_compatibility() + config = _validated_text_config(model_or_config) + if FROZEN_QWEN35_EQUAL_BYTE_LAYOUT.multibit_marginal_steps != 27_030: + raise RuntimeError("dynamic Q468 oracle layout no longer has K27030") + cache = create_qwen35_equal_byte_cache( + config, + codec=RHT_Q4_Q6_Q8, + layout=FROZEN_QWEN35_EQUAL_BYTE_LAYOUT, + record_evidence=record_evidence, + ) + cache.method_id = DYNAMIC_Q468_ORACLE_METHOD # type: ignore[attr-defined] + return cache diff --git a/src/recurquant/static_q468_calibration.py b/src/recurquant/static_q468_calibration.py new file mode 100644 index 0000000..b5ccfcb --- /dev/null +++ b/src/recurquant/static_q468_calibration.py @@ -0,0 +1,2830 @@ +"""Pure calibration math and split-half gates for Experiment 013. + +This module deliberately stops at deterministic CPU-FP64 score reduction, +exact code-map allocation, and policy-stability evidence. It does not load a +dataset or model, resolve protected identities, quantize a live state, or open +an evaluation result. + +The frozen reduction is:: + + sequence = mean(anchor_energy * per_row_codec_mse, anchors) + ruler = mean(mean(sequence, each of four categories), categories) + D_b = mean(MBPP_b, PG19_b, RULER_b) + +All input order is canonicalized before floating-point reductions. Artifact +arrays use contiguous little-endian binary64 bytes rather than JSON decimal +renderings, and validation recomputes both exact allocations from those bytes. +""" + +from __future__ import annotations + +import base64 +import binascii +import hashlib +import json +import math +import re +import unicodedata +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Any, Literal, TypeAlias, cast + +import numpy as np +import torch + +from .evidence import canonical_json_bytes +from .metrics import spearman_correlation +from .multibit_policy import allocate_exact_multibit_codes_fast +from .multibit_quantization import _pack_precision_codes +from .static_q468 import ( + FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + FROZEN_STATIC_Q468_ABLATION_STEPS, + FROZEN_STATIC_Q468_PRIMARY_STEPS, + STATIC_Q468_ALLOCATOR_REVISION, + StaticRhtQ468Geometry, + static_q468_distortion_sha256, +) + +CalibrationFamily: TypeAlias = Literal["mbpp", "pg19", "ruler"] +RulerCategory: TypeAlias = Literal[ + "retrieval", + "multi_hop_tracing", + "aggregation", + "question_answering", +] +SplitHalf: TypeAlias = Literal["a", "b"] + +FROZEN_ANCHOR_COUNT = 16 +CALIBRATION_FAMILY_ORDER = ("mbpp", "pg19", "ruler") +RULER_CATEGORY_ORDER: tuple[RulerCategory, ...] = ( + "retrieval", + "multi_hop_tracing", + "aggregation", + "question_answering", +) +CALIBRATION_SPLIT_NAMESPACE = "recurquant.experiment013.calibration-split.v1\0" + +MIN_SPLIT_HALF_SPEARMAN = 0.70 +MIN_SPLIT_HALF_Q8_JACCARD = 0.50 +MAX_LAYER_MEAN_BITWIDTH_SHIFT = 0.25 + +CALIBRATION_SCORE_ARTIFACT_KIND = "recurquant_experiment013_static_q468_scores" +CALIBRATION_SCORE_ARTIFACT_SCHEMA_VERSION = 1 +CALIBRATION_SCORE_ARTIFACT_REVISION = "experiment-013-static-q468-scores-v1" +CALIBRATION_SCORE_ARTIFACT_PROFILE = "experiment-013-qwen35-0.8b-static-q468-frozen-v1" +GENERIC_CALIBRATION_SCORE_ARTIFACT_KIND = "recurquant_static_q468_scores_generic" +GENERIC_CALIBRATION_SCORE_ARTIFACT_REVISION = "static-q468-scores-generic-v1" +GENERIC_CALIBRATION_SCORE_ARTIFACT_PROFILE = "generic-static-q468-calibration-v1" +CALIBRATION_SCORE_DTYPE = "float64-le" + +SPLIT_HALF_STABILITY_ARTIFACT_KIND = "recurquant_experiment013_static_q468_split_half_stability" +SPLIT_HALF_STABILITY_ARTIFACT_SCHEMA_VERSION = 1 +SPLIT_HALF_STABILITY_ARTIFACT_REVISION = "experiment-013-static-q468-split-half-stability-v1" +SPLIT_HALF_STABILITY_ARTIFACT_PROFILE = ( + "experiment-013-qwen35-0.8b-static-k29334-split-half-frozen-v1" +) + +GENERIC_REDUCTION_PROFILE = "generic-anchor-row-reduction-v1" +FROZEN_REDUCTION_PROFILE = "experiment-013-qwen35-0.8b-anchor-reduction-v1" +FROZEN_SOURCE_AXIS_ORDER = ("anchor", "layer", "head", "key_row") +_SOURCE_TENSOR_NAMES = ("query_energy", "q4_mse", "q6_mse", "q8_mse") + +_SHA256_RE = re.compile(r"[0-9a-f]{64}") +_SCORE_HASH_DOMAIN = b"recurquant.experiment013.sequence-scores.v1\0" +_ANCHOR_INPUT_HASH_DOMAIN = b"recurquant.experiment013.anchor-inputs.v1\0" +_CODE_MAP_HASH_DOMAIN = b"recurquant.static-q468-code-map.v1\0" +_IDENTITY_RECORD_HASH_DOMAIN = b"recurquant.experiment013.identity-record.v1\0" +_TOKEN_SPAN_ORDER = ( + "prefill_start", + "prefill_stop", + "scored_start", + "scored_stop", + "cache_exposed_start", + "cache_exposed_stop", +) +_IDENTITY_RECORD_PAYLOAD_FIELDS = frozenset( + { + "family", + "canonical_id", + "config", + "selection_rank", + "selection_sha256", + "seed", + "configured_length", + "sequence_length", + "ruler_category", + "generator_receipt_sha256", + "source_content_sha256", + "formatted_content_sha256", + "prompt_token_ids_sha256", + "target_token_ids_sha256", + "sequence_token_ids_sha256", + "tokenizer_manifest_sha256", + "token_span", + "anchor_manifest_sha256", + } +) +_AGGREGATION_CONTRACT = { + "accumulator": "cpu-float64", + "anchor_reduction": "equal-weight arithmetic mean within each sequence", + "bitwidths": [4, 6, 8], + "broad_family_reduction": "equal-weight arithmetic mean of MBPP, PG19, and RULER", + "ruler_reduction": ( + "equal-weight sequence mean within each of four categories, then " + "equal-weight category macro" + ), + "sequence_reduction": "equal-weight arithmetic mean within MBPP and PG19", +} + + +class CalibrationArtifactError(ValueError): + """Raised when a calibration score artifact fails closed.""" + + +def _strict_positive_int(value: object, *, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{name} must be an integer") + if value <= 0: + raise ValueError(f"{name} must be positive") + return value + + +def _strict_nonnegative_int(value: object, *, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{name} must be an integer") + if value < 0: + raise ValueError(f"{name} must be non-negative") + return value + + +def _canonical_text(value: object, *, name: str) -> str: + if not isinstance(value, str) or not value or value != value.strip(): + raise ValueError(f"{name} must be a non-empty canonical string") + if value != unicodedata.normalize("NFC", value): + raise ValueError(f"{name} must use NFC Unicode normalization") + if "\0" in value: + raise ValueError(f"{name} cannot contain a NUL separator") + return value + + +def _sha256(value: object, *, name: str) -> str: + if not isinstance(value, str) or _SHA256_RE.fullmatch(value) is None: + raise ValueError(f"{name} must be a lowercase SHA-256 digest") + return value + + +def _metadata( + *, + family: object, + config: object, + ruler_category: object, + canonical_id: object, + seed: object, + configured_length: object, + token_count: object, +) -> tuple[ + CalibrationFamily, + str, + RulerCategory | None, + str, + int | None, + int | None, + int, +]: + if family not in CALIBRATION_FAMILY_ORDER: + raise ValueError("family must be one of mbpp, pg19, or ruler") + normalized_family = cast(CalibrationFamily, family) + normalized_config = _canonical_text(config, name="config") + if normalized_family == "ruler": + if ruler_category not in RULER_CATEGORY_ORDER: + raise ValueError( + "ruler_category must be one of retrieval, multi_hop_tracing, " + "aggregation, or question_answering for RULER" + ) + normalized_category = cast(RulerCategory, ruler_category) + normalized_configured_length = _strict_positive_int( + configured_length, + name="configured_length", + ) + else: + if ruler_category is not None: + raise ValueError("ruler_category must be None for MBPP and PG19") + if configured_length is not None: + raise ValueError("configured_length must be None for MBPP and PG19") + normalized_category = None + normalized_configured_length = None + normalized_id = _canonical_text(canonical_id, name="canonical_id") + if seed is not None: + if isinstance(seed, bool) or not isinstance(seed, int): + raise TypeError("seed must be an integer or None") + if seed < 0: + raise ValueError("seed must be non-negative") + normalized_tokens = _strict_positive_int(token_count, name="token_count") + if ( + normalized_configured_length is not None + and normalized_tokens > normalized_configured_length + ): + raise ValueError("token_count cannot exceed the RULER configured_length") + return ( + normalized_family, + normalized_config, + normalized_category, + normalized_id, + seed, + normalized_configured_length, + normalized_tokens, + ) + + +def frozen_anchor_positions(token_count: int) -> tuple[int, ...]: + """Return the frozen unique zero-based post-token anchor positions. + + For ``T >= 16`` this is exactly + ``floor((j + 1) * T / 16) - 1`` for ``j = 0,...,15``. Shorter non-empty + sequences capture every token. + """ + + tokens = _strict_positive_int(token_count, name="token_count") + if tokens < FROZEN_ANCHOR_COUNT: + positions = tuple(range(tokens)) + else: + positions = tuple( + (index + 1) * tokens // FROZEN_ANCHOR_COUNT - 1 for index in range(FROZEN_ANCHOR_COUNT) + ) + if len(positions) != len(set(positions)): + raise RuntimeError("frozen anchor equation produced duplicate positions") + if not positions or positions[0] < 0 or positions[-1] >= tokens: + raise RuntimeError("frozen anchor equation produced an out-of-range position") + return positions + + +def _cpu_fp64_matrix( + value: object, + *, + name: str, + anchors: int, + expected_shape: tuple[int, ...] | None = None, +) -> torch.Tensor: + if not isinstance(value, torch.Tensor): + raise TypeError(f"{name} must be a torch.Tensor") + if value.device.type == "meta": + raise ValueError(f"{name} must be materialized") + if not value.is_floating_point(): + raise TypeError(f"{name} must use a floating-point dtype") + if value.ndim < 2 or value.shape[0] != anchors or value.numel() == 0: + raise ValueError(f"{name} must have shape [anchor, row...] with {anchors} anchors") + if expected_shape is not None and tuple(value.shape) != expected_shape: + raise ValueError(f"{name} must match shape {expected_shape}") + normalized = value.detach().to(device="cpu", dtype=torch.float64).contiguous() + if not torch.isfinite(normalized).all().item(): + raise ValueError(f"{name} must contain only finite values") + if (normalized < 0).any().item(): + raise ValueError(f"{name} must contain only non-negative values") + return normalized.reshape(anchors, -1).clone() + + +def _cpu_fp64_scores(value: object, *, name: str, expected_rows: int | None = None) -> torch.Tensor: + if not isinstance(value, torch.Tensor): + raise TypeError(f"{name} must be a torch.Tensor") + if value.device.type == "meta": + raise ValueError(f"{name} must be materialized") + if not value.is_floating_point(): + raise TypeError(f"{name} must use a floating-point dtype") + if value.numel() == 0: + raise ValueError(f"{name} must be non-empty") + if expected_rows is not None and value.numel() != expected_rows: + raise ValueError(f"{name} must contain exactly {expected_rows} rows") + normalized = value.detach().to(device="cpu", dtype=torch.float64).reshape(-1).contiguous() + if not torch.isfinite(normalized).all().item(): + raise ValueError(f"{name} must contain only finite values") + if (normalized < 0).any().item(): + raise ValueError(f"{name} must contain only non-negative values") + return normalized.clone() + + +def _tensor_bytes(value: torch.Tensor) -> bytes: + array = value.detach().to(device="cpu", dtype=torch.float64).contiguous().numpy() + return array.astype(" None: + _canonical_text(self.reduction_profile, name="reduction_profile") + if not isinstance(self.axis_order, tuple) or len(self.axis_order) < 2: + raise ValueError("source axis_order must contain anchor and at least one row axis") + for axis in self.axis_order: + _canonical_text(axis, name="source axis name") + if self.axis_order[0] != "anchor" or len(set(self.axis_order)) != len(self.axis_order): + raise ValueError("source axis_order must start with unique anchor axis") + if ( + not isinstance(self.trailing_shape, tuple) + or len(self.trailing_shape) != len(self.axis_order) - 1 + ): + raise ValueError("source trailing_shape must match non-anchor axes") + for index, extent in enumerate(self.trailing_shape): + _strict_positive_int(extent, name=f"source trailing_shape[{index}]") + if not isinstance(self.dtypes, tuple): + raise TypeError("source dtypes must be a tuple") + if tuple(name for name, _dtype in self.dtypes) != _SOURCE_TENSOR_NAMES: + raise ValueError("source dtypes must cover query_energy, q4_mse, q6_mse, and q8_mse") + for name, dtype in self.dtypes: + _canonical_text(name, name="source tensor name") + _canonical_text(dtype, name=f"{name} source dtype") + + def canonical_dict(self) -> dict[str, object]: + return { + "axis_order": list(self.axis_order), + "dtypes": {name: dtype for name, dtype in self.dtypes}, + "reduction_profile": self.reduction_profile, + "trailing_shape": list(self.trailing_shape), + } + + +FROZEN_SOURCE_TENSOR_CONTRACT = CalibrationSourceTensorContract( + reduction_profile=FROZEN_REDUCTION_PROFILE, + axis_order=FROZEN_SOURCE_AXIS_ORDER, + trailing_shape=( + FROZEN_QWEN35_STATIC_Q468_GEOMETRY.layers, + FROZEN_QWEN35_STATIC_Q468_GEOMETRY.heads, + FROZEN_QWEN35_STATIC_Q468_GEOMETRY.key_rows, + ), + dtypes=tuple((name, "torch.float64") for name in _SOURCE_TENSOR_NAMES), +) + + +def _hash_score_triplet( + d4: torch.Tensor, + d6: torch.Tensor, + d8: torch.Tensor, + *, + domain: bytes, + metadata: dict[str, object], +) -> str: + digest = hashlib.sha256() + digest.update(domain) + digest.update(canonical_json_bytes(metadata)) + for label, tensor in zip((b"D4\0", b"D6\0", b"D8\0"), (d4, d6, d8), strict=True): + digest.update(label) + digest.update(_tensor_bytes(tensor)) + return digest.hexdigest() + + +def _resolver_canonical_json_bytes(value: object) -> bytes: + """Match the compact canonical JSON used by the identity resolver.""" + + return ( + json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + b"\n" + ) + + +def sequence_token_ids_sha256(token_ids: Sequence[int]) -> str: + """Hash exact ordered token IDs with the capture/resolver representation.""" + + if isinstance(token_ids, (str, bytes, bytearray)) or not isinstance(token_ids, Sequence): + raise TypeError("sequence_token_ids must be an integer sequence") + normalized: list[int] = [] + for index, token_id in enumerate(token_ids): + if isinstance(token_id, bool) or not isinstance(token_id, int) or token_id < 0: + raise ValueError(f"sequence_token_ids[{index}] must be a non-negative integer") + normalized.append(token_id) + if not normalized: + raise ValueError("sequence_token_ids cannot be empty") + return hashlib.sha256(_resolver_canonical_json_bytes(normalized)).hexdigest() + + +def _normalize_token_span(value: object, *, sequence_length: int) -> tuple[tuple[str, int], ...]: + if not isinstance(value, Mapping) or set(value) != set(_TOKEN_SPAN_ORDER): + raise ValueError("token_span must contain the exact six capture span fields") + normalized: list[tuple[str, int]] = [] + for name in _TOKEN_SPAN_ORDER: + raw = value[name] + if isinstance(raw, bool) or not isinstance(raw, int) or raw < 0: + raise ValueError(f"token_span.{name} must be a non-negative integer") + normalized.append((name, raw)) + span = dict(normalized) + if ( + span["prefill_start"] != 0 + or span["prefill_stop"] != span["scored_start"] + or span["prefill_stop"] < 1 + or span["scored_stop"] < span["scored_start"] + or span["scored_stop"] != sequence_length + ): + raise ValueError("token_span is not contiguous and canonical") + if ( + span["cache_exposed_start"] != span["scored_stop"] + or span["cache_exposed_stop"] != span["scored_stop"] + ): + raise ValueError("calibration cache-exposed span must be empty at continuation stop") + return tuple(normalized) + + +def identity_anchor_manifest_sha256( + *, + canonical_id: str, + sequence_length: int, + sequence_token_ids_sha256_value: str, + token_span: tuple[tuple[str, int], ...] | Mapping[str, int], +) -> str: + """Recompute the exact compact-JSON anchor hash emitted by capture.""" + + normalized_id = _canonical_text(canonical_id, name="canonical_id") + length = _strict_positive_int(sequence_length, name="sequence_length") + token_hash = _sha256( + sequence_token_ids_sha256_value, + name="sequence_token_ids_sha256", + ) + span = _normalize_token_span(dict(token_span), sequence_length=length) + manifest = { + "canonical_id": normalized_id, + "positions": list(frozen_anchor_positions(length)), + "sequence_token_ids_sha256": token_hash, + "token_span": dict(span), + } + return hashlib.sha256(_resolver_canonical_json_bytes(manifest)).hexdigest() + + +def identity_record_sha256(record: Mapping[str, object]) -> str: + """Recompute the domain-separated capture record self-hash.""" + + missing = _IDENTITY_RECORD_PAYLOAD_FIELDS - set(record) + if missing: + raise ValueError(f"identity record is missing fields: {sorted(missing)}") + payload = {name: record[name] for name in sorted(_IDENTITY_RECORD_PAYLOAD_FIELDS)} + return hashlib.sha256( + _IDENTITY_RECORD_HASH_DOMAIN + _resolver_canonical_json_bytes(payload) + ).hexdigest() + + +def calibration_identity_record_manifest_sha256( + records: Sequence[Mapping[str, object]], +) -> str: + """Hash ordered calibration identity-record commitments for score binding.""" + + entries: list[dict[str, object]] = [] + for index, record in enumerate(records): + try: + family = record["family"] + category = record["ruler_category"] + config = record["config"] + canonical_id = record["canonical_id"] + seed = record["seed"] + configured_length = record["configured_length"] + sequence_length = record["sequence_length"] + record_hash = record["identity_record_sha256"] + except KeyError as exc: + raise ValueError(f"identity manifest record {index} is incomplete") from exc + _metadata( + family=family, + config=config, + ruler_category=category, + canonical_id=canonical_id, + seed=seed, + configured_length=configured_length, + token_count=sequence_length, + ) + entries.append( + { + "canonical_id": canonical_id, + "config": config, + "configured_length": configured_length, + "family": family, + "identity_record_sha256": _sha256( + record_hash, + name=f"identity manifest record {index} SHA-256", + ), + "ruler_category": category, + "seed": seed, + "sequence_length": sequence_length, + } + ) + if not entries: + raise ValueError("identity record manifest cannot be empty") + entries.sort( + key=lambda item: ( + CALIBRATION_FAMILY_ORDER.index(cast(str, item["family"])), + "" if item["ruler_category"] is None else str(item["ruler_category"]), + str(item["config"]), + str(item["canonical_id"]), + -1 if item["seed"] is None else cast(int, item["seed"]), + -1 if item["configured_length"] is None else cast(int, item["configured_length"]), + cast(int, item["sequence_length"]), + ) + ) + identities = [ + ( + item["family"], + item["ruler_category"], + item["config"], + item["canonical_id"], + item["seed"], + item["configured_length"], + item["sequence_length"], + ) + for item in entries + ] + if len(identities) != len(set(identities)): + raise ValueError("identity record manifest contains duplicate sequence identities") + return hashlib.sha256(_resolver_canonical_json_bytes(entries)).hexdigest() + + +@dataclass(frozen=True, slots=True) +class AnchorDistortionBatch: + """Unaggregated per-anchor energy and codec MSE for one sequence.""" + + family: CalibrationFamily + config: str + ruler_category: RulerCategory | None + canonical_id: str + seed: int | None + configured_length: int | None + token_count: int + anchor_positions: tuple[int, ...] + query_energy: torch.Tensor + q4_mse: torch.Tensor + q6_mse: torch.Tensor + q8_mse: torch.Tensor + sequence_token_ids: tuple[int, ...] | None = None + identity_record: Mapping[str, object] | None = None + + +@dataclass(frozen=True, slots=True) +class CalibrationSequenceScores: + """CPU-FP64 per-row scores after the frozen within-sequence mean.""" + + family: CalibrationFamily + config: str + ruler_category: RulerCategory | None + canonical_id: str + seed: int | None + configured_length: int | None + token_count: int + anchor_positions: tuple[int, ...] + anchor_manifest_sha256: str + anchor_inputs_sha256: str + sequence_scores_sha256: str + d4: torch.Tensor + d6: torch.Tensor + d8: torch.Tensor + source_contract: CalibrationSourceTensorContract + source_shape: tuple[int, ...] + sequence_token_ids_sha256: str | None = None + token_span: tuple[tuple[str, int], ...] | None = None + identity_anchor_manifest_sha256: str | None = None + identity_record_sha256: str | None = None + + @property + def row_count(self) -> int: + return self.d4.numel() + + def identity_tuple(self) -> tuple[object, ...]: + return ( + self.family, + self.ruler_category, + self.config, + self.canonical_id, + self.seed, + self.configured_length, + self.token_count, + ) + + def manifest_record(self) -> dict[str, object]: + return { + "anchor_inputs_sha256": self.anchor_inputs_sha256, + "anchor_manifest_sha256": self.anchor_manifest_sha256, + "anchor_positions": list(self.anchor_positions), + "canonical_id": self.canonical_id, + "config": self.config, + "family": self.family, + "ruler_category": self.ruler_category, + "seed": self.seed, + "configured_length": self.configured_length, + "source_shape": list(self.source_shape), + "source_tensor_contract": self.source_contract.canonical_dict(), + "sequence_token_ids_sha256": self.sequence_token_ids_sha256, + "token_span": None if self.token_span is None else dict(self.token_span), + "identity_anchor_manifest_sha256": self.identity_anchor_manifest_sha256, + "identity_record_sha256": self.identity_record_sha256, + "sequence_scores_sha256": self.sequence_scores_sha256, + "token_count": self.token_count, + } + + +@dataclass(frozen=True, slots=True) +class _ValidatedIdentityLineage: + sequence_token_ids_sha256: str + token_span: tuple[tuple[str, int], ...] + identity_anchor_manifest_sha256: str + identity_record_sha256: str + + +def _validate_frozen_identity_lineage( + batch: AnchorDistortionBatch, +) -> _ValidatedIdentityLineage: + token_ids = batch.sequence_token_ids + record = batch.identity_record + if token_ids is None or record is None: + raise ValueError( + "frozen reduction requires sequence_token_ids and an exact capture identity_record" + ) + if not isinstance(token_ids, tuple): + raise TypeError("frozen sequence_token_ids must be a tuple") + if not isinstance(record, Mapping): + raise TypeError("frozen identity_record must be a mapping") + allowed_fields = _IDENTITY_RECORD_PAYLOAD_FIELDS | { + "identity_record_sha256", + "anchor_positions", + "anchor_positions_sha256", + } + if set(record) not in ( + _IDENTITY_RECORD_PAYLOAD_FIELDS | {"identity_record_sha256"}, + allowed_fields, + ): + missing = sorted( + (_IDENTITY_RECORD_PAYLOAD_FIELDS | {"identity_record_sha256"}) - set(record) + ) + extra = sorted(set(record) - allowed_fields) + raise ValueError(f"identity_record fields drifted; missing={missing}, extra={extra}") + + token_hash = sequence_token_ids_sha256(token_ids) + if len(token_ids) != batch.token_count or record["sequence_length"] != batch.token_count: + raise ValueError("identity sequence length differs from the processed token count") + if record["sequence_token_ids_sha256"] != token_hash: + raise ValueError("sequence token-ID SHA-256 differs from the capture identity") + for field, actual in ( + ("family", batch.family), + ("config", batch.config), + ("canonical_id", batch.canonical_id), + ("ruler_category", batch.ruler_category), + ("seed", batch.seed), + ("configured_length", batch.configured_length), + ): + if record[field] != actual: + raise ValueError(f"identity_record.{field} differs from the reduction batch") + span = _normalize_token_span(record["token_span"], sequence_length=batch.token_count) + computed_anchor_hash = identity_anchor_manifest_sha256( + canonical_id=batch.canonical_id, + sequence_length=batch.token_count, + sequence_token_ids_sha256_value=token_hash, + token_span=span, + ) + recorded_anchor_hash = _sha256( + record["anchor_manifest_sha256"], + name="identity_record.anchor_manifest_sha256", + ) + if recorded_anchor_hash != computed_anchor_hash: + raise ValueError("identity anchor manifest SHA-256 does not recompute") + recorded_record_hash = _sha256( + record["identity_record_sha256"], + name="identity_record.identity_record_sha256", + ) + computed_record_hash = identity_record_sha256(record) + if recorded_record_hash != computed_record_hash: + raise ValueError("identity record SHA-256 does not recompute") + + positions = frozen_anchor_positions(batch.token_count) + if "anchor_positions" in record: + if record["anchor_positions"] != list(positions): + raise ValueError("resolver anchor positions differ from the frozen anchor equation") + positions_hash = hashlib.sha256(_resolver_canonical_json_bytes(positions)).hexdigest() + if record["anchor_positions_sha256"] != positions_hash: + raise ValueError("resolver anchor-position SHA-256 does not recompute") + return _ValidatedIdentityLineage( + sequence_token_ids_sha256=token_hash, + token_span=span, + identity_anchor_manifest_sha256=computed_anchor_hash, + identity_record_sha256=computed_record_hash, + ) + + +def _reduce_anchor_distortions( + batch: AnchorDistortionBatch, + *, + source_contract: CalibrationSourceTensorContract, + identity_lineage: _ValidatedIdentityLineage | None, +) -> CalibrationSequenceScores: + """Apply energy weighting after validating a declared source contract.""" + + if not isinstance(batch, AnchorDistortionBatch): + raise TypeError("batch must be an AnchorDistortionBatch") + ( + family, + config, + ruler_category, + canonical_id, + seed, + configured_length, + token_count, + ) = _metadata( + family=batch.family, + config=batch.config, + ruler_category=batch.ruler_category, + canonical_id=batch.canonical_id, + seed=batch.seed, + configured_length=batch.configured_length, + token_count=batch.token_count, + ) + expected_positions = frozen_anchor_positions(token_count) + if not isinstance(batch.anchor_positions, tuple): + raise TypeError("anchor_positions must be a tuple") + if batch.anchor_positions != expected_positions: + raise ValueError("anchor_positions differ from the frozen 16-anchor equation") + if len(batch.anchor_positions) != len(set(batch.anchor_positions)): + raise ValueError("anchor_positions must be unique") + + anchors = len(expected_positions) + energy = _cpu_fp64_matrix(batch.query_energy, name="query_energy", anchors=anchors) + source_shape = tuple(batch.query_energy.shape) + if source_shape != (anchors, *source_contract.trailing_shape): + raise ValueError( + "query_energy shape differs from the declared source tensor contract: " + f"expected {(anchors, *source_contract.trailing_shape)}, got {source_shape}" + ) + source_dtypes = tuple((name, str(getattr(batch, name).dtype)) for name in _SOURCE_TENSOR_NAMES) + if source_dtypes != source_contract.dtypes: + raise ValueError("source tensor dtypes differ from the declared source tensor contract") + q4 = _cpu_fp64_matrix( + batch.q4_mse, + name="q4_mse", + anchors=anchors, + expected_shape=source_shape, + ) + q6 = _cpu_fp64_matrix( + batch.q6_mse, + name="q6_mse", + anchors=anchors, + expected_shape=source_shape, + ) + q8 = _cpu_fp64_matrix( + batch.q8_mse, + name="q8_mse", + anchors=anchors, + expected_shape=source_shape, + ) + + weighted = tuple((energy * distortion).mean(dim=0) for distortion in (q4, q6, q8)) + if any(not torch.isfinite(value).all().item() for value in weighted): + raise ValueError("energy-weighted anchor aggregation produced a non-finite score") + + identity = { + "canonical_id": canonical_id, + "config": config, + "family": family, + "ruler_category": ruler_category, + "seed": seed, + "configured_length": configured_length, + "source_shape": list(source_shape), + "source_tensor_contract": source_contract.canonical_dict(), + "sequence_token_ids_sha256": ( + None if identity_lineage is None else identity_lineage.sequence_token_ids_sha256 + ), + "token_span": (None if identity_lineage is None else dict(identity_lineage.token_span)), + "identity_anchor_manifest_sha256": ( + None if identity_lineage is None else identity_lineage.identity_anchor_manifest_sha256 + ), + "identity_record_sha256": ( + None if identity_lineage is None else identity_lineage.identity_record_sha256 + ), + "token_count": token_count, + } + anchor_manifest = { + **identity, + "anchor_positions": list(expected_positions), + } + anchor_manifest_sha256 = hashlib.sha256(canonical_json_bytes(anchor_manifest)).hexdigest() + + input_digest = hashlib.sha256() + input_digest.update(_ANCHOR_INPUT_HASH_DOMAIN) + input_digest.update(canonical_json_bytes(anchor_manifest)) + for label, tensor in zip( + (b"ENERGY\0", b"MSE4\0", b"MSE6\0", b"MSE8\0"), + (energy, q4, q6, q8), + strict=True, + ): + input_digest.update(label) + input_digest.update(_tensor_bytes(tensor)) + + score_metadata = {**anchor_manifest, "row_count": energy.shape[1]} + score_sha256 = _hash_score_triplet( + *weighted, + domain=_SCORE_HASH_DOMAIN, + metadata=score_metadata, + ) + return CalibrationSequenceScores( + family=family, + config=config, + ruler_category=ruler_category, + canonical_id=canonical_id, + seed=seed, + configured_length=configured_length, + token_count=token_count, + anchor_positions=expected_positions, + anchor_manifest_sha256=anchor_manifest_sha256, + anchor_inputs_sha256=input_digest.hexdigest(), + sequence_scores_sha256=score_sha256, + d4=weighted[0].contiguous(), + d6=weighted[1].contiguous(), + d8=weighted[2].contiguous(), + source_contract=source_contract, + source_shape=source_shape, + sequence_token_ids_sha256=( + None if identity_lineage is None else identity_lineage.sequence_token_ids_sha256 + ), + token_span=None if identity_lineage is None else identity_lineage.token_span, + identity_anchor_manifest_sha256=( + None if identity_lineage is None else identity_lineage.identity_anchor_manifest_sha256 + ), + identity_record_sha256=( + None if identity_lineage is None else identity_lineage.identity_record_sha256 + ), + ) + + +def reduce_anchor_distortions(batch: AnchorDistortionBatch) -> CalibrationSequenceScores: + """Apply the generic, explicitly shape-bound equal-anchor CPU-FP64 reduction.""" + + if not isinstance(batch, AnchorDistortionBatch): + raise TypeError("batch must be an AnchorDistortionBatch") + if not isinstance(batch.query_energy, torch.Tensor): + raise TypeError("query_energy must be a torch.Tensor") + shape = tuple(batch.query_energy.shape) + if len(shape) < 2: + raise ValueError("query_energy must have shape [anchor, row...]") + axis_order = ( + ("anchor", "flattened_row") + if len(shape) == 2 + else ("anchor", *(f"row_axis_{index}" for index in range(len(shape) - 1))) + ) + dtypes: list[tuple[str, str]] = [] + for name in _SOURCE_TENSOR_NAMES: + value = getattr(batch, name) + if not isinstance(value, torch.Tensor): + raise TypeError(f"{name} must be a torch.Tensor") + dtypes.append((name, str(value.dtype))) + contract = CalibrationSourceTensorContract( + reduction_profile=GENERIC_REDUCTION_PROFILE, + axis_order=axis_order, + trailing_shape=shape[1:], + dtypes=tuple(dtypes), + ) + return _reduce_anchor_distortions( + batch, + source_contract=contract, + identity_lineage=None, + ) + + +def reduce_frozen_anchor_distortions( + batch: AnchorDistortionBatch, +) -> CalibrationSequenceScores: + """Apply the official Experiment 013 reduction with exact CPU-FP64 source axes.""" + + if not isinstance(batch, AnchorDistortionBatch): + raise TypeError("batch must be an AnchorDistortionBatch") + anchors = len(frozen_anchor_positions(batch.token_count)) + expected_shape = (anchors, *FROZEN_SOURCE_TENSOR_CONTRACT.trailing_shape) + for name in _SOURCE_TENSOR_NAMES: + value = getattr(batch, name) + if not isinstance(value, torch.Tensor): + raise TypeError(f"{name} must be a torch.Tensor") + if tuple(value.shape) != expected_shape: + raise ValueError( + f"{name} must have frozen source shape {expected_shape}; got {tuple(value.shape)}" + ) + if value.device.type != "cpu" or value.dtype != torch.float64: + raise TypeError(f"{name} must be a CPU torch.float64 tensor for frozen reduction") + identity_lineage = _validate_frozen_identity_lineage(batch) + return _reduce_anchor_distortions( + batch, + source_contract=FROZEN_SOURCE_TENSOR_CONTRACT, + identity_lineage=identity_lineage, + ) + + +def _sequence_sort_key(sequence: CalibrationSequenceScores) -> tuple[object, ...]: + return ( + CALIBRATION_FAMILY_ORDER.index(sequence.family), + "" if sequence.ruler_category is None else sequence.ruler_category, + sequence.config, + sequence.canonical_id, + -1 if sequence.seed is None else sequence.seed, + -1 if sequence.configured_length is None else sequence.configured_length, + sequence.token_count, + ) + + +def _validate_sequence_score(sequence: object, *, expected_rows: int | None) -> None: + if not isinstance(sequence, CalibrationSequenceScores): + raise TypeError("sequences must contain CalibrationSequenceScores") + _metadata( + family=sequence.family, + config=sequence.config, + ruler_category=sequence.ruler_category, + canonical_id=sequence.canonical_id, + seed=sequence.seed, + configured_length=sequence.configured_length, + token_count=sequence.token_count, + ) + if sequence.anchor_positions != frozen_anchor_positions(sequence.token_count): + raise ValueError("sequence anchor positions differ from the frozen equation") + _sha256(sequence.anchor_manifest_sha256, name="anchor_manifest_sha256") + _sha256(sequence.anchor_inputs_sha256, name="anchor_inputs_sha256") + _sha256(sequence.sequence_scores_sha256, name="sequence_scores_sha256") + if not isinstance(sequence.source_contract, CalibrationSourceTensorContract): + raise TypeError("source_contract must be a CalibrationSourceTensorContract") + if not isinstance(sequence.source_shape, tuple) or len(sequence.source_shape) != len( + sequence.source_contract.axis_order + ): + raise ValueError("source_shape must match the declared source axis order") + for index, extent in enumerate(sequence.source_shape): + _strict_positive_int(extent, name=f"source_shape[{index}]") + if sequence.source_shape != ( + len(sequence.anchor_positions), + *sequence.source_contract.trailing_shape, + ): + raise ValueError("source_shape differs from anchor count or source tensor contract") + rows = expected_rows if expected_rows is not None else sequence.d4.numel() + if math.prod(sequence.source_contract.trailing_shape) != rows: + raise ValueError("source tensor contract does not flatten to the score row count") + lineage_values = ( + sequence.sequence_token_ids_sha256, + sequence.token_span, + sequence.identity_anchor_manifest_sha256, + sequence.identity_record_sha256, + ) + if sequence.source_contract == FROZEN_SOURCE_TENSOR_CONTRACT: + if any(value is None for value in lineage_values): + raise ValueError("frozen sequence scores require complete capture identity lineage") + token_hash = _sha256( + sequence.sequence_token_ids_sha256, + name="sequence_token_ids_sha256", + ) + span = _normalize_token_span( + dict(cast(tuple[tuple[str, int], ...], sequence.token_span)), + sequence_length=sequence.token_count, + ) + anchor_hash = identity_anchor_manifest_sha256( + canonical_id=sequence.canonical_id, + sequence_length=sequence.token_count, + sequence_token_ids_sha256_value=token_hash, + token_span=span, + ) + if anchor_hash != sequence.identity_anchor_manifest_sha256: + raise ValueError("identity anchor manifest SHA-256 does not match sequence lineage") + _sha256(sequence.identity_record_sha256, name="identity_record_sha256") + elif any(value is not None for value in lineage_values): + raise ValueError("generic sequence scores cannot claim frozen capture identity lineage") + values = tuple( + _cpu_fp64_scores(value, name=name, expected_rows=rows) + for name, value in (("D4", sequence.d4), ("D6", sequence.d6), ("D8", sequence.d8)) + ) + score_metadata = { + "anchor_positions": list(sequence.anchor_positions), + "canonical_id": sequence.canonical_id, + "config": sequence.config, + "family": sequence.family, + "ruler_category": sequence.ruler_category, + "row_count": rows, + "seed": sequence.seed, + "configured_length": sequence.configured_length, + "source_shape": list(sequence.source_shape), + "source_tensor_contract": sequence.source_contract.canonical_dict(), + "sequence_token_ids_sha256": sequence.sequence_token_ids_sha256, + "token_span": None if sequence.token_span is None else dict(sequence.token_span), + "identity_anchor_manifest_sha256": sequence.identity_anchor_manifest_sha256, + "identity_record_sha256": sequence.identity_record_sha256, + "token_count": sequence.token_count, + } + computed = _hash_score_triplet( + *values, + domain=_SCORE_HASH_DOMAIN, + metadata=score_metadata, + ) + if computed != sequence.sequence_scores_sha256: + raise ValueError("sequence_scores_sha256 does not match the score arrays") + + +@dataclass(frozen=True, slots=True) +class CalibrationAggregate: + """Frozen broad-family macro scores and their ordered source manifest.""" + + d4: torch.Tensor + d6: torch.Tensor + d8: torch.Tensor + family_sequence_counts: tuple[tuple[str, int], ...] + ruler_category_sequence_counts: tuple[tuple[str, int], ...] + sequence_score_manifest_sha256: str + source_contract: CalibrationSourceTensorContract + identity_record_manifest_sha256: str | None = None + + @property + def row_count(self) -> int: + return self.d4.numel() + + def scores(self) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + return self.d4, self.d6, self.d8 + + +def _mean_score_rows(rows: list[torch.Tensor], *, context: str) -> torch.Tensor: + if not rows: + raise ValueError(f"{context} cannot be empty") + result = torch.stack(rows, dim=0).mean(dim=0) + if result.dtype != torch.float64 or result.device.type != "cpu": + raise RuntimeError("calibration aggregation left the CPU-FP64 contract") + if not torch.isfinite(result).all().item(): + raise ValueError(f"{context} produced a non-finite aggregate") + return result.contiguous() + + +def aggregate_calibration_scores( + sequences: list[CalibrationSequenceScores] | tuple[CalibrationSequenceScores, ...], +) -> CalibrationAggregate: + """Compute the frozen MBPP + PG19 + equal-RULER-category macro.""" + + if not isinstance(sequences, (list, tuple)) or not sequences: + raise ValueError("sequences must be a non-empty list or tuple") + ordered = sorted(sequences, key=_sequence_sort_key) + expected_rows: int | None = None + identities: set[tuple[object, ...]] = set() + for sequence in ordered: + _validate_sequence_score(sequence, expected_rows=expected_rows) + expected_rows = sequence.row_count if expected_rows is None else expected_rows + identity = sequence.identity_tuple() + if identity in identities: + raise ValueError(f"duplicate calibration sequence identity: {identity!r}") + identities.add(identity) + assert expected_rows is not None + source_contracts = {sequence.source_contract for sequence in ordered} + if len(source_contracts) != 1: + raise ValueError("all calibration sequences must use one source tensor contract") + source_contract = next(iter(source_contracts)) + if source_contract == FROZEN_SOURCE_TENSOR_CONTRACT: + lineage_records = [ + { + "family": sequence.family, + "ruler_category": sequence.ruler_category, + "config": sequence.config, + "canonical_id": sequence.canonical_id, + "seed": sequence.seed, + "configured_length": sequence.configured_length, + "sequence_length": sequence.token_count, + "identity_record_sha256": sequence.identity_record_sha256, + } + for sequence in ordered + ] + identity_manifest_sha256 = calibration_identity_record_manifest_sha256(lineage_records) + else: + identity_manifest_sha256 = None + + grouped = { + family: [sequence for sequence in ordered if sequence.family == family] + for family in CALIBRATION_FAMILY_ORDER + } + if any(not grouped[family] for family in CALIBRATION_FAMILY_ORDER): + raise ValueError("MBPP, PG19, and RULER must each contain at least one sequence") + ruler_categories = tuple( + category + for category in RULER_CATEGORY_ORDER + if any(sequence.ruler_category == category for sequence in grouped["ruler"]) + ) + if ruler_categories != RULER_CATEGORY_ORDER: + raise ValueError("RULER calibration must contain all four frozen categories") + + aggregate_by_bit: list[torch.Tensor] = [] + for attribute in ("d4", "d6", "d8"): + mbpp = _mean_score_rows( + [getattr(sequence, attribute) for sequence in grouped["mbpp"]], + context=f"MBPP {attribute}", + ) + pg19 = _mean_score_rows( + [getattr(sequence, attribute) for sequence in grouped["pg19"]], + context=f"PG19 {attribute}", + ) + ruler_category_means = [ + _mean_score_rows( + [ + getattr(sequence, attribute) + for sequence in grouped["ruler"] + if sequence.ruler_category == category + ], + context=f"RULER {category} {attribute}", + ) + for category in ruler_categories + ] + ruler = _mean_score_rows(ruler_category_means, context=f"RULER macro {attribute}") + aggregate_by_bit.append( + _mean_score_rows([mbpp, pg19, ruler], context=f"broad-family macro {attribute}") + ) + + manifest = [sequence.manifest_record() for sequence in ordered] + manifest_sha256 = hashlib.sha256(canonical_json_bytes(manifest)).hexdigest() + family_counts = tuple((family, len(grouped[family])) for family in CALIBRATION_FAMILY_ORDER) + ruler_counts = tuple( + ( + category, + sum(sequence.ruler_category == category for sequence in grouped["ruler"]), + ) + for category in ruler_categories + ) + return CalibrationAggregate( + d4=aggregate_by_bit[0], + d6=aggregate_by_bit[1], + d8=aggregate_by_bit[2], + family_sequence_counts=family_counts, + ruler_category_sequence_counts=ruler_counts, + sequence_score_manifest_sha256=manifest_sha256, + source_contract=source_contract, + identity_record_manifest_sha256=identity_manifest_sha256, + ) + + +def calibration_sequence_rank_sha256(sequence: CalibrationSequenceScores) -> str: + """Hash the exact domain-separated identity used by the frozen resolver.""" + + _validate_sequence_score(sequence, expected_rows=None) + return _calibration_sequence_rank_sha256_unchecked(sequence) + + +def _calibration_sequence_rank_sha256_unchecked( + sequence: CalibrationSequenceScores, +) -> str: + identity = "\0".join( + ( + sequence.family, + str(sequence.ruler_category), + sequence.config, + sequence.canonical_id, + str(sequence.seed), + str(sequence.configured_length), + str(sequence.token_count), + ) + ) + return hashlib.sha256( + CALIBRATION_SPLIT_NAMESPACE.encode("utf-8") + identity.encode("utf-8") + ).hexdigest() + + +@dataclass(frozen=True, slots=True) +class SplitAssignment: + group: str + canonical_id: str + config: str + ruler_category: RulerCategory | None + configured_length: int | None + sequence_length: int + seed: int | None + rank: int + half: SplitHalf + rank_sha256: str + + def canonical_dict(self) -> dict[str, object]: + return { + "canonical_id": self.canonical_id, + "config": self.config, + "configured_length": self.configured_length, + "group": self.group, + "half": self.half, + "rank": self.rank, + "rank_sha256": self.rank_sha256, + "ruler_category": self.ruler_category, + "seed": self.seed, + "sequence_length": self.sequence_length, + } + + +@dataclass(frozen=True, slots=True) +class CalibrationHalfSplit: + half_a: tuple[CalibrationSequenceScores, ...] + half_b: tuple[CalibrationSequenceScores, ...] + assignments: tuple[SplitAssignment, ...] + assignment_sha256: str + + +def balanced_sha_rank_halves( + sequences: list[CalibrationSequenceScores] | tuple[CalibrationSequenceScores, ...], +) -> CalibrationHalfSplit: + """Alternate SHA-ranked identities within MBPP, PG19, and each RULER family.""" + + if not isinstance(sequences, (list, tuple)) or not sequences: + raise ValueError("sequences must be a non-empty list or tuple") + ordered = sorted(sequences, key=_sequence_sort_key) + expected_rows: int | None = None + identities: set[tuple[object, ...]] = set() + for sequence in ordered: + _validate_sequence_score(sequence, expected_rows=expected_rows) + expected_rows = sequence.row_count if expected_rows is None else expected_rows + if sequence.identity_tuple() in identities: + raise ValueError("calibration split contains a duplicate sequence identity") + identities.add(sequence.identity_tuple()) + + rank_hashes = { + sequence.identity_tuple(): _calibration_sequence_rank_sha256_unchecked(sequence) + for sequence in ordered + } + + ruler_categories = tuple( + category + for category in RULER_CATEGORY_ORDER + if any( + sequence.family == "ruler" and sequence.ruler_category == category + for sequence in ordered + ) + ) + if ruler_categories != RULER_CATEGORY_ORDER: + raise ValueError("calibration split requires all four frozen RULER categories") + groups = ( + ("mbpp", [sequence for sequence in ordered if sequence.family == "mbpp"]), + ("pg19", [sequence for sequence in ordered if sequence.family == "pg19"]), + *( + ( + f"ruler:{category}", + [ + sequence + for sequence in ordered + if sequence.family == "ruler" and sequence.ruler_category == category + ], + ) + for category in ruler_categories + ), + ) + + half_a: list[CalibrationSequenceScores] = [] + half_b: list[CalibrationSequenceScores] = [] + assignments: list[SplitAssignment] = [] + for group, members in groups: + if len(members) < 2: + raise ValueError(f"split group {group!r} needs at least two sequences") + ranked = sorted( + members, + key=lambda sequence: ( + rank_hashes[sequence.identity_tuple()], + sequence.canonical_id, + ), + ) + for rank, sequence in enumerate(ranked): + half: SplitHalf = "a" if rank % 2 == 0 else "b" + (half_a if half == "a" else half_b).append(sequence) + assignments.append( + SplitAssignment( + group=group, + canonical_id=sequence.canonical_id, + config=sequence.config, + ruler_category=sequence.ruler_category, + configured_length=sequence.configured_length, + sequence_length=sequence.token_count, + seed=sequence.seed, + rank=rank, + half=half, + rank_sha256=rank_hashes[sequence.identity_tuple()], + ) + ) + + assignments.sort(key=lambda item: (item.group, item.rank)) + assignment_payload = [item.canonical_dict() for item in assignments] + assignment_sha256 = hashlib.sha256( + _resolver_canonical_json_bytes(assignment_payload) + ).hexdigest() + return CalibrationHalfSplit( + half_a=tuple(sorted(half_a, key=_sequence_sort_key)), + half_b=tuple(sorted(half_b, key=_sequence_sort_key)), + assignments=tuple(assignments), + assignment_sha256=assignment_sha256, + ) + + +def allocate_static_q468_code_map( + aggregate: CalibrationAggregate, + *, + marginal_steps: int, +) -> torch.Tensor: + """Feed aggregate D4/D6/D8 directly to the existing exact allocator.""" + + if not isinstance(aggregate, CalibrationAggregate): + raise TypeError("aggregate must be a CalibrationAggregate") + rows = aggregate.row_count + steps = _strict_nonnegative_int(marginal_steps, name="marginal_steps") + if steps > 2 * rows: + raise ValueError("marginal_steps exceeds two steps per row") + scores = tuple( + _cpu_fp64_scores(value, name=name, expected_rows=rows).reshape(1, rows) + for name, value in (("D4", aggregate.d4), ("D6", aggregate.d6), ("D8", aggregate.d8)) + ) + result = allocate_exact_multibit_codes_fast(*scores, marginal_steps=steps).reshape(-1) + if result.dtype != torch.uint8 or result.device.type != "cpu": + raise RuntimeError("exact allocator returned a non-canonical code map") + if int(result.to(torch.int64).sum().item()) != steps: + raise RuntimeError("exact allocator did not satisfy the requested marginal budget") + return result.contiguous() + + +def allocate_frozen_static_q468_code_maps( + aggregate: CalibrationAggregate, +) -> dict[int, torch.Tensor]: + """Allocate the frozen K27030 diagnostic and K29334 primary maps.""" + + if aggregate.row_count != FROZEN_QWEN35_STATIC_Q468_GEOMETRY.total_rows: + raise ValueError( + f"frozen Q468 maps require exactly {FROZEN_QWEN35_STATIC_Q468_GEOMETRY.total_rows} rows" + ) + return { + steps: allocate_static_q468_code_map(aggregate, marginal_steps=steps) + for steps in ( + FROZEN_STATIC_Q468_ABLATION_STEPS, + FROZEN_STATIC_Q468_PRIMARY_STEPS, + ) + } + + +def _precision_codes(value: object, *, name: str, expected_rows: int | None = None) -> torch.Tensor: + if not isinstance(value, torch.Tensor): + raise TypeError(f"{name} must be a torch.Tensor") + if value.device.type == "meta": + raise ValueError(f"{name} must be materialized") + if value.dtype != torch.uint8: + raise TypeError(f"{name} must use torch.uint8") + normalized = value.detach().to("cpu").reshape(-1).contiguous() + if normalized.numel() == 0: + raise ValueError(f"{name} must be non-empty") + if expected_rows is not None and normalized.numel() != expected_rows: + raise ValueError(f"{name} must contain exactly {expected_rows} codes") + if (normalized > 2).any().item(): + raise ValueError(f"{name} may contain only Q4/Q6/Q8 codes 0, 1, and 2") + return normalized.clone() + + +def q8_set_jaccard(left_codes: torch.Tensor, right_codes: torch.Tensor) -> float: + """Jaccard similarity of code-2 sets; two empty sets are defined as 1.""" + + left = _precision_codes(left_codes, name="left_codes") + right = _precision_codes(right_codes, name="right_codes", expected_rows=left.numel()) + left_q8 = left == 2 + right_q8 = right == 2 + union = int(torch.logical_or(left_q8, right_q8).sum().item()) + if union == 0: + return 1.0 + intersection = int(torch.logical_and(left_q8, right_q8).sum().item()) + return intersection / union + + +def per_layer_mean_bitwidth_shifts( + left_codes: torch.Tensor, + right_codes: torch.Tensor, + *, + layer_indices: tuple[int, ...], + rows_per_layer: int, +) -> tuple[tuple[int, float], ...]: + """Return absolute shifts between layer mean bitwidths in actual bits. + + A precision code is a two-bit step above Q4, so the mean-bitwidth shift is + ``2 * abs(mean(code_A) - mean(code_B))``. It is not the mean per-row + churn; Q8-set Jaccard separately measures set overlap. + """ + + if not isinstance(layer_indices, tuple) or not layer_indices: + raise ValueError("layer_indices must be a non-empty tuple") + if any( + isinstance(index, bool) or not isinstance(index, int) or index < 0 + for index in layer_indices + ) or len(set(layer_indices)) != len(layer_indices): + raise ValueError("layer_indices must contain unique non-negative integers") + layer_rows = _strict_positive_int(rows_per_layer, name="rows_per_layer") + total_rows = len(layer_indices) * layer_rows + left = _precision_codes(left_codes, name="left_codes", expected_rows=total_rows) + right = _precision_codes(right_codes, name="right_codes", expected_rows=total_rows) + left_layers = left.to(torch.float64).reshape(len(layer_indices), layer_rows) + right_layers = right.to(torch.float64).reshape(len(layer_indices), layer_rows) + shifts = 2.0 * (left_layers.mean(dim=1) - right_layers.mean(dim=1)).abs() + return tuple( + (layer_index, float(shift.item())) + for layer_index, shift in zip(layer_indices, shifts, strict=True) + ) + + +@dataclass(frozen=True, slots=True) +class PolicyStabilityResult: + """Conjunctive split-half stability result for one exact-K policy.""" + + passed: bool + spearman_average_ties: float | None + q8_jaccard: float + layer_mean_bitwidth_shifts: tuple[tuple[int, float], ...] + checks: tuple[tuple[str, bool], ...] + + @property + def max_layer_mean_bitwidth_shift(self) -> float: + return max(shift for _layer, shift in self.layer_mean_bitwidth_shifts) + + +def evaluate_policy_stability( + half_a_codes: torch.Tensor, + half_b_codes: torch.Tensor, + *, + layer_indices: tuple[int, ...], + rows_per_layer: int, + expected_marginal_steps: int | None = None, +) -> PolicyStabilityResult: + """Evaluate the frozen Spearman, Q8-Jaccard, and layer-shift conjunction. + + Spearman uses average (mid-)ranks for exact ties. A constant code vector + has undefined correlation and therefore fails closed instead of receiving + an arbitrary correlation. + """ + + rows = len(layer_indices) * _strict_positive_int(rows_per_layer, name="rows_per_layer") + left = _precision_codes(half_a_codes, name="half_a_codes", expected_rows=rows) + right = _precision_codes(half_b_codes, name="half_b_codes", expected_rows=rows) + if expected_marginal_steps is not None: + steps = _strict_nonnegative_int( + expected_marginal_steps, + name="expected_marginal_steps", + ) + if steps > 2 * rows: + raise ValueError("expected_marginal_steps exceeds two steps per row") + if int(left.to(torch.int64).sum().item()) != steps: + raise ValueError("half A code map does not satisfy the expected exact-K budget") + if int(right.to(torch.int64).sum().item()) != steps: + raise ValueError("half B code map does not satisfy the expected exact-K budget") + + try: + spearman = spearman_correlation( + left.to(torch.float64).tolist(), + right.to(torch.float64).tolist(), + ) + except ValueError as exc: + if "constant input" not in str(exc): + raise + spearman = None + jaccard = q8_set_jaccard(left, right) + layer_shifts = per_layer_mean_bitwidth_shifts( + left, + right, + layer_indices=layer_indices, + rows_per_layer=rows_per_layer, + ) + max_shift = max(shift for _layer, shift in layer_shifts) + checks = ( + ( + "spearman_at_least_0_70", + spearman is not None and spearman >= MIN_SPLIT_HALF_SPEARMAN, + ), + ("q8_jaccard_at_least_0_50", jaccard >= MIN_SPLIT_HALF_Q8_JACCARD), + ( + "every_layer_mean_bitwidth_shift_at_most_0_25", + max_shift <= MAX_LAYER_MEAN_BITWIDTH_SHIFT, + ), + ) + return PolicyStabilityResult( + passed=all(passed for _name, passed in checks), + spearman_average_ties=spearman, + q8_jaccard=jaccard, + layer_mean_bitwidth_shifts=layer_shifts, + checks=checks, + ) + + +@dataclass(frozen=True, slots=True) +class SplitHalfPolicyFit: + split: CalibrationHalfSplit + half_a_aggregate: CalibrationAggregate + half_b_aggregate: CalibrationAggregate + half_a_codes: torch.Tensor + half_b_codes: torch.Tensor + stability: PolicyStabilityResult + + +def fit_split_half_policy( + sequences: list[CalibrationSequenceScores] | tuple[CalibrationSequenceScores, ...], + *, + layer_indices: tuple[int, ...], + rows_per_layer: int, + marginal_steps: int, +) -> SplitHalfPolicyFit: + """Split, independently aggregate, allocate exact-K maps, and gate them.""" + + split = balanced_sha_rank_halves(sequences) + half_a = aggregate_calibration_scores(split.half_a) + half_b = aggregate_calibration_scores(split.half_b) + codes_a = allocate_static_q468_code_map(half_a, marginal_steps=marginal_steps) + codes_b = allocate_static_q468_code_map(half_b, marginal_steps=marginal_steps) + stability = evaluate_policy_stability( + codes_a, + codes_b, + layer_indices=layer_indices, + rows_per_layer=rows_per_layer, + expected_marginal_steps=marginal_steps, + ) + return SplitHalfPolicyFit( + split=split, + half_a_aggregate=half_a, + half_b_aggregate=half_b, + half_a_codes=codes_a, + half_b_codes=codes_b, + stability=stability, + ) + + +def static_q468_code_map_sha256( + codes: torch.Tensor, + *, + geometry: StaticRhtQ468Geometry, + marginal_steps: int, +) -> str: + """Return the code-map hash used by ``StaticRhtQ468Policy``.""" + + if not isinstance(geometry, StaticRhtQ468Geometry): + raise TypeError("geometry must be a StaticRhtQ468Geometry") + normalized = _precision_codes(codes, name="codes", expected_rows=geometry.total_rows) + steps = _strict_nonnegative_int(marginal_steps, name="marginal_steps") + if int(normalized.to(torch.int64).sum().item()) != steps: + raise ValueError("code-map marginal sum does not match marginal_steps") + packed = _pack_precision_codes(normalized) + digest = hashlib.sha256() + digest.update(_CODE_MAP_HASH_DOMAIN) + digest.update(bytes.fromhex(geometry.geometry_sha256)) + digest.update(steps.to_bytes(8, "little", signed=False)) + digest.update(packed.numpy().tobytes(order="C")) + return digest.hexdigest() + + +def _validate_aggregate(aggregate: object, *, expected_rows: int) -> CalibrationAggregate: + if not isinstance(aggregate, CalibrationAggregate): + raise TypeError("aggregate must be a CalibrationAggregate") + values = tuple( + _cpu_fp64_scores(value, name=name, expected_rows=expected_rows) + for name, value in (("D4", aggregate.d4), ("D6", aggregate.d6), ("D8", aggregate.d8)) + ) + if not isinstance(aggregate.family_sequence_counts, tuple): + raise ValueError("family_sequence_counts must be a tuple") + expected_families = tuple(name for name, _count in aggregate.family_sequence_counts) + if expected_families != CALIBRATION_FAMILY_ORDER: + raise ValueError("family_sequence_counts must use MBPP, PG19, RULER order") + for name, count in aggregate.family_sequence_counts: + _canonical_text(name, name="family count key") + _strict_positive_int(count, name=f"{name} sequence count") + if tuple(name for name, _count in aggregate.ruler_category_sequence_counts) != ( + RULER_CATEGORY_ORDER + ): + raise ValueError("RULER category counts must use the frozen category order") + for name, count in aggregate.ruler_category_sequence_counts: + _canonical_text(name, name="RULER category") + _strict_positive_int(count, name=f"RULER {name} sequence count") + family_count_map = dict(aggregate.family_sequence_counts) + if family_count_map["ruler"] != sum( + count for _name, count in aggregate.ruler_category_sequence_counts + ): + raise ValueError("RULER broad-family count must equal its four category counts") + _sha256( + aggregate.sequence_score_manifest_sha256, + name="sequence_score_manifest_sha256", + ) + if not isinstance(aggregate.source_contract, CalibrationSourceTensorContract): + raise TypeError("source_contract must be a CalibrationSourceTensorContract") + if math.prod(aggregate.source_contract.trailing_shape) != expected_rows: + raise ValueError("source tensor contract does not match aggregate row count") + if aggregate.source_contract == FROZEN_SOURCE_TENSOR_CONTRACT: + identity_manifest_sha256 = _sha256( + aggregate.identity_record_manifest_sha256, + name="identity_record_manifest_sha256", + ) + else: + if aggregate.identity_record_manifest_sha256 is not None: + raise ValueError("generic aggregate cannot claim a frozen identity-record manifest") + identity_manifest_sha256 = None + return CalibrationAggregate( + d4=values[0], + d6=values[1], + d8=values[2], + family_sequence_counts=aggregate.family_sequence_counts, + ruler_category_sequence_counts=aggregate.ruler_category_sequence_counts, + sequence_score_manifest_sha256=aggregate.sequence_score_manifest_sha256, + source_contract=aggregate.source_contract, + identity_record_manifest_sha256=identity_manifest_sha256, + ) + + +def _budget_tuple(values: object, *, total_rows: int) -> tuple[int, ...]: + if not isinstance(values, (list, tuple)) or not values: + raise ValueError("marginal_steps must be a non-empty list or tuple") + normalized = tuple( + _strict_nonnegative_int(value, name="marginal_steps entry") for value in values + ) + if any(value > 2 * total_rows for value in normalized): + raise ValueError("a marginal_steps entry exceeds two steps per row") + if len(set(normalized)) != len(normalized): + raise ValueError("marginal_steps entries must be unique") + return tuple(sorted(normalized)) + + +def _score_data_b64(scores: tuple[torch.Tensor, torch.Tensor, torch.Tensor]) -> str: + raw = b"".join(_tensor_bytes(score) for score in scores) + return base64.b64encode(raw).decode("ascii") + + +def _allocation_record( + aggregate: CalibrationAggregate, + *, + geometry: StaticRhtQ468Geometry, + marginal_steps: int, +) -> tuple[dict[str, object], torch.Tensor]: + codes = allocate_static_q468_code_map(aggregate, marginal_steps=marginal_steps) + counts = [int((codes == code).sum().item()) for code in range(3)] + return ( + { + "allocator_revision": STATIC_Q468_ALLOCATOR_REVISION, + "code_counts_q4_q6_q8": counts, + "code_map_sha256": static_q468_code_map_sha256( + codes, + geometry=geometry, + marginal_steps=marginal_steps, + ), + "marginal_steps": marginal_steps, + "packed_precision_bytes": math.ceil(geometry.total_rows * 2 / 8), + }, + codes, + ) + + +def _build_calibration_score_artifact( + aggregate: CalibrationAggregate, + *, + geometry: StaticRhtQ468Geometry, + calibration_identity_sha256: str, + marginal_steps: tuple[int, ...] | list[int], + artifact_kind: str, + artifact_profile: str, + artifact_revision: str, +) -> bytes: + """Build canonical, hash-bound score evidence for one explicit profile.""" + + if not isinstance(geometry, StaticRhtQ468Geometry): + raise TypeError("geometry must be a StaticRhtQ468Geometry") + normalized = _validate_aggregate(aggregate, expected_rows=geometry.total_rows) + identity_sha256 = _sha256( + calibration_identity_sha256, + name="calibration_identity_sha256", + ) + budgets = _budget_tuple(marginal_steps, total_rows=geometry.total_rows) + score_sha256 = static_q468_distortion_sha256( + *normalized.scores(), + geometry=geometry, + ) + allocation_records = [ + _allocation_record(normalized, geometry=geometry, marginal_steps=steps)[0] + for steps in budgets + ] + evidence = { + "aggregation_contract": _AGGREGATION_CONTRACT, + "allocations": allocation_records, + "artifact_profile": artifact_profile, + "artifact_revision": artifact_revision, + "calibration_identity_sha256": identity_sha256, + "calibration_scores_sha256": score_sha256, + "family_sequence_counts": [ + {"count": count, "family": family} + for family, count in normalized.family_sequence_counts + ], + "geometry": geometry.canonical_dict(), + "geometry_sha256": geometry.geometry_sha256, + "ruler_category_sequence_counts": [ + {"category": category, "count": count} + for category, count in normalized.ruler_category_sequence_counts + ], + "scores": { + "axis_order": ["bitwidth", "flattened_layer_head_key_row"], + "bitwidths": [4, 6, 8], + "data_base64": _score_data_b64(normalized.scores()), + "dtype": CALIBRATION_SCORE_DTYPE, + "shape": [3, geometry.total_rows], + }, + "sequence_score_manifest_sha256": normalized.sequence_score_manifest_sha256, + "identity_record_manifest_sha256": normalized.identity_record_manifest_sha256, + "source_tensor_contract": normalized.source_contract.canonical_dict(), + } + canonical_evidence_sha256 = hashlib.sha256(canonical_json_bytes(evidence)).hexdigest() + document = { + "artifact_kind": artifact_kind, + "canonical_evidence_sha256": canonical_evidence_sha256, + "evidence": evidence, + "schema_version": CALIBRATION_SCORE_ARTIFACT_SCHEMA_VERSION, + } + return canonical_json_bytes(document) + + +def build_calibration_score_artifact( + aggregate: CalibrationAggregate, + *, + geometry: StaticRhtQ468Geometry, + calibration_identity_sha256: str, + marginal_steps: tuple[int, ...] | list[int], +) -> bytes: + """Build a generic score artifact that cannot impersonate Experiment 013 evidence.""" + + return _build_calibration_score_artifact( + aggregate, + geometry=geometry, + calibration_identity_sha256=calibration_identity_sha256, + marginal_steps=marginal_steps, + artifact_kind=GENERIC_CALIBRATION_SCORE_ARTIFACT_KIND, + artifact_profile=GENERIC_CALIBRATION_SCORE_ARTIFACT_PROFILE, + artifact_revision=GENERIC_CALIBRATION_SCORE_ARTIFACT_REVISION, + ) + + +def build_frozen_calibration_score_artifact( + aggregate: CalibrationAggregate, + *, + calibration_identity_sha256: str, +) -> bytes: + """Build the frozen real-geometry K27030/K29334 score artifact.""" + + if aggregate.family_sequence_counts != ( + ("mbpp", 128), + ("pg19", 16), + ("ruler", 16), + ): + raise ValueError("frozen calibration counts must be MBPP=128, PG19=16, RULER=16") + if tuple(count for _name, count in aggregate.ruler_category_sequence_counts) != ( + 4, + 4, + 4, + 4, + ): + raise ValueError("each frozen RULER category must contain four sequences") + if aggregate.source_contract != FROZEN_SOURCE_TENSOR_CONTRACT: + raise ValueError("frozen calibration requires the exact Experiment 013 source contract") + return _build_calibration_score_artifact( + aggregate, + geometry=FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + calibration_identity_sha256=calibration_identity_sha256, + marginal_steps=( + FROZEN_STATIC_Q468_ABLATION_STEPS, + FROZEN_STATIC_Q468_PRIMARY_STEPS, + ), + artifact_kind=CALIBRATION_SCORE_ARTIFACT_KIND, + artifact_profile=CALIBRATION_SCORE_ARTIFACT_PROFILE, + artifact_revision=CALIBRATION_SCORE_ARTIFACT_REVISION, + ) + + +@dataclass(frozen=True, slots=True) +class DecodedCalibrationScoreArtifact: + artifact_kind: str + artifact_profile: str + artifact_revision: str + aggregate: CalibrationAggregate + geometry: StaticRhtQ468Geometry + calibration_identity_sha256: str + calibration_scores_sha256: str + allocations: tuple[tuple[int, torch.Tensor, str], ...] + canonical_evidence_sha256: str + file_sha256: str + + +def _reject_json_constant(value: str) -> None: + raise CalibrationArtifactError(f"non-finite JSON constant is forbidden: {value}") + + +def _unique_json_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise CalibrationArtifactError(f"duplicate JSON object key: {key!r}") + result[key] = value + return result + + +def _mapping(value: object, *, context: str) -> dict[str, object]: + if not isinstance(value, dict) or any(not isinstance(key, str) for key in value): + raise CalibrationArtifactError(f"{context} must be a JSON object") + return value + + +def _sequence(value: object, *, context: str) -> list[object]: + if not isinstance(value, list): + raise CalibrationArtifactError(f"{context} must be a JSON array") + return value + + +def _exact_keys(value: dict[str, object], expected: set[str], *, context: str) -> None: + if set(value) != expected: + missing = sorted(expected - set(value)) + extra = sorted(set(value) - expected) + raise CalibrationArtifactError( + f"{context} fields drifted; missing={missing}, extra={extra}" + ) + + +def _artifact_int(value: object, *, context: str, minimum: int = 0) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise CalibrationArtifactError(f"{context} must be an integer >= {minimum}") + return value + + +def _artifact_sha(value: object, *, context: str) -> str: + try: + return _sha256(value, name=context) + except (TypeError, ValueError) as exc: + raise CalibrationArtifactError(str(exc)) from exc + + +def _parse_counts( + value: object, + *, + context: str, + key_name: str, + expected_names: tuple[str, ...] | None = None, +) -> tuple[tuple[str, int], ...]: + rows = _sequence(value, context=context) + result: list[tuple[str, int]] = [] + for index, raw in enumerate(rows): + row = _mapping(raw, context=f"{context}[{index}]") + _exact_keys(row, {key_name, "count"}, context=f"{context}[{index}]") + try: + name = _canonical_text(row[key_name], name=f"{context}[{index}].{key_name}") + except (TypeError, ValueError) as exc: + raise CalibrationArtifactError(str(exc)) from exc + count = _artifact_int(row["count"], context=f"{context}[{index}].count", minimum=1) + result.append((name, count)) + names = tuple(name for name, _count in result) + if len(set(names)) != len(names): + raise CalibrationArtifactError(f"{context} contains duplicate names") + if expected_names is not None and names != expected_names: + raise CalibrationArtifactError(f"{context} differs from the frozen family order") + if expected_names is None and tuple(sorted(result)) != tuple(result): + raise CalibrationArtifactError(f"{context} must be sorted canonically") + return tuple(result) + + +def _parse_geometry(value: object) -> StaticRhtQ468Geometry: + geometry = _mapping(value, context="evidence.geometry") + _exact_keys( + geometry, + {"heads", "key_rows", "layer_indices", "target_resident_bytes", "value_width"}, + context="evidence.geometry", + ) + raw_layers = _sequence(geometry["layer_indices"], context="evidence.geometry.layer_indices") + layers = tuple( + _artifact_int(item, context=f"evidence.geometry.layer_indices[{index}]") + for index, item in enumerate(raw_layers) + ) + try: + return StaticRhtQ468Geometry( + layer_indices=layers, + heads=_artifact_int(geometry["heads"], context="evidence.geometry.heads", minimum=1), + key_rows=_artifact_int( + geometry["key_rows"], + context="evidence.geometry.key_rows", + minimum=1, + ), + value_width=_artifact_int( + geometry["value_width"], + context="evidence.geometry.value_width", + minimum=1, + ), + target_resident_bytes=_artifact_int( + geometry["target_resident_bytes"], + context="evidence.geometry.target_resident_bytes", + minimum=1, + ), + ) + except (TypeError, ValueError) as exc: + raise CalibrationArtifactError(f"invalid geometry: {exc}") from exc + + +def _parse_source_tensor_contract(value: object) -> CalibrationSourceTensorContract: + contract = _mapping(value, context="evidence.source_tensor_contract") + _exact_keys( + contract, + {"axis_order", "dtypes", "reduction_profile", "trailing_shape"}, + context="evidence.source_tensor_contract", + ) + raw_axes = _sequence( + contract["axis_order"], + context="evidence.source_tensor_contract.axis_order", + ) + raw_shape = _sequence( + contract["trailing_shape"], + context="evidence.source_tensor_contract.trailing_shape", + ) + raw_dtypes = _mapping( + contract["dtypes"], + context="evidence.source_tensor_contract.dtypes", + ) + if set(raw_dtypes) != set(_SOURCE_TENSOR_NAMES): + raise CalibrationArtifactError( + "source tensor dtypes must cover the exact source tensor names" + ) + try: + return CalibrationSourceTensorContract( + reduction_profile=_canonical_text( + contract["reduction_profile"], + name="source reduction_profile", + ), + axis_order=tuple( + _canonical_text(axis, name=f"source axis_order[{index}]") + for index, axis in enumerate(raw_axes) + ), + trailing_shape=tuple( + _artifact_int( + extent, + context=f"evidence.source_tensor_contract.trailing_shape[{index}]", + minimum=1, + ) + for index, extent in enumerate(raw_shape) + ), + dtypes=tuple( + ( + name, + _canonical_text(raw_dtypes[name], name=f"source dtype {name}"), + ) + for name in _SOURCE_TENSOR_NAMES + ), + ) + except (TypeError, ValueError) as exc: + if isinstance(exc, CalibrationArtifactError): + raise + raise CalibrationArtifactError(f"invalid source tensor contract: {exc}") from exc + + +def deserialize_calibration_score_artifact( + data: bytes, + *, + expected_file_sha256: str | None = None, +) -> DecodedCalibrationScoreArtifact: + """Strictly decode and recompute a canonical score artifact.""" + + if not isinstance(data, bytes): + raise TypeError("data must be bytes") + file_sha256 = hashlib.sha256(data).hexdigest() + if expected_file_sha256 is not None: + expected = _artifact_sha(expected_file_sha256, context="expected_file_sha256") + if file_sha256 != expected: + raise CalibrationArtifactError( + f"file SHA-256 mismatch: expected {expected}, computed {file_sha256}" + ) + try: + document = json.loads( + data.decode("utf-8"), + parse_constant=_reject_json_constant, + object_pairs_hook=_unique_json_object, + ) + except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc: + if isinstance(exc, CalibrationArtifactError): + raise + raise CalibrationArtifactError(f"artifact is not strict UTF-8 JSON: {exc}") from exc + root = _mapping(document, context="artifact") + _exact_keys( + root, + {"artifact_kind", "canonical_evidence_sha256", "evidence", "schema_version"}, + context="artifact", + ) + if root["artifact_kind"] not in { + CALIBRATION_SCORE_ARTIFACT_KIND, + GENERIC_CALIBRATION_SCORE_ARTIFACT_KIND, + }: + raise CalibrationArtifactError("artifact kind is not a supported calibration profile") + if ( + _artifact_int(root["schema_version"], context="artifact.schema_version", minimum=1) + != CALIBRATION_SCORE_ARTIFACT_SCHEMA_VERSION + ): + raise CalibrationArtifactError("artifact schema version differs from the frozen value") + recorded_canonical = _artifact_sha( + root["canonical_evidence_sha256"], + context="artifact.canonical_evidence_sha256", + ) + evidence = _mapping(root["evidence"], context="artifact.evidence") + _exact_keys( + evidence, + { + "aggregation_contract", + "allocations", + "artifact_profile", + "artifact_revision", + "calibration_identity_sha256", + "calibration_scores_sha256", + "family_sequence_counts", + "geometry", + "geometry_sha256", + "identity_record_manifest_sha256", + "ruler_category_sequence_counts", + "scores", + "sequence_score_manifest_sha256", + "source_tensor_contract", + }, + context="artifact.evidence", + ) + computed_canonical = hashlib.sha256(canonical_json_bytes(evidence)).hexdigest() + if recorded_canonical != computed_canonical: + raise CalibrationArtifactError("canonical evidence SHA-256 mismatch") + if canonical_json_bytes(root) != data: + raise CalibrationArtifactError("artifact bytes are not canonical JSON") + profile_triplet = ( + root["artifact_kind"], + evidence["artifact_profile"], + evidence["artifact_revision"], + ) + official_profile = ( + CALIBRATION_SCORE_ARTIFACT_KIND, + CALIBRATION_SCORE_ARTIFACT_PROFILE, + CALIBRATION_SCORE_ARTIFACT_REVISION, + ) + generic_profile = ( + GENERIC_CALIBRATION_SCORE_ARTIFACT_KIND, + GENERIC_CALIBRATION_SCORE_ARTIFACT_PROFILE, + GENERIC_CALIBRATION_SCORE_ARTIFACT_REVISION, + ) + if profile_triplet != official_profile and profile_triplet != generic_profile: + raise CalibrationArtifactError("artifact kind, profile, and revision are inconsistent") + is_official = profile_triplet == official_profile + if evidence["aggregation_contract"] != _AGGREGATION_CONTRACT: + raise CalibrationArtifactError("aggregation contract differs from the frozen equation") + + identity_sha256 = _artifact_sha( + evidence["calibration_identity_sha256"], + context="evidence.calibration_identity_sha256", + ) + sequence_manifest_sha256 = _artifact_sha( + evidence["sequence_score_manifest_sha256"], + context="evidence.sequence_score_manifest_sha256", + ) + raw_identity_manifest_sha256 = evidence["identity_record_manifest_sha256"] + if is_official: + identity_record_manifest_sha256 = _artifact_sha( + raw_identity_manifest_sha256, + context="evidence.identity_record_manifest_sha256", + ) + else: + if raw_identity_manifest_sha256 is not None: + raise CalibrationArtifactError( + "generic score artifact cannot claim a frozen identity-record manifest" + ) + identity_record_manifest_sha256 = None + family_counts = _parse_counts( + evidence["family_sequence_counts"], + context="evidence.family_sequence_counts", + key_name="family", + expected_names=CALIBRATION_FAMILY_ORDER, + ) + ruler_counts = _parse_counts( + evidence["ruler_category_sequence_counts"], + context="evidence.ruler_category_sequence_counts", + key_name="category", + expected_names=RULER_CATEGORY_ORDER, + ) + + geometry = _parse_geometry(evidence["geometry"]) + geometry_sha256 = _artifact_sha( + evidence["geometry_sha256"], + context="evidence.geometry_sha256", + ) + if geometry.geometry_sha256 != geometry_sha256: + raise CalibrationArtifactError("geometry SHA-256 mismatch") + source_contract = _parse_source_tensor_contract(evidence["source_tensor_contract"]) + + if is_official: + if geometry != FROZEN_QWEN35_STATIC_Q468_GEOMETRY: + raise CalibrationArtifactError( + "official Experiment 013 artifact requires the exact frozen geometry" + ) + if family_counts != (("mbpp", 128), ("pg19", 16), ("ruler", 16)): + raise CalibrationArtifactError( + "official Experiment 013 artifact requires MBPP=128, PG19=16, RULER=16" + ) + if ruler_counts != tuple((category, 4) for category in RULER_CATEGORY_ORDER): + raise CalibrationArtifactError( + "official Experiment 013 artifact requires four sequences per RULER category" + ) + if source_contract != FROZEN_SOURCE_TENSOR_CONTRACT: + raise CalibrationArtifactError( + "official Experiment 013 artifact requires the frozen source tensor contract" + ) + + allocation_values = _sequence(evidence["allocations"], context="evidence.allocations") + if is_official: + official_budgets: list[int] = [] + for index, raw_allocation in enumerate(allocation_values): + allocation = _mapping(raw_allocation, context=f"evidence.allocations[{index}]") + if "marginal_steps" not in allocation: + raise CalibrationArtifactError( + f"evidence.allocations[{index}] is missing marginal_steps" + ) + official_budgets.append( + _artifact_int( + allocation["marginal_steps"], + context=f"evidence.allocations[{index}].marginal_steps", + ) + ) + if tuple(official_budgets) != ( + FROZEN_STATIC_Q468_ABLATION_STEPS, + FROZEN_STATIC_Q468_PRIMARY_STEPS, + ): + raise CalibrationArtifactError( + "official Experiment 013 artifact requires exactly K27030 and K29334" + ) + + score_record = _mapping(evidence["scores"], context="evidence.scores") + _exact_keys( + score_record, + {"axis_order", "bitwidths", "data_base64", "dtype", "shape"}, + context="evidence.scores", + ) + if score_record["dtype"] != CALIBRATION_SCORE_DTYPE: + raise CalibrationArtifactError("score dtype must be float64-le") + if score_record["axis_order"] != ["bitwidth", "flattened_layer_head_key_row"]: + raise CalibrationArtifactError("score axis order differs from the frozen layout") + if score_record["bitwidths"] != [4, 6, 8]: + raise CalibrationArtifactError("score bitwidth order must be Q4, Q6, Q8") + if score_record["shape"] != [3, geometry.total_rows]: + raise CalibrationArtifactError("score shape does not match geometry") + encoded = score_record["data_base64"] + if not isinstance(encoded, str): + raise CalibrationArtifactError("score data_base64 must be a string") + try: + raw_scores = base64.b64decode(encoded, validate=True) + except (ValueError, binascii.Error) as exc: + raise CalibrationArtifactError("score data_base64 is invalid") from exc + if base64.b64encode(raw_scores).decode("ascii") != encoded: + raise CalibrationArtifactError("score data_base64 is not canonical") + expected_bytes = 3 * geometry.total_rows * 8 + if len(raw_scores) != expected_bytes: + raise CalibrationArtifactError( + f"score byte length differs: expected {expected_bytes}, got {len(raw_scores)}" + ) + array = np.frombuffer(raw_scores, dtype=" dict[str, Any]: + """Return a JSON-compatible fail-closed verification report.""" + + file_sha256 = hashlib.sha256(data).hexdigest() if isinstance(data, bytes) else None + try: + artifact = deserialize_calibration_score_artifact( + data, + expected_file_sha256=expected_file_sha256, + ) + except (TypeError, ValueError) as exc: + return { + "allocations": [], + "calibration_scores_sha256": None, + "canonical_evidence_sha256": None, + "errors": [str(exc)], + "file_sha256": file_sha256, + "valid": False, + } + return { + "allocations": [ + {"code_map_sha256": digest, "marginal_steps": steps} + for steps, _codes, digest in artifact.allocations + ], + "calibration_scores_sha256": artifact.calibration_scores_sha256, + "canonical_evidence_sha256": artifact.canonical_evidence_sha256, + "errors": [], + "file_sha256": artifact.file_sha256, + "valid": True, + } + + +def _stability_threshold_record() -> dict[str, str]: + return { + "maximum_layer_mean_bitwidth_shift": MAX_LAYER_MEAN_BITWIDTH_SHIFT.hex(), + "minimum_q8_jaccard": MIN_SPLIT_HALF_Q8_JACCARD.hex(), + "minimum_spearman_average_ties": MIN_SPLIT_HALF_SPEARMAN.hex(), + } + + +def _stability_metric_record(result: PolicyStabilityResult) -> dict[str, object]: + if result.spearman_average_ties is None: + spearman: str | None = None + else: + spearman = result.spearman_average_ties.hex() + return { + "checks": [{"name": name, "passed": passed} for name, passed in result.checks], + "layer_mean_bitwidth_shifts": [ + {"layer_index": layer, "shift": shift.hex()} + for layer, shift in result.layer_mean_bitwidth_shifts + ], + "maximum_layer_mean_bitwidth_shift": (result.max_layer_mean_bitwidth_shift.hex()), + "passed": result.passed, + "q8_jaccard": result.q8_jaccard.hex(), + "spearman_average_ties": spearman, + } + + +def _split_half_evidence_record( + aggregate: CalibrationAggregate, + *, + half: SplitHalf, +) -> tuple[dict[str, object], torch.Tensor]: + geometry = FROZEN_QWEN35_STATIC_Q468_GEOMETRY + normalized = _validate_aggregate(aggregate, expected_rows=geometry.total_rows) + if normalized.family_sequence_counts != ( + ("mbpp", 64), + ("pg19", 8), + ("ruler", 8), + ): + raise ValueError("each frozen split half must contain MBPP=64, PG19=8, RULER=8") + if normalized.ruler_category_sequence_counts != tuple( + (category, 2) for category in RULER_CATEGORY_ORDER + ): + raise ValueError("each frozen split half must contain two sequences per RULER category") + if normalized.source_contract != FROZEN_SOURCE_TENSOR_CONTRACT: + raise ValueError("split-half evidence requires the frozen source tensor contract") + scores_sha256 = static_q468_distortion_sha256( + *normalized.scores(), + geometry=geometry, + ) + codes = allocate_static_q468_code_map( + normalized, + marginal_steps=FROZEN_STATIC_Q468_PRIMARY_STEPS, + ) + return ( + { + "calibration_scores_sha256": scores_sha256, + "code_map": { + "code_map_sha256": static_q468_code_map_sha256( + codes, + geometry=geometry, + marginal_steps=FROZEN_STATIC_Q468_PRIMARY_STEPS, + ), + "codes_base64": base64.b64encode(codes.numpy().tobytes()).decode("ascii"), + "dtype": "uint8", + "shape": [geometry.total_rows], + }, + "family_sequence_counts": [ + {"count": count, "family": family} + for family, count in normalized.family_sequence_counts + ], + "half": half, + "identity_record_manifest_sha256": (normalized.identity_record_manifest_sha256), + "ruler_category_sequence_counts": [ + {"category": category, "count": count} + for category, count in normalized.ruler_category_sequence_counts + ], + "scores": { + "axis_order": ["bitwidth", "flattened_layer_head_key_row"], + "bitwidths": [4, 6, 8], + "data_base64": _score_data_b64(normalized.scores()), + "dtype": CALIBRATION_SCORE_DTYPE, + "shape": [3, geometry.total_rows], + }, + "sequence_score_manifest_sha256": (normalized.sequence_score_manifest_sha256), + "source_tensor_contract": normalized.source_contract.canonical_dict(), + }, + codes, + ) + + +def build_frozen_split_half_stability_artifact( + half_a_aggregate: CalibrationAggregate, + half_b_aggregate: CalibrationAggregate, + *, + identity_file_sha256: str, + canonical_identity_sha256: str, + resolver_assignment_sha256: str, + full_sequence_score_manifest_sha256: str, + full_calibration_scores_sha256: str, +) -> bytes: + """Build the strict passing K29334 split-half stability artifact.""" + + half_a, codes_a = _split_half_evidence_record(half_a_aggregate, half="a") + half_b, codes_b = _split_half_evidence_record(half_b_aggregate, half="b") + geometry = FROZEN_QWEN35_STATIC_Q468_GEOMETRY + stability = evaluate_policy_stability( + codes_a, + codes_b, + layer_indices=geometry.layer_indices, + rows_per_layer=geometry.rows_per_layer, + expected_marginal_steps=FROZEN_STATIC_Q468_PRIMARY_STEPS, + ) + if not stability.passed: + raise ValueError("frozen K29334 split-half stability gate did not pass") + evidence = { + "artifact_profile": SPLIT_HALF_STABILITY_ARTIFACT_PROFILE, + "artifact_revision": SPLIT_HALF_STABILITY_ARTIFACT_REVISION, + "full_calibration": { + "calibration_scores_sha256": _sha256( + full_calibration_scores_sha256, + name="full_calibration_scores_sha256", + ), + "sequence_score_manifest_sha256": _sha256( + full_sequence_score_manifest_sha256, + name="full_sequence_score_manifest_sha256", + ), + }, + "geometry": geometry.canonical_dict(), + "geometry_sha256": geometry.geometry_sha256, + "halves": [half_a, half_b], + "identity": { + "canonical_identity_sha256": _sha256( + canonical_identity_sha256, + name="canonical_identity_sha256", + ), + "identity_file_sha256": _sha256( + identity_file_sha256, + name="identity_file_sha256", + ), + "resolver_assignment_sha256": _sha256( + resolver_assignment_sha256, + name="resolver_assignment_sha256", + ), + }, + "marginal_steps": FROZEN_STATIC_Q468_PRIMARY_STEPS, + "metrics": _stability_metric_record(stability), + "thresholds": _stability_threshold_record(), + } + document = { + "artifact_kind": SPLIT_HALF_STABILITY_ARTIFACT_KIND, + "canonical_evidence_sha256": hashlib.sha256(canonical_json_bytes(evidence)).hexdigest(), + "evidence": evidence, + "schema_version": SPLIT_HALF_STABILITY_ARTIFACT_SCHEMA_VERSION, + } + return canonical_json_bytes(document) + + +@dataclass(frozen=True, slots=True) +class DecodedSplitHalfStabilityArtifact: + identity_file_sha256: str + canonical_identity_sha256: str + resolver_assignment_sha256: str + full_sequence_score_manifest_sha256: str + full_calibration_scores_sha256: str + half_a_aggregate: CalibrationAggregate + half_b_aggregate: CalibrationAggregate + half_a_codes: torch.Tensor + half_b_codes: torch.Tensor + stability: PolicyStabilityResult + canonical_evidence_sha256: str + file_sha256: str + + +def _decode_frozen_split_half( + value: object, + *, + expected_half: SplitHalf, +) -> tuple[CalibrationAggregate, torch.Tensor]: + geometry = FROZEN_QWEN35_STATIC_Q468_GEOMETRY + half = _mapping(value, context=f"evidence.halves[{expected_half}]") + _exact_keys( + half, + { + "calibration_scores_sha256", + "code_map", + "family_sequence_counts", + "half", + "identity_record_manifest_sha256", + "ruler_category_sequence_counts", + "scores", + "sequence_score_manifest_sha256", + "source_tensor_contract", + }, + context=f"evidence.halves[{expected_half}]", + ) + if half["half"] != expected_half: + raise CalibrationArtifactError("split-half labels must be canonical A then B") + family_counts = _parse_counts( + half["family_sequence_counts"], + context=f"evidence.halves[{expected_half}].family_sequence_counts", + key_name="family", + expected_names=CALIBRATION_FAMILY_ORDER, + ) + if family_counts != (("mbpp", 64), ("pg19", 8), ("ruler", 8)): + raise CalibrationArtifactError("each split half must contain MBPP=64, PG19=8, RULER=8") + ruler_counts = _parse_counts( + half["ruler_category_sequence_counts"], + context=f"evidence.halves[{expected_half}].ruler_category_sequence_counts", + key_name="category", + expected_names=RULER_CATEGORY_ORDER, + ) + if ruler_counts != tuple((category, 2) for category in RULER_CATEGORY_ORDER): + raise CalibrationArtifactError( + "each split half must contain two sequences per RULER category" + ) + source_contract = _parse_source_tensor_contract(half["source_tensor_contract"]) + if source_contract != FROZEN_SOURCE_TENSOR_CONTRACT: + raise CalibrationArtifactError("split-half source tensor contract drifted") + sequence_manifest_sha256 = _artifact_sha( + half["sequence_score_manifest_sha256"], + context=f"evidence.halves[{expected_half}].sequence_score_manifest_sha256", + ) + identity_manifest_sha256 = _artifact_sha( + half["identity_record_manifest_sha256"], + context=f"evidence.halves[{expected_half}].identity_record_manifest_sha256", + ) + score_record = _mapping( + half["scores"], + context=f"evidence.halves[{expected_half}].scores", + ) + _exact_keys( + score_record, + {"axis_order", "bitwidths", "data_base64", "dtype", "shape"}, + context=f"evidence.halves[{expected_half}].scores", + ) + if ( + score_record["axis_order"] != ["bitwidth", "flattened_layer_head_key_row"] + or score_record["bitwidths"] != [4, 6, 8] + or score_record["dtype"] != CALIBRATION_SCORE_DTYPE + or score_record["shape"] != [3, geometry.total_rows] + ): + raise CalibrationArtifactError("split-half score layout drifted") + encoded_scores = score_record["data_base64"] + if not isinstance(encoded_scores, str): + raise CalibrationArtifactError("split-half score data must be base64 text") + try: + raw_scores = base64.b64decode(encoded_scores, validate=True) + except (ValueError, binascii.Error) as exc: + raise CalibrationArtifactError("split-half score data_base64 is invalid") from exc + if base64.b64encode(raw_scores).decode("ascii") != encoded_scores: + raise CalibrationArtifactError("split-half score data_base64 is not canonical") + if len(raw_scores) != 3 * geometry.total_rows * 8: + raise CalibrationArtifactError("split-half score byte length drifted") + score_array = np.frombuffer(raw_scores, dtype=" DecodedSplitHalfStabilityArtifact: + """Strictly decode and recompute the frozen passing K29334 split artifact.""" + + if not isinstance(data, bytes): + raise TypeError("split-half stability artifact must be bytes") + file_sha256 = hashlib.sha256(data).hexdigest() + if expected_file_sha256 is not None and file_sha256 != _artifact_sha( + expected_file_sha256, + context="expected_file_sha256", + ): + raise CalibrationArtifactError("split-half artifact file SHA-256 mismatch") + try: + document = json.loads( + data.decode("utf-8"), + parse_constant=_reject_json_constant, + object_pairs_hook=_unique_json_object, + ) + except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc: + if isinstance(exc, CalibrationArtifactError): + raise + raise CalibrationArtifactError( + f"split-half artifact is not strict UTF-8 JSON: {exc}" + ) from exc + root = _mapping(document, context="split-half artifact") + _exact_keys( + root, + {"artifact_kind", "canonical_evidence_sha256", "evidence", "schema_version"}, + context="split-half artifact", + ) + if root["artifact_kind"] != SPLIT_HALF_STABILITY_ARTIFACT_KIND: + raise CalibrationArtifactError("split-half artifact kind drifted") + if root["schema_version"] != SPLIT_HALF_STABILITY_ARTIFACT_SCHEMA_VERSION: + raise CalibrationArtifactError("split-half schema version drifted") + evidence = _mapping(root["evidence"], context="split-half evidence") + _exact_keys( + evidence, + { + "artifact_profile", + "artifact_revision", + "full_calibration", + "geometry", + "geometry_sha256", + "halves", + "identity", + "marginal_steps", + "metrics", + "thresholds", + }, + context="split-half evidence", + ) + if ( + evidence["artifact_profile"] != SPLIT_HALF_STABILITY_ARTIFACT_PROFILE + or evidence["artifact_revision"] != SPLIT_HALF_STABILITY_ARTIFACT_REVISION + ): + raise CalibrationArtifactError("split-half artifact profile or revision drifted") + recorded_canonical = _artifact_sha( + root["canonical_evidence_sha256"], + context="split-half canonical evidence SHA-256", + ) + computed_canonical = hashlib.sha256(canonical_json_bytes(evidence)).hexdigest() + if recorded_canonical != computed_canonical: + raise CalibrationArtifactError("split-half canonical evidence SHA-256 mismatch") + if canonical_json_bytes(root) != data: + raise CalibrationArtifactError("split-half artifact bytes are not canonical JSON") + geometry = _parse_geometry(evidence["geometry"]) + if geometry != FROZEN_QWEN35_STATIC_Q468_GEOMETRY: + raise CalibrationArtifactError("split-half geometry differs from the frozen geometry") + if evidence["geometry_sha256"] != geometry.geometry_sha256: + raise CalibrationArtifactError("split-half geometry SHA-256 drifted") + if evidence["marginal_steps"] != FROZEN_STATIC_Q468_PRIMARY_STEPS: + raise CalibrationArtifactError("split-half policy must be exact K29334") + if evidence["thresholds"] != _stability_threshold_record(): + raise CalibrationArtifactError("split-half stability thresholds drifted") + + identity = _mapping(evidence["identity"], context="split-half identity") + _exact_keys( + identity, + { + "canonical_identity_sha256", + "identity_file_sha256", + "resolver_assignment_sha256", + }, + context="split-half identity", + ) + identity_file_sha256 = _artifact_sha( + identity["identity_file_sha256"], context="split identity file SHA-256" + ) + canonical_identity_sha256 = _artifact_sha( + identity["canonical_identity_sha256"], + context="split canonical identity SHA-256", + ) + resolver_assignment_sha256 = _artifact_sha( + identity["resolver_assignment_sha256"], + context="split resolver assignment SHA-256", + ) + expected_identity_values = ( + ( + expected_identity_file_sha256, + identity_file_sha256, + "split identity file SHA-256", + ), + ( + expected_canonical_identity_sha256, + canonical_identity_sha256, + "split canonical identity SHA-256", + ), + ( + expected_resolver_assignment_sha256, + resolver_assignment_sha256, + "split resolver assignment SHA-256", + ), + ) + for expected_value, recorded_value, context in expected_identity_values: + if expected_value is not None and recorded_value != _artifact_sha( + expected_value, + context=f"expected {context}", + ): + raise CalibrationArtifactError(f"{context} differs from expected identity") + full_calibration = _mapping(evidence["full_calibration"], context="split full calibration") + _exact_keys( + full_calibration, + {"calibration_scores_sha256", "sequence_score_manifest_sha256"}, + context="split full calibration", + ) + full_scores_sha256 = _artifact_sha( + full_calibration["calibration_scores_sha256"], + context="split full calibration scores SHA-256", + ) + full_manifest_sha256 = _artifact_sha( + full_calibration["sequence_score_manifest_sha256"], + context="split full sequence manifest SHA-256", + ) + halves = _sequence(evidence["halves"], context="split halves") + if len(halves) != 2: + raise CalibrationArtifactError("split artifact must contain exactly halves A and B") + aggregate_a, codes_a = _decode_frozen_split_half(halves[0], expected_half="a") + aggregate_b, codes_b = _decode_frozen_split_half(halves[1], expected_half="b") + stability = evaluate_policy_stability( + codes_a, + codes_b, + layer_indices=geometry.layer_indices, + rows_per_layer=geometry.rows_per_layer, + expected_marginal_steps=FROZEN_STATIC_Q468_PRIMARY_STEPS, + ) + if not stability.passed: + raise CalibrationArtifactError("recomputed split-half stability gate did not pass") + if evidence["metrics"] != _stability_metric_record(stability): + raise CalibrationArtifactError("split-half metrics or checks drifted") + return DecodedSplitHalfStabilityArtifact( + identity_file_sha256=identity_file_sha256, + canonical_identity_sha256=canonical_identity_sha256, + resolver_assignment_sha256=resolver_assignment_sha256, + full_sequence_score_manifest_sha256=full_manifest_sha256, + full_calibration_scores_sha256=full_scores_sha256, + half_a_aggregate=aggregate_a, + half_b_aggregate=aggregate_b, + half_a_codes=codes_a, + half_b_codes=codes_b, + stability=stability, + canonical_evidence_sha256=computed_canonical, + file_sha256=file_sha256, + ) + + +def verify_frozen_split_half_stability_artifact( + data: bytes, + *, + expected_file_sha256: str | None = None, + expected_identity_file_sha256: str | None = None, + expected_canonical_identity_sha256: str | None = None, + expected_resolver_assignment_sha256: str | None = None, +) -> dict[str, Any]: + """Return a fail-closed report for frozen split-half stability evidence.""" + + file_sha256 = hashlib.sha256(data).hexdigest() if isinstance(data, bytes) else None + try: + decoded = deserialize_frozen_split_half_stability_artifact( + data, + expected_file_sha256=expected_file_sha256, + expected_identity_file_sha256=expected_identity_file_sha256, + expected_canonical_identity_sha256=expected_canonical_identity_sha256, + expected_resolver_assignment_sha256=expected_resolver_assignment_sha256, + ) + except (TypeError, ValueError) as exc: + return { + "errors": [str(exc)], + "file_sha256": file_sha256, + "passed": False, + "valid": False, + } + return { + "errors": [], + "file_sha256": decoded.file_sha256, + "passed": decoded.stability.passed, + "valid": True, + } diff --git a/tests/test_static_q468.py b/tests/test_static_q468.py index 266625b..009a968 100644 --- a/tests/test_static_q468.py +++ b/tests/test_static_q468.py @@ -414,6 +414,9 @@ def test_q48_policy_rejects_unsupported_runtime_contracts( with pytest.raises(ValueError, match="frozen geometry"): replace(_tiny_q48_policy(), method_id=STATIC_Q48_COMPARATOR_METHOD) + with pytest.raises(ValueError, match="Q468 method cannot identify a Q48 policy"): + replace(_tiny_q48_policy(), method_id=STATIC_Q468_PRIMARY_METHOD) + @pytest.mark.parametrize( ("field", "value", "message"), @@ -456,6 +459,9 @@ def test_q468_policy_rejects_unsupported_runtime_contracts( with pytest.raises(ValueError, match="frozen geometry"): replace(_tiny_policy(), method_id=STATIC_Q468_PRIMARY_METHOD) + with pytest.raises(ValueError, match="Q48 method cannot identify a Q468 policy"): + replace(_tiny_policy(), method_id=STATIC_Q48_COMPARATOR_METHOD) + def test_pool_offsets_are_canonical_prefix_indices_within_each_pool() -> None: policy = _tiny_policy() diff --git a/tests/test_static_q468_cache.py b/tests/test_static_q468_cache.py new file mode 100644 index 0000000..f8f0f05 --- /dev/null +++ b/tests/test_static_q468_cache.py @@ -0,0 +1,595 @@ +from __future__ import annotations + +import copy +from collections.abc import Iterator + +import pytest +import torch +from transformers import Qwen3_5ForCausalLM + +from recurquant.statelease_equal_byte_baselines import ( + FROZEN_QWEN35_EQUAL_BYTE_LAYOUT, + RHT_Q4_Q6_Q8, + EqualByteLayout, +) +from recurquant.statelease_equal_byte_cache import ( + EqualByteLinearAttentionLayer, + Qwen35EqualByteObserver, +) +from recurquant.static_q468 import ( + FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + FROZEN_STATIC_Q48_PROMOTIONS, + FROZEN_STATIC_Q468_ABLATION_STEPS, + FROZEN_STATIC_Q468_PRIMARY_STEPS, + STATIC_Q48_COMPARATOR_METHOD, + STATIC_Q468_ABLATION_METHOD, + STATIC_Q468_PRIMARY_METHOD, + StaticRhtQ48Policy, + StaticRhtQ468Geometry, + StaticRhtQ468Policy, + build_static_rht_q48_policy, + build_static_rht_q468_policy, +) +from recurquant.static_q468_cache import ( + DYNAMIC_Q468_ORACLE_METHOD, + StaticRhtQwen35Cache, + create_qwen35_dynamic_q468_oracle_cache, + create_qwen35_static_rht_cache, +) +from tests.test_transformers_cache import tiny_config + +MANIFEST_SHA256 = "12" * 32 +IDENTITY_SHA256 = "34" * 32 +TOKENIZER_MANIFEST_SHA256 = "56" * 32 +SOURCE_COMMIT = "78" * 20 +BINDINGS = { + "calibration_manifest_sha256": MANIFEST_SHA256, + "identity_artifact_sha256": IDENTITY_SHA256, + "tokenizer_manifest_sha256": TOKENIZER_MANIFEST_SHA256, + "source_commit": SOURCE_COMMIT, +} + +# This layout is valid for every inherited equal-byte layer implementation and +# structurally matches tests.test_transformers_cache.tiny_config(). The static +# policies intentionally remain below its 180-byte target. +TINY_LAYOUT = EqualByteLayout( + layer_indices=(0,), + heads=2, + key_rows=8, + value_width=8, + expanded_q8_promotions=4, + multibit_marginal_steps=8, + residual_q4_rows=3, + expanded_padding_bytes=2, + multibit_padding_bytes=0, + residual_padding_bytes=0, + expected_resident_bytes=180, +) +TINY_GEOMETRY = StaticRhtQ468Geometry( + layer_indices=(0,), + heads=2, + key_rows=8, + value_width=8, + target_resident_bytes=TINY_LAYOUT.expected_resident_bytes, +) + + +def _tiny_distortions() -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + rows = TINY_GEOMETRY.total_rows + order = torch.arange(rows, dtype=torch.float64).reshape(1, 2, 8) + return 5.0 + order / 17.0, 2.0 + order.flip(-1) / 19.0, 0.5 + order / 23.0 + + +def _tiny_q468_policy() -> StaticRhtQ468Policy: + return build_static_rht_q468_policy( + *_tiny_distortions(), + geometry=TINY_GEOMETRY, + marginal_steps=8, + **BINDINGS, + ) + + +def _tiny_q48_policy() -> StaticRhtQ48Policy: + d4, _, d8 = _tiny_distortions() + return build_static_rht_q48_policy( + d4, + d8, + geometry=TINY_GEOMETRY, + promoted_rows=4, + **BINDINGS, + ) + + +def _cache(policy: StaticRhtQ468Policy | StaticRhtQ48Policy) -> StaticRhtQwen35Cache: + return StaticRhtQwen35Cache( + tiny_config(), + policy=policy, + expected_policy_sha256=policy.policy_sha256, + layout=TINY_LAYOUT, + record_evidence=True, + ) + + +def _model() -> Qwen3_5ForCausalLM: + return Qwen3_5ForCausalLM._from_config( + tiny_config(), + attn_implementation="eager", + ).eval() + + +def _checkpoint_tensors( + cache: StaticRhtQwen35Cache, +) -> tuple[tuple[str, torch.Tensor], ...]: + assert cache.checkpoint is not None + return tuple( + (name, tensor.detach().clone()) for name, tensor in cache.checkpoint.persistent_tensors() + ) + + +def _assert_checkpoint_equal(left: StaticRhtQwen35Cache, right: StaticRhtQwen35Cache) -> None: + actual = _checkpoint_tensors(left) + expected = _checkpoint_tensors(right) + assert [name for name, _tensor in actual] == [name for name, _tensor in expected] + for (_, actual_tensor), (_, expected_tensor) in zip(actual, expected, strict=True): + torch.testing.assert_close(actual_tensor, expected_tensor, rtol=0, atol=0) + + +def _mutable_nonrecurrent_tensors(cache: StaticRhtQwen35Cache) -> Iterator[torch.Tensor]: + for layer in cache.layers: + attributes = getattr(layer, "__dict__", {}) + conv_states = attributes.get("conv_states", {}) + if isinstance(conv_states, dict): + for value in conv_states.values(): + if isinstance(value, torch.Tensor): + yield value + for name in ("keys", "values"): + value = attributes.get(name) + if isinstance(value, torch.Tensor): + yield value + + +@pytest.mark.parametrize("policy_factory", [_tiny_q468_policy, _tiny_q48_policy]) +def test_static_runtime_runs_tiny_qwen_prefill_and_decode_without_policy_duplicate( + policy_factory, +) -> None: + torch.manual_seed(1301) + model = _model() + policy = policy_factory() + cache = StaticRhtQwen35Cache( + model.config, + policy=policy, + expected_policy_sha256=policy.policy_sha256, + layout=TINY_LAYOUT, + record_evidence=True, + ) + + policy_attributes = { + name: value for name, value in cache.__dict__.items() if "policy" in name.lower() + } + assert policy_attributes + assert isinstance(cache._serialized_policy, bytes) + assert all(not isinstance(value, torch.Tensor) for value in policy_attributes.values()) + assert all( + not isinstance(value, (StaticRhtQ468Policy, StaticRhtQ48Policy)) + for value in policy_attributes.values() + ) + assert cache.persistent_recurrent_tensors() == () + + prompt = torch.randint(0, model.config.vocab_size, (1, 4)) + next_token = torch.randint(0, model.config.vocab_size, (1, 1)) + with torch.inference_mode(), Qwen35EqualByteObserver(model, caches=[cache]): + prefill = model(prompt, past_key_values=cache, use_cache=True) + decode = model(next_token, past_key_values=cache, use_cache=True) + + assert prefill.logits.shape == (1, 4, model.config.vocab_size) + assert decode.logits.shape == (1, 1, model.config.vocab_size) + assert cache.update_count == 2 + assert cache.successful_tokens == 5 + assert len(cache.update_evidence) == 2 + assert cache.last_evidence is cache.update_evidence[-1] + assert cache.checkpoint is not None + assert cache.checkpoint.policy.policy_sha256 == policy.policy_sha256 + assert cache.checkpoint.policy is not policy + assert cache.persistent_raw_state_bytes() == 0 + persistent = cache.checkpoint.persistent_tensors() + assert persistent + assert all(tensor.dtype not in (torch.float32, torch.float64) for _, tensor in persistent) + nonempty = [tensor for _, tensor in persistent if tensor.numel()] + assert len({tensor.untyped_storage().data_ptr() for tensor in nonempty}) == len(nonempty) + assert sum(tensor.untyped_storage().nbytes() for _, tensor in persistent) == ( + cache.checkpoint.resident_bytes + ) + + evidence = cache.last_evidence + assert evidence is not None + assert evidence.workspace_measurement_scope == "cache_retained_forward_tensors_only" + assert evidence.cuda_allocator_peak_measured is False + assert evidence.cuda_allocator_peak_bytes is None + assert evidence.resident_tensor_storage_bytes == cache.checkpoint.resident_bytes + summary = cache.storage_summary() + assert summary["policy_tensor_storage_bytes_outside_checkpoint"] == 0 + assert summary["resident_tensor_storage_bytes"] == cache.checkpoint.resident_bytes + assert summary["cuda_allocator_peak_measured"] is False + assert summary["cuda_allocator_peak_bytes"] is None + + +def test_static_runtime_keeps_k27030_style_under_budget_without_padding() -> None: + policy = _tiny_q468_policy() + cache = _cache(policy) + model = _model() + prompt = torch.randint(0, model.config.vocab_size, (1, 3)) + with torch.inference_mode(), Qwen35EqualByteObserver(model, caches=[cache]): + model(prompt, past_key_values=cache, use_cache=True) + + assert cache.checkpoint is not None + ledger = cache.checkpoint.ledger + assert ledger.alignment_bytes == 0 + assert ledger.resident_bytes < ledger.target_resident_bytes + assert ledger.budget_delta_bytes == ledger.target_resident_bytes - ledger.resident_bytes + assert ledger.exact_budget_eligible is False + assert cache.checkpoint.state.padding.numel() == 0 + summary = cache.storage_summary() + assert summary["expected_resident_bytes"] == ledger.resident_bytes + assert summary["target_resident_bytes"] == ledger.target_resident_bytes + assert summary["budget_delta_bytes"] == ledger.budget_delta_bytes + + +def test_static_runtime_reauthenticates_mutable_policy_storage_before_decode() -> None: + model = _model() + policy = _tiny_q468_policy() + cache = _cache(policy) + prompt = torch.randint(0, model.config.vocab_size, (1, 3)) + with torch.inference_mode(), Qwen35EqualByteObserver(model, caches=[cache]): + model(prompt, past_key_values=cache, use_cache=True) + + assert cache.checkpoint is not None + codes = cache.checkpoint.policy.packed_precision_codes + with torch.inference_mode(): + codes[0] = int(codes[0].item()) ^ 0x01 + + with pytest.raises(ValueError): + cache.begin_statelease_forward_transaction() + assert cache._active_equal_byte_transaction is None + + +def test_lm_head_failure_rolls_back_static_checkpoint_and_retry_matches_clean() -> None: + torch.manual_seed(1302) + model = _model() + clean_model = copy.deepcopy(model).eval() + policy = _tiny_q468_policy() + cache = StaticRhtQwen35Cache( + model.config, + policy=policy, + expected_policy_sha256=policy.policy_sha256, + layout=TINY_LAYOUT, + record_evidence=True, + ) + clean_cache = StaticRhtQwen35Cache( + clean_model.config, + policy=policy, + expected_policy_sha256=policy.policy_sha256, + layout=TINY_LAYOUT, + record_evidence=True, + ) + prompt = torch.randint(0, model.config.vocab_size, (1, 5)) + next_token = torch.randint(0, model.config.vocab_size, (1, 1)) + + with torch.inference_mode(): + with Qwen35EqualByteObserver(model, caches=[cache]): + model(prompt, past_key_values=cache, use_cache=True) + with Qwen35EqualByteObserver(clean_model, caches=[clean_cache]): + clean_model(prompt, past_key_values=clean_cache, use_cache=True) + + checkpoint_before = cache.checkpoint + evidence_before = tuple(cache.update_evidence) + count_before = cache.update_count + nonrecurrent_before = tuple( + tensor.detach().clone() for tensor in _mutable_nonrecurrent_tensors(cache) + ) + + def fail_after_lm_head( + module: torch.nn.Module, + inputs: tuple[torch.Tensor, ...], + output: torch.Tensor, + ) -> None: + del module, inputs, output + raise RuntimeError("injected static later-layer failure") + + handle = model.lm_head.register_forward_hook(fail_after_lm_head) + try: + with ( + torch.inference_mode(), + Qwen35EqualByteObserver(model, caches=[cache]), + pytest.raises(RuntimeError, match="injected static later-layer failure"), + ): + model(next_token, past_key_values=cache, use_cache=True) + finally: + handle.remove() + + assert cache.checkpoint is checkpoint_before + assert tuple(cache.update_evidence) == evidence_before + assert cache.update_count == count_before + assert not cache._pending_observations + assert not cache._previous_states + assert not cache._final_states + assert not cache._queries + nonrecurrent_after = tuple(_mutable_nonrecurrent_tensors(cache)) + assert len(nonrecurrent_after) == len(nonrecurrent_before) + for actual, expected in zip(nonrecurrent_after, nonrecurrent_before, strict=True): + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + with torch.inference_mode(): + with Qwen35EqualByteObserver(model, caches=[cache]): + retry = model(next_token, past_key_values=cache, use_cache=True) + with Qwen35EqualByteObserver(clean_model, caches=[clean_cache]): + clean = clean_model(next_token, past_key_values=clean_cache, use_cache=True) + torch.testing.assert_close(retry.logits, clean.logits, rtol=0, atol=0) + _assert_checkpoint_equal(cache, clean_cache) + + +def test_static_pack_failure_never_replaces_previous_checkpoint( + monkeypatch: pytest.MonkeyPatch, +) -> None: + torch.manual_seed(1303) + model = _model() + policy = _tiny_q48_policy() + cache = StaticRhtQwen35Cache( + model.config, + policy=policy, + expected_policy_sha256=policy.policy_sha256, + layout=TINY_LAYOUT, + record_evidence=True, + ) + prompt = torch.randint(0, model.config.vocab_size, (1, 4)) + next_token = torch.randint(0, model.config.vocab_size, (1, 1)) + with torch.inference_mode(), Qwen35EqualByteObserver(model, caches=[cache]): + model(prompt, past_key_values=cache, use_cache=True) + + checkpoint_before = cache.checkpoint + evidence_before = tuple(cache.update_evidence) + count_before = cache.update_count + + def fail_pack(states: dict[int, torch.Tensor]): + del states + raise RuntimeError("injected static global pack failure") + + monkeypatch.setattr(cache, "_pack_static_candidate", fail_pack) + with ( + torch.inference_mode(), + Qwen35EqualByteObserver(model, caches=[cache]), + pytest.raises(RuntimeError, match="injected static global pack failure"), + ): + model(next_token, past_key_values=cache, use_cache=True) + + assert cache.checkpoint is checkpoint_before + assert tuple(cache.update_evidence) == evidence_before + assert cache.update_count == count_before + assert not cache._pending_observations + assert not cache._previous_states + assert not cache._final_states + assert not cache._queries + + +def test_incomplete_static_transaction_and_direct_write_fail_closed() -> None: + cache = _cache(_tiny_q468_policy()) + state = torch.randn(1, 2, 8, 8) + with pytest.raises(RuntimeError, match="immutable-policy packer"): + cache._pack_candidate({}, torch.empty(0)) + with pytest.raises(RuntimeError, match="active root-model"): + cache.update_recurrent_state(state, 0) + + transaction = cache.begin_statelease_forward_transaction() + with pytest.raises(RuntimeError, match="exactly one recurrent receipt"): + cache.commit_statelease_forward_transaction(transaction) + cache.rollback_statelease_forward_transaction(transaction) + assert cache.checkpoint is None + + +def test_static_runtime_lifecycle_offload_prefetch_and_reset() -> None: + model = _model() + policy = _tiny_q48_policy() + cache = StaticRhtQwen35Cache( + model.config, + policy=policy, + expected_policy_sha256=policy.policy_sha256, + layout=TINY_LAYOUT, + record_evidence=True, + ) + prompt = torch.randint(0, model.config.vocab_size, (1, 3)) + with torch.inference_mode(), Qwen35EqualByteObserver(model, caches=[cache]): + model(prompt, past_key_values=cache, use_cache=True) + + cache.reorder_cache(torch.tensor([0], dtype=torch.long)) + with pytest.raises(ValueError, match="batch-one"): + cache.reorder_cache(torch.tensor([0, 0], dtype=torch.long)) + with pytest.raises(RuntimeError, match="speculative past recording"): + cache.activate_past_recording() + with pytest.raises(RuntimeError, match="cannot crop"): + cache.crop(-1) + + cache.offload_all() + assert cache.checkpoint is not None + assert all(tensor.device.type == "cpu" for _, tensor in cache.checkpoint.persistent_tensors()) + cache.prefetch_all() + assert cache.checkpoint is not None + assert cache.checkpoint.policy.policy_sha256 == policy.policy_sha256 + + cache.reset() + assert cache.checkpoint is None + assert cache.update_count == 0 + assert not cache.update_evidence + layer = cache.layers[0] + assert isinstance(layer, EqualByteLinearAttentionLayer) + assert not layer.has_previous_state[0] + + +def test_static_cache_rejects_policy_hash_geometry_and_public_method_drift() -> None: + policy = _tiny_q468_policy() + with pytest.raises(ValueError, match="policy SHA-256"): + StaticRhtQwen35Cache( + tiny_config(), + policy=policy, + expected_policy_sha256="ab" * 32, + layout=TINY_LAYOUT, + ) + + mismatched_layout = EqualByteLayout( + layer_indices=(0,), + heads=1, + key_rows=16, + value_width=8, + expanded_q8_promotions=4, + multibit_marginal_steps=8, + residual_q4_rows=3, + expanded_padding_bytes=2, + multibit_padding_bytes=0, + residual_padding_bytes=0, + expected_resident_bytes=180, + ) + with pytest.raises(ValueError, match="geometry"): + StaticRhtQwen35Cache( + tiny_config(), + policy=policy, + expected_policy_sha256=policy.policy_sha256, + layout=mismatched_layout, + ) + + with pytest.raises(ValueError, match="three frozen methods"): + create_qwen35_static_rht_cache( + tiny_config(), + policy=policy, + expected_policy_sha256=policy.policy_sha256, + ) + + +def _frozen_config(): + layer_types = [ + ( + "linear_attention" + if index in FROZEN_QWEN35_STATIC_Q468_GEOMETRY.layer_indices + else "full_attention" + ) + for index in range(23) + ] + config = tiny_config(layer_types) + config.linear_num_value_heads = FROZEN_QWEN35_STATIC_Q468_GEOMETRY.heads + config.linear_key_head_dim = FROZEN_QWEN35_STATIC_Q468_GEOMETRY.key_rows + config.linear_value_head_dim = FROZEN_QWEN35_STATIC_Q468_GEOMETRY.value_width + return config + + +@pytest.fixture(scope="module") +def frozen_policies() -> tuple[StaticRhtQ468Policy, StaticRhtQ468Policy, StaticRhtQ48Policy]: + rows = FROZEN_QWEN35_STATIC_Q468_GEOMETRY.total_rows + order = torch.arange(rows, dtype=torch.float64).reshape(1, rows) + d4 = 5.0 + order / (rows + 1) + d6 = 2.0 + order.flip(-1) / (rows + 3) + d8 = 0.5 + order / (rows + 5) + primary = build_static_rht_q468_policy( + d4, + d6, + d8, + geometry=FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + marginal_steps=FROZEN_STATIC_Q468_PRIMARY_STEPS, + method_id=STATIC_Q468_PRIMARY_METHOD, + **BINDINGS, + ) + ablation = build_static_rht_q468_policy( + d4, + d6, + d8, + geometry=FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + marginal_steps=FROZEN_STATIC_Q468_ABLATION_STEPS, + method_id=STATIC_Q468_ABLATION_METHOD, + **BINDINGS, + ) + q48 = build_static_rht_q48_policy( + d4, + d8, + geometry=FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + promoted_rows=FROZEN_STATIC_Q48_PROMOTIONS, + method_id=STATIC_Q48_COMPARATOR_METHOD, + **BINDINGS, + ) + return primary, ablation, q48 + + +def test_public_factory_supports_all_reserved_methods_without_loading_weights( + frozen_policies, +) -> None: + primary, ablation, q48 = frozen_policies + expected = { + STATIC_Q468_PRIMARY_METHOD: (3_454_664, 0, True), + STATIC_Q468_ABLATION_METHOD: (3_380_928, 73_736, False), + STATIC_Q48_COMPARATOR_METHOD: (3_454_664, 0, True), + } + for policy in (primary, ablation, q48): + cache = create_qwen35_static_rht_cache( + _frozen_config(), + policy=policy, + expected_policy_sha256=policy.policy_sha256, + ) + summary = cache.storage_summary() + resident, delta, exact = expected[policy.method_id] + assert cache.method_id == policy.method_id + assert cache.checkpoint is None + assert summary["expected_resident_bytes"] == resident + assert summary["target_resident_bytes"] == 3_454_664 + assert summary["budget_delta_bytes"] == delta + assert summary["exact_budget_eligible"] is exact + assert summary["policy_tensor_storage_bytes_outside_checkpoint"] == 0 + assert all( + not isinstance(value, (torch.Tensor, StaticRhtQ468Policy, StaticRhtQ48Policy)) + for name, value in cache.__dict__.items() + if "policy" in name.lower() + ) + + +def test_named_dynamic_q468_oracle_is_existing_exact_k27030_path() -> None: + cache = create_qwen35_dynamic_q468_oracle_cache(_frozen_config(), record_evidence=True) + assert cache.codec == RHT_Q4_Q6_Q8 + assert cache.method_id == DYNAMIC_Q468_ORACLE_METHOD + assert cache.layout is FROZEN_QWEN35_EQUAL_BYTE_LAYOUT + assert cache.layout.multibit_marginal_steps == 27_030 + assert cache.storage_summary()["expected_resident_bytes"] == 3_454_664 + + +def test_public_factories_validate_model_runtime_before_cache_construction( + frozen_policies, +) -> None: + primary = frozen_policies[0] + training_model = Qwen3_5ForCausalLM._from_config( + tiny_config(), + attn_implementation="eager", + ) + with pytest.raises(ValueError, match="inference-only"): + create_qwen35_static_rht_cache( + training_model, + policy=primary, + expected_policy_sha256=primary.policy_sha256, + ) + with pytest.raises(ValueError, match="inference-only"): + create_qwen35_dynamic_q468_oracle_cache(training_model) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is unavailable") +def test_static_runtime_tiny_qwen_cuda_smoke() -> None: + torch.manual_seed(1304) + model = _model().to("cuda") + policy = _tiny_q468_policy() + cache = StaticRhtQwen35Cache( + model.config, + policy=policy, + expected_policy_sha256=policy.policy_sha256, + layout=TINY_LAYOUT, + ) + prompt = torch.randint(0, model.config.vocab_size, (1, 3), device="cuda") + next_token = torch.randint(0, model.config.vocab_size, (1, 1), device="cuda") + with torch.inference_mode(), Qwen35EqualByteObserver(model, caches=[cache]): + model(prompt, past_key_values=cache, use_cache=True) + output = model(next_token, past_key_values=cache, use_cache=True) + + assert output.logits.device.type == "cuda" + assert cache.checkpoint is not None + assert all(tensor.device.type == "cuda" for _, tensor in cache.checkpoint.persistent_tensors()) + assert cache.persistent_raw_state_bytes() == 0 diff --git a/tests/test_static_q468_calibration.py b/tests/test_static_q468_calibration.py new file mode 100644 index 0000000..fa22ae3 --- /dev/null +++ b/tests/test_static_q468_calibration.py @@ -0,0 +1,1138 @@ +from __future__ import annotations + +import base64 +import hashlib +import importlib.util +import json +import math +import sys +import unicodedata +from pathlib import Path + +import pytest +import torch + +from recurquant.evidence import canonical_json_bytes +from recurquant.multibit_policy import allocate_exact_multibit_codes +from recurquant.static_q468 import ( + FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + FROZEN_STATIC_Q468_ABLATION_STEPS, + FROZEN_STATIC_Q468_PRIMARY_STEPS, + StaticRhtQ468Geometry, + build_static_rht_q468_policy, +) +from recurquant.static_q468_calibration import ( + CALIBRATION_SCORE_ARTIFACT_KIND, + CALIBRATION_SCORE_ARTIFACT_PROFILE, + CALIBRATION_SCORE_ARTIFACT_REVISION, + FROZEN_SOURCE_TENSOR_CONTRACT, + GENERIC_CALIBRATION_SCORE_ARTIFACT_KIND, + GENERIC_CALIBRATION_SCORE_ARTIFACT_PROFILE, + GENERIC_CALIBRATION_SCORE_ARTIFACT_REVISION, + AnchorDistortionBatch, + CalibrationAggregate, + CalibrationSequenceScores, + aggregate_calibration_scores, + allocate_frozen_static_q468_code_maps, + allocate_static_q468_code_map, + balanced_sha_rank_halves, + build_calibration_score_artifact, + build_frozen_calibration_score_artifact, + build_frozen_split_half_stability_artifact, + calibration_sequence_rank_sha256, + deserialize_calibration_score_artifact, + deserialize_frozen_split_half_stability_artifact, + evaluate_policy_stability, + fit_split_half_policy, + frozen_anchor_positions, + per_layer_mean_bitwidth_shifts, + q8_set_jaccard, + reduce_anchor_distortions, + reduce_frozen_anchor_distortions, + static_q468_code_map_sha256, + verify_calibration_score_artifact, + verify_frozen_split_half_stability_artifact, +) + +TINY_GEOMETRY = StaticRhtQ468Geometry( + layer_indices=(0, 2), + heads=1, + key_rows=2, + value_width=4, + target_resident_bytes=64, +) +FAKE_IDENTITY_SHA256 = hashlib.sha256(b"experiment-013-identity").hexdigest() +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +CAPTURE_SPEC = importlib.util.spec_from_file_location( + "calibration_test_capture_static_q468_identity_input", + REPOSITORY_ROOT / "scripts" / "capture_static_q468_identity_input.py", +) +assert CAPTURE_SPEC is not None and CAPTURE_SPEC.loader is not None +capture = importlib.util.module_from_spec(CAPTURE_SPEC) +sys.modules[CAPTURE_SPEC.name] = capture +CAPTURE_SPEC.loader.exec_module(capture) + + +def _batch( + *, + family: str, + config: str, + canonical_id: str, + values: torch.Tensor, + ruler_category: str | None = None, + seed: int | None = None, + configured_length: int | None = None, + token_count: int = 1, + energy: torch.Tensor | None = None, +) -> AnchorDistortionBatch: + anchors = frozen_anchor_positions(token_count) + row = values.to(torch.float32).reshape(1, -1) + repeated = row.repeat(len(anchors), 1) + selected_energy = torch.ones_like(repeated) if energy is None else energy.to(torch.float32) + return AnchorDistortionBatch( + family=family, # type: ignore[arg-type] + config=config, + ruler_category=ruler_category, # type: ignore[arg-type] + canonical_id=canonical_id, + seed=seed, + configured_length=configured_length, + token_count=token_count, + anchor_positions=anchors, + query_energy=selected_energy, + q4_mse=repeated, + q6_mse=repeated / 2, + q8_mse=repeated / 4, + ) + + +def _score( + family: str, + config: str, + canonical_id: str, + scalar: float, + *, + ruler_category: str | None = None, + seed: int | None = None, + configured_length: int | None = None, +) -> CalibrationSequenceScores: + return reduce_anchor_distortions( + _batch( + family=family, + config=config, + canonical_id=canonical_id, + values=scalar * torch.arange(1, 5, dtype=torch.float64), + ruler_category=ruler_category, + seed=seed, + configured_length=configured_length, + ) + ) + + +def _calibration_sequences(*, identical_pairs: bool = False) -> list[CalibrationSequenceScores]: + pairs: list[tuple[str, str | None, tuple[str, str], tuple[float, float]]] = [ + ("mbpp", None, ("default", "default"), (3.0, 3.0 if identical_pairs else 9.0)), + ("pg19", None, ("default", "default"), (12.0, 12.0 if identical_pairs else 18.0)), + ( + "ruler", + "retrieval", + ("niah_single_1", "niah_multikey_1"), + (3.0, 3.0 if identical_pairs else 9.0), + ), + ( + "ruler", + "multi_hop_tracing", + ("vt", "vt"), + (6.0, 6.0 if identical_pairs else 12.0), + ), + ( + "ruler", + "aggregation", + ("cwe", "fwe"), + (9.0, 9.0 if identical_pairs else 15.0), + ), + ( + "ruler", + "question_answering", + ("qa_1", "qa_2"), + (12.0, 12.0 if identical_pairs else 18.0), + ), + ] + result: list[CalibrationSequenceScores] = [] + for group_index, (family, category, configs, values) in enumerate(pairs): + for member_index, value in enumerate(values): + config = configs[member_index] + result.append( + _score( + family, + config, + f"{family}-{config}-{group_index}-{member_index}", + value, + ruler_category=category, + seed=100 + member_index if family == "ruler" else None, + configured_length=(2_048, 4_096)[member_index] if family == "ruler" else None, + ) + ) + return result + + +def _tiny_aggregate() -> CalibrationAggregate: + return aggregate_calibration_scores(_calibration_sequences()) + + +def _synthetic_frozen_aggregate() -> CalibrationAggregate: + rows = FROZEN_QWEN35_STATIC_Q468_GEOMETRY.total_rows + return CalibrationAggregate( + d4=torch.full((rows,), 4.0, dtype=torch.float64), + d6=torch.full((rows,), 2.0, dtype=torch.float64), + d8=torch.full((rows,), 1.0, dtype=torch.float64), + family_sequence_counts=(("mbpp", 128), ("pg19", 16), ("ruler", 16)), + ruler_category_sequence_counts=( + ("retrieval", 4), + ("multi_hop_tracing", 4), + ("aggregation", 4), + ("question_answering", 4), + ), + sequence_score_manifest_sha256="d" * 64, + source_contract=FROZEN_SOURCE_TENSOR_CONTRACT, + identity_record_manifest_sha256="e" * 64, + ) + + +def _synthetic_frozen_half_aggregate(identity_manifest: str) -> CalibrationAggregate: + rows = FROZEN_QWEN35_STATIC_Q468_GEOMETRY.total_rows + row_axis = torch.arange(rows, dtype=torch.float64) + return CalibrationAggregate( + d4=4.0 + row_axis / rows, + d6=2.0 + row_axis / (2 * rows), + d8=1.0 + row_axis / (4 * rows), + family_sequence_counts=(("mbpp", 64), ("pg19", 8), ("ruler", 8)), + ruler_category_sequence_counts=( + ("retrieval", 2), + ("multi_hop_tracing", 2), + ("aggregation", 2), + ("question_answering", 2), + ), + sequence_score_manifest_sha256="c" * 64, + source_contract=FROZEN_SOURCE_TENSOR_CONTRACT, + identity_record_manifest_sha256=identity_manifest, + ) + + +def _rehashed_document(document: dict[str, object]) -> bytes: + evidence = document["evidence"] + assert isinstance(evidence, dict) + document["canonical_evidence_sha256"] = hashlib.sha256( + canonical_json_bytes(evidence) + ).hexdigest() + return canonical_json_bytes(document) + + +def _relabel_generic_document_as_official(raw: bytes) -> dict[str, object]: + document = json.loads(raw) + document["artifact_kind"] = CALIBRATION_SCORE_ARTIFACT_KIND + evidence = document["evidence"] + assert isinstance(evidence, dict) + evidence["artifact_profile"] = CALIBRATION_SCORE_ARTIFACT_PROFILE + evidence["artifact_revision"] = CALIBRATION_SCORE_ARTIFACT_REVISION + evidence["identity_record_manifest_sha256"] = "e" * 64 + return document + + +@pytest.mark.parametrize( + ("token_count", "expected"), + [ + (1, (0,)), + (5, (0, 1, 2, 3, 4)), + (16, tuple(range(16))), + (17, (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16)), + ( + 2_304, + ( + 143, + 287, + 431, + 575, + 719, + 863, + 1007, + 1151, + 1295, + 1439, + 1583, + 1727, + 1871, + 2015, + 2159, + 2303, + ), + ), + ], +) +def test_frozen_anchor_equation(token_count: int, expected: tuple[int, ...]) -> None: + assert frozen_anchor_positions(token_count) == expected + + +@pytest.mark.parametrize("bad", [0, -1, True, 1.5]) +def test_frozen_anchor_equation_rejects_empty_or_non_integer_counts(bad: object) -> None: + with pytest.raises((TypeError, ValueError)): + frozen_anchor_positions(bad) # type: ignore[arg-type] + + +def test_anchor_reduction_is_energy_weighted_cpu_fp64_and_hash_bound() -> None: + token_count = 4 + energy = torch.tensor( + [ + [1.0, 2.0, 3.0, 4.0], + [2.0, 3.0, 4.0, 5.0], + [3.0, 4.0, 5.0, 6.0], + [4.0, 5.0, 6.0, 7.0], + ] + ) + batch = _batch( + family="mbpp", + config="default", + canonical_id="task-1", + values=torch.tensor([2.0, 4.0, 6.0, 8.0]), + token_count=token_count, + energy=energy, + ) + + result = reduce_anchor_distortions(batch) + + expected_q4 = (energy.to(torch.float64) * batch.q4_mse.to(torch.float64)).mean(dim=0) + assert result.d4.dtype == torch.float64 + assert result.d4.device.type == "cpu" + assert torch.equal(result.d4, expected_q4) + assert torch.equal(result.d6, expected_q4 / 2) + assert torch.equal(result.d8, expected_q4 / 4) + assert len(result.anchor_manifest_sha256) == 64 + assert len(result.anchor_inputs_sha256) == 64 + assert len(result.sequence_scores_sha256) == 64 + assert reduce_anchor_distortions(batch).sequence_scores_sha256 == result.sequence_scores_sha256 + + +@pytest.mark.parametrize( + ("field", "replacement", "message"), + [ + ("anchor_positions", (0,), "anchor_positions"), + ("query_energy", torch.tensor([[math.nan] * 4] * 4), "finite"), + ("q4_mse", torch.tensor([[-1.0] * 4] * 4), "non-negative"), + ("q6_mse", torch.ones(4, 5), "match shape"), + ], +) +def test_anchor_reduction_rejects_malformed_or_nonfinite_inputs( + field: str, + replacement: object, + message: str, +) -> None: + original = _batch( + family="mbpp", + config="default", + canonical_id="task-bad", + values=torch.ones(4), + token_count=4, + ) + values = {name: getattr(original, name) for name in AnchorDistortionBatch.__dataclass_fields__} + values[field] = replacement + with pytest.raises((TypeError, ValueError), match=message): + reduce_anchor_distortions(AnchorDistortionBatch(**values)) + + +@pytest.mark.parametrize( + ("family", "category", "message"), + [ + ("ruler", None, "ruler_category must be one of"), + ("ruler", "reasoning", "ruler_category must be one of"), + ("mbpp", "retrieval", "must be None"), + ("pg19", "question_answering", "must be None"), + ], +) +def test_ruler_category_is_required_only_for_ruler( + family: str, + category: str | None, + message: str, +) -> None: + batch = _batch( + family=family, + config="exact-config", + canonical_id=f"{family}-category-contract", + values=torch.ones(4), + ruler_category=category, + ) + with pytest.raises(ValueError, match=message): + reduce_anchor_distortions(batch) + + +@pytest.mark.parametrize( + ("family", "category", "configured_length", "message"), + [ + ("ruler", "retrieval", None, "configured_length must be an integer"), + ("ruler", "retrieval", 0, "configured_length must be positive"), + ("mbpp", None, 2_048, "configured_length must be None"), + ], +) +def test_configured_length_is_separate_and_ruler_only( + family: str, + category: str | None, + configured_length: int | None, + message: str, +) -> None: + batch = _batch( + family=family, + config="exact-config", + canonical_id=f"{family}-length-contract", + values=torch.ones(4), + ruler_category=category, + configured_length=configured_length, + ) + with pytest.raises((TypeError, ValueError), match=message): + reduce_anchor_distortions(batch) + + +def test_ruler_configured_length_can_differ_from_actual_anchor_length() -> None: + valid = reduce_anchor_distortions( + _batch( + family="ruler", + config="niah_single_1", + canonical_id="shorter-generated-sequence", + values=torch.ones(4), + ruler_category="retrieval", + configured_length=4_096, + token_count=3, + ) + ) + assert valid.configured_length == 4_096 + + with pytest.raises(ValueError, match="cannot exceed"): + reduce_anchor_distortions( + _batch( + family="ruler", + config="niah_single_1", + canonical_id="ruler-too-long", + values=torch.ones(4), + ruler_category="retrieval", + configured_length=4, + token_count=5, + ) + ) + assert valid.token_count == 3 + assert valid.anchor_positions == (0, 1, 2) + + +def test_metadata_requires_nfc_text_and_nonnegative_seed() -> None: + decomposed_id = unicodedata.normalize("NFD", "café") + assert decomposed_id != "café" + with pytest.raises(ValueError, match="NFC"): + reduce_anchor_distortions( + _batch( + family="mbpp", + config="full", + canonical_id=decomposed_id, + values=torch.ones(4), + ) + ) + with pytest.raises(ValueError, match="non-negative"): + reduce_anchor_distortions( + _batch( + family="ruler", + config="vt", + canonical_id="vt-negative-seed", + values=torch.ones(4), + ruler_category="multi_hop_tracing", + seed=-1, + configured_length=2_048, + ) + ) + + +def test_source_shape_and_dtype_are_bound_into_sequence_hashes() -> None: + def make(dtype: torch.dtype, shape: tuple[int, ...]) -> AnchorDistortionBatch: + values = torch.ones(shape, dtype=dtype) + return AnchorDistortionBatch( + family="mbpp", + config="full", + ruler_category=None, + canonical_id="source-contract", + seed=None, + configured_length=None, + token_count=1, + anchor_positions=(0,), + query_energy=values, + q4_mse=values, + q6_mse=values / 2, + q8_mse=values / 4, + ) + + fp32 = reduce_anchor_distortions(make(torch.float32, (1, 4))) + fp64 = reduce_anchor_distortions(make(torch.float64, (1, 4))) + reshaped = reduce_anchor_distortions(make(torch.float32, (1, 1, 4))) + + assert torch.equal(fp32.d4, fp64.d4) + assert torch.equal(fp32.d4, reshaped.d4) + assert fp32.sequence_scores_sha256 != fp64.sequence_scores_sha256 + assert fp32.anchor_inputs_sha256 != fp64.anchor_inputs_sha256 + assert fp32.sequence_scores_sha256 != reshaped.sequence_scores_sha256 + assert fp32.source_contract != fp64.source_contract + assert fp32.source_contract != reshaped.source_contract + + +def test_frozen_reduction_requires_exact_layer_head_key_row_cpu_fp64_shape() -> None: + shape = (1, *FROZEN_SOURCE_TENSOR_CONTRACT.trailing_shape) + values = torch.ones(shape, dtype=torch.float64) + sequence_token_ids = (17,) + tokenizer_manifest_sha256 = "a" * 64 + captured_record = capture._base_record( + phase="calibration", + family="mbpp", + canonical_id="601", + config="full", + seed=None, + configured_length=None, + ruler_category=None, + generator_receipt_sha256=None, + source_payload={"task_id": 601}, + formatted_payload={"prompt": "fixture"}, + prompt_ids=sequence_token_ids, + target_ids=(), + tokenizer_manifest_sha256=tokenizer_manifest_sha256, + ) + captured_record = capture._assign_sha_ranks([captured_record])[0] + identity_record = capture.resolver._normalize_record( + captured_record, + index=0, + phase="calibration", + tokenizer_hash=tokenizer_manifest_sha256, + ) + batch = AnchorDistortionBatch( + family="mbpp", + config="full", + ruler_category=None, + canonical_id="601", + seed=None, + configured_length=None, + token_count=1, + anchor_positions=(0,), + query_energy=values, + q4_mse=values, + q6_mse=values / 2, + q8_mse=values / 4, + sequence_token_ids=sequence_token_ids, + identity_record=identity_record, + ) + + result = reduce_frozen_anchor_distortions(batch) + assert result.source_shape == shape + assert result.source_contract == FROZEN_SOURCE_TENSOR_CONTRACT + assert result.row_count == FROZEN_QWEN35_STATIC_Q468_GEOMETRY.total_rows + assert result.sequence_token_ids_sha256 == captured_record["sequence_token_ids_sha256"] + assert result.identity_anchor_manifest_sha256 == captured_record["anchor_manifest_sha256"] + assert result.identity_record_sha256 == captured_record["identity_record_sha256"] + + with pytest.raises(ValueError, match="frozen source shape"): + reduce_frozen_anchor_distortions( + _batch( + family="mbpp", + config="full", + canonical_id="wrong-frozen-shape", + values=torch.ones(4), + ) + ) + + wrong_dtype_values = values.to(torch.float32) + wrong_dtype = AnchorDistortionBatch( + family=batch.family, + config=batch.config, + ruler_category=batch.ruler_category, + canonical_id="wrong-frozen-dtype", + seed=batch.seed, + configured_length=batch.configured_length, + token_count=batch.token_count, + anchor_positions=batch.anchor_positions, + query_energy=wrong_dtype_values, + q4_mse=wrong_dtype_values, + q6_mse=wrong_dtype_values, + q8_mse=wrong_dtype_values, + ) + with pytest.raises(TypeError, match="CPU torch.float64"): + reduce_frozen_anchor_distortions(wrong_dtype) + + +def test_family_aggregation_uses_equal_broad_and_ruler_category_weights() -> None: + sequences = _calibration_sequences() + aggregate = aggregate_calibration_scores(sequences) + base = torch.arange(1, 5, dtype=torch.float64) + + # MBPP mean=6, PG19 mean=15, RULER category macro=(6+9+12+15)/4=10.5. + expected_q4 = ((6.0 + 15.0 + 10.5) / 3) * base + assert torch.equal(aggregate.d4, expected_q4) + assert torch.equal(aggregate.d6, expected_q4 / 2) + assert torch.equal(aggregate.d8, expected_q4 / 4) + assert aggregate.family_sequence_counts == (("mbpp", 2), ("pg19", 2), ("ruler", 8)) + assert aggregate.ruler_category_sequence_counts == ( + ("retrieval", 2), + ("multi_hop_tracing", 2), + ("aggregation", 2), + ("question_answering", 2), + ) + assert { + sequence.config for sequence in sequences if sequence.ruler_category == "retrieval" + } == {"niah_single_1", "niah_multikey_1"} + + +def test_aggregation_and_split_are_invariant_to_input_order() -> None: + sequences = _calibration_sequences() + forward = aggregate_calibration_scores(sequences) + reverse = aggregate_calibration_scores(list(reversed(sequences))) + assert torch.equal(forward.d4, reverse.d4) + assert torch.equal(forward.d6, reverse.d6) + assert torch.equal(forward.d8, reverse.d8) + assert forward.sequence_score_manifest_sha256 == reverse.sequence_score_manifest_sha256 + + split_forward = balanced_sha_rank_halves(sequences) + split_reverse = balanced_sha_rank_halves(list(reversed(sequences))) + assert split_forward.assignment_sha256 == split_reverse.assignment_sha256 + assert [item.canonical_dict() for item in split_forward.assignments] == [ + item.canonical_dict() for item in split_reverse.assignments + ] + assert {item.group for item in split_forward.assignments} == { + "mbpp", + "pg19", + "ruler:retrieval", + "ruler:multi_hop_tracing", + "ruler:aggregation", + "ruler:question_answering", + } + assert all( + {item.half for item in split_forward.assignments if item.group == group} == {"a", "b"} + for group in {item.group for item in split_forward.assignments} + ) + resolver_bytes = ( + json.dumps( + [item.canonical_dict() for item in split_forward.assignments], + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + b"\n" + ) + assert split_forward.assignment_sha256 == hashlib.sha256(resolver_bytes).hexdigest() + retrieval_assignments = [ + item for item in split_forward.assignments if item.group == "ruler:retrieval" + ] + assert {item.config for item in retrieval_assignments} == { + "niah_single_1", + "niah_multikey_1", + } + assert {item.configured_length for item in retrieval_assignments} == {2_048, 4_096} + assert all( + item.sequence_length == 1 and item.seed is not None for item in retrieval_assignments + ) + + +def test_split_rank_hash_matches_the_domain_separated_identity_equation() -> None: + sequence = _score( + "ruler", + "vt", + "ruler-vt-2048-12339", + 1.0, + ruler_category="multi_hop_tracing", + seed=12_339, + configured_length=2_048, + ) + identity = "\0".join( + ( + sequence.family, + str(sequence.ruler_category), + sequence.config, + sequence.canonical_id, + str(sequence.seed), + str(sequence.configured_length), + str(sequence.token_count), + ) + ) + expected = hashlib.sha256( + b"recurquant.experiment013.calibration-split.v1\0" + identity.encode() + ).hexdigest() + assert calibration_sequence_rank_sha256(sequence) == expected + + +def test_ruler_category_is_bound_into_anchor_score_and_split_hashes() -> None: + retrieval = _score( + "ruler", + "shared-exact-config", + "shared-id", + 1.0, + ruler_category="retrieval", + seed=12_339, + configured_length=2_048, + ) + aggregation = _score( + "ruler", + "shared-exact-config", + "shared-id", + 1.0, + ruler_category="aggregation", + seed=12_339, + configured_length=2_048, + ) + other_configured_length = _score( + "ruler", + "shared-exact-config", + "shared-id", + 1.0, + ruler_category="retrieval", + seed=12_339, + configured_length=4_096, + ) + + assert retrieval.anchor_manifest_sha256 != aggregation.anchor_manifest_sha256 + assert retrieval.anchor_inputs_sha256 != aggregation.anchor_inputs_sha256 + assert retrieval.sequence_scores_sha256 != aggregation.sequence_scores_sha256 + assert calibration_sequence_rank_sha256(retrieval) != calibration_sequence_rank_sha256( + aggregation + ) + assert retrieval.sequence_scores_sha256 != other_configured_length.sequence_scores_sha256 + assert calibration_sequence_rank_sha256(retrieval) != calibration_sequence_rank_sha256( + other_configured_length + ) + + +def test_split_half_fit_allocates_independently_and_passes_identical_halves() -> None: + fit = fit_split_half_policy( + _calibration_sequences(identical_pairs=True), + layer_indices=TINY_GEOMETRY.layer_indices, + rows_per_layer=TINY_GEOMETRY.rows_per_layer, + marginal_steps=4, + ) + + assert torch.equal(fit.half_a_codes, fit.half_b_codes) + assert int(fit.half_a_codes.to(torch.int64).sum().item()) == 4 + assert fit.stability.passed is True + assert fit.stability.spearman_average_ties == pytest.approx(1.0) + assert fit.stability.q8_jaccard == 1.0 + assert fit.stability.max_layer_mean_bitwidth_shift == 0.0 + + +def test_frozen_split_half_artifact_recomputes_k29334_and_rejects_tampering() -> None: + identity_file_sha256 = "1" * 64 + canonical_identity_sha256 = "2" * 64 + resolver_assignment_sha256 = "3" * 64 + raw = build_frozen_split_half_stability_artifact( + _synthetic_frozen_half_aggregate("a" * 64), + _synthetic_frozen_half_aggregate("b" * 64), + identity_file_sha256=identity_file_sha256, + canonical_identity_sha256=canonical_identity_sha256, + resolver_assignment_sha256=resolver_assignment_sha256, + full_sequence_score_manifest_sha256="c" * 64, + full_calibration_scores_sha256="d" * 64, + ) + + decoded = deserialize_frozen_split_half_stability_artifact( + raw, + expected_identity_file_sha256=identity_file_sha256, + expected_canonical_identity_sha256=canonical_identity_sha256, + expected_resolver_assignment_sha256=resolver_assignment_sha256, + ) + assert decoded.stability.passed is True + assert decoded.stability.spearman_average_ties == pytest.approx(1.0) + assert torch.equal(decoded.half_a_codes, decoded.half_b_codes) + assert int(decoded.half_a_codes.to(torch.int64).sum().item()) == ( + FROZEN_STATIC_Q468_PRIMARY_STEPS + ) + + threshold_tamper = json.loads(raw) + threshold_tamper["evidence"]["thresholds"]["minimum_spearman_average_ties"] = (0.69).hex() + report = verify_frozen_split_half_stability_artifact(_rehashed_document(threshold_tamper)) + assert report["valid"] is False + assert "thresholds drifted" in report["errors"][0] + + metric_tamper = json.loads(raw) + metric_tamper["evidence"]["metrics"]["q8_jaccard"] = (0.5).hex() + report = verify_frozen_split_half_stability_artifact(_rehashed_document(metric_tamper)) + assert report["valid"] is False + assert "metrics or checks drifted" in report["errors"][0] + + map_tamper = json.loads(raw) + encoded_codes = map_tamper["evidence"]["halves"][0]["code_map"]["codes_base64"] + code_bytes = bytearray(base64.b64decode(encoded_codes)) + code_bytes[0] = (code_bytes[0] + 1) % 3 + map_tamper["evidence"]["halves"][0]["code_map"]["codes_base64"] = base64.b64encode( + code_bytes + ).decode("ascii") + report = verify_frozen_split_half_stability_artifact(_rehashed_document(map_tamper)) + assert report["valid"] is False + assert "differs from exact allocation" in report["errors"][0] + + identity_tamper = json.loads(raw) + identity_tamper["evidence"]["identity"]["identity_file_sha256"] = "9" * 64 + report = verify_frozen_split_half_stability_artifact( + _rehashed_document(identity_tamper), + expected_identity_file_sha256=identity_file_sha256, + expected_canonical_identity_sha256=canonical_identity_sha256, + expected_resolver_assignment_sha256=resolver_assignment_sha256, + ) + assert report["valid"] is False + assert "differs from expected identity" in report["errors"][0] + + +def test_fast_allocation_matches_exhaustive_oracle_for_every_tiny_budget() -> None: + aggregate = CalibrationAggregate( + d4=torch.tensor([10.0, 8.0, 4.0, 2.0], dtype=torch.float64), + d6=torch.tensor([4.0, 5.0, 3.0, 1.5], dtype=torch.float64), + d8=torch.tensor([1.0, 2.0, 2.0, 1.0], dtype=torch.float64), + family_sequence_counts=(("mbpp", 2), ("pg19", 2), ("ruler", 8)), + ruler_category_sequence_counts=( + ("retrieval", 2), + ("multi_hop_tracing", 2), + ("aggregation", 2), + ("question_answering", 2), + ), + sequence_score_manifest_sha256="a" * 64, + source_contract=_tiny_aggregate().source_contract, + ) + for marginal_steps in range(2 * aggregate.row_count + 1): + actual = allocate_static_q468_code_map( + aggregate, + marginal_steps=marginal_steps, + ) + expected = allocate_exact_multibit_codes( + aggregate.d4.reshape(1, -1), + aggregate.d6.reshape(1, -1), + aggregate.d8.reshape(1, -1), + marginal_steps=marginal_steps, + ).reshape(-1) + assert torch.equal(actual, expected) + + +def test_stability_metrics_define_ties_empty_q8_and_bitwidth_shift() -> None: + tied = evaluate_policy_stability( + torch.tensor([0, 0, 2], dtype=torch.uint8), + torch.tensor([0, 2, 2], dtype=torch.uint8), + layer_indices=(0,), + rows_per_layer=3, + ) + assert tied.spearman_average_ties == pytest.approx(0.5) + assert tied.q8_jaccard == pytest.approx(0.5) + assert tied.layer_mean_bitwidth_shifts == ((0, pytest.approx(4 / 3)),) + assert tied.passed is False + + constant = evaluate_policy_stability( + torch.zeros(4, dtype=torch.uint8), + torch.zeros(4, dtype=torch.uint8), + layer_indices=(0, 1), + rows_per_layer=2, + expected_marginal_steps=0, + ) + assert constant.spearman_average_ties is None + assert constant.q8_jaccard == 1.0 + assert constant.passed is False + + shifts = per_layer_mean_bitwidth_shifts( + torch.tensor([2, 0, 1, 1], dtype=torch.uint8), + torch.tensor([1, 0, 1, 1], dtype=torch.uint8), + layer_indices=(0, 1), + rows_per_layer=2, + ) + assert shifts == ((0, 1.0), (1, 0.0)) + assert ( + q8_set_jaccard(torch.zeros(2, dtype=torch.uint8), torch.zeros(2, dtype=torch.uint8)) == 1.0 + ) + + +def test_stability_gate_enforces_exact_budget_and_rejects_bad_codes() -> None: + with pytest.raises(ValueError, match="half B"): + evaluate_policy_stability( + torch.tensor([2, 1, 0, 1], dtype=torch.uint8), + torch.tensor([2, 1, 0, 0], dtype=torch.uint8), + layer_indices=(0, 1), + rows_per_layer=2, + expected_marginal_steps=4, + ) + with pytest.raises(ValueError, match="0, 1, and 2"): + evaluate_policy_stability( + torch.tensor([3, 0], dtype=torch.uint8), + torch.tensor([2, 1], dtype=torch.uint8), + layer_indices=(0,), + rows_per_layer=2, + ) + + +def test_canonical_score_artifact_round_trips_and_matches_policy_hashes() -> None: + aggregate = _tiny_aggregate() + first = build_calibration_score_artifact( + aggregate, + geometry=TINY_GEOMETRY, + calibration_identity_sha256=FAKE_IDENTITY_SHA256, + marginal_steps=[4, 2], + ) + second = build_calibration_score_artifact( + aggregate, + geometry=TINY_GEOMETRY, + calibration_identity_sha256=FAKE_IDENTITY_SHA256, + marginal_steps=[2, 4], + ) + assert first == second + document = json.loads(first) + evidence = document["evidence"] + assert document["artifact_kind"] == GENERIC_CALIBRATION_SCORE_ARTIFACT_KIND + assert evidence["artifact_profile"] == GENERIC_CALIBRATION_SCORE_ARTIFACT_PROFILE + assert evidence["artifact_revision"] == GENERIC_CALIBRATION_SCORE_ARTIFACT_REVISION + assert "ruler_category_sequence_counts" in evidence + assert "ruler_subfamily_sequence_counts" not in evidence + assert evidence["source_tensor_contract"] == aggregate.source_contract.canonical_dict() + + decoded = deserialize_calibration_score_artifact( + first, + expected_file_sha256=hashlib.sha256(first).hexdigest(), + ) + report = verify_calibration_score_artifact(first) + assert report["valid"] is True + assert report["errors"] == [] + assert decoded.geometry == TINY_GEOMETRY + assert decoded.artifact_kind == GENERIC_CALIBRATION_SCORE_ARTIFACT_KIND + assert decoded.artifact_profile == GENERIC_CALIBRATION_SCORE_ARTIFACT_PROFILE + assert decoded.aggregate.sequence_score_manifest_sha256 == ( + aggregate.sequence_score_manifest_sha256 + ) + assert [steps for steps, _codes, _digest in decoded.allocations] == [2, 4] + + for steps, codes, digest in decoded.allocations: + policy = build_static_rht_q468_policy( + aggregate.d4, + aggregate.d6, + aggregate.d8, + geometry=TINY_GEOMETRY, + marginal_steps=steps, + calibration_manifest_sha256=aggregate.sequence_score_manifest_sha256, + identity_artifact_sha256=FAKE_IDENTITY_SHA256, + tokenizer_manifest_sha256="b" * 64, + source_commit="c" * 40, + ) + assert digest == policy.code_map_sha256 + assert digest == static_q468_code_map_sha256( + codes, + geometry=TINY_GEOMETRY, + marginal_steps=steps, + ) + assert decoded.calibration_scores_sha256 == policy.calibration_scores_sha256 + + +def test_official_frozen_artifact_round_trips_only_with_exact_profile() -> None: + raw = build_frozen_calibration_score_artifact( + _synthetic_frozen_aggregate(), + calibration_identity_sha256=FAKE_IDENTITY_SHA256, + ) + + decoded = deserialize_calibration_score_artifact(raw) + document = json.loads(raw) + + assert decoded.artifact_kind == CALIBRATION_SCORE_ARTIFACT_KIND + assert decoded.artifact_profile == CALIBRATION_SCORE_ARTIFACT_PROFILE + assert decoded.artifact_revision == CALIBRATION_SCORE_ARTIFACT_REVISION + assert decoded.geometry == FROZEN_QWEN35_STATIC_Q468_GEOMETRY + assert decoded.aggregate.source_contract == FROZEN_SOURCE_TENSOR_CONTRACT + assert [steps for steps, _codes, _digest in decoded.allocations] == [ + FROZEN_STATIC_Q468_ABLATION_STEPS, + FROZEN_STATIC_Q468_PRIMARY_STEPS, + ] + assert document["artifact_kind"] == CALIBRATION_SCORE_ARTIFACT_KIND + assert verify_calibration_score_artifact(raw)["valid"] is True + + +def test_generic_artifact_cannot_be_relabelled_as_official_geometry_counts_or_budget() -> None: + raw = build_calibration_score_artifact( + _tiny_aggregate(), + geometry=TINY_GEOMETRY, + calibration_identity_sha256=FAKE_IDENTITY_SHA256, + marginal_steps=[2, 4], + ) + + tiny_geometry = _relabel_generic_document_as_official(raw) + report = verify_calibration_score_artifact(_rehashed_document(tiny_geometry)) + assert report["valid"] is False + assert "exact frozen geometry" in report["errors"][0] + + bad_counts = _relabel_generic_document_as_official(raw) + evidence = bad_counts["evidence"] + assert isinstance(evidence, dict) + evidence["geometry"] = FROZEN_QWEN35_STATIC_Q468_GEOMETRY.canonical_dict() + evidence["geometry_sha256"] = FROZEN_QWEN35_STATIC_Q468_GEOMETRY.geometry_sha256 + report = verify_calibration_score_artifact(_rehashed_document(bad_counts)) + assert report["valid"] is False + assert "MBPP=128, PG19=16, RULER=16" in report["errors"][0] + + bad_budget = _relabel_generic_document_as_official(raw) + evidence = bad_budget["evidence"] + assert isinstance(evidence, dict) + evidence["geometry"] = FROZEN_QWEN35_STATIC_Q468_GEOMETRY.canonical_dict() + evidence["geometry_sha256"] = FROZEN_QWEN35_STATIC_Q468_GEOMETRY.geometry_sha256 + evidence["family_sequence_counts"] = [ + {"count": 128, "family": "mbpp"}, + {"count": 16, "family": "pg19"}, + {"count": 16, "family": "ruler"}, + ] + evidence["ruler_category_sequence_counts"] = [ + {"category": category, "count": 4} + for category in ( + "retrieval", + "multi_hop_tracing", + "aggregation", + "question_answering", + ) + ] + evidence["source_tensor_contract"] = FROZEN_SOURCE_TENSOR_CONTRACT.canonical_dict() + report = verify_calibration_score_artifact(_rehashed_document(bad_budget)) + assert report["valid"] is False + assert "exactly K27030 and K29334" in report["errors"][0] + + +def test_artifact_validation_rejects_noncanonical_malformed_and_tampered_data() -> None: + raw = build_calibration_score_artifact( + _tiny_aggregate(), + geometry=TINY_GEOMETRY, + calibration_identity_sha256=FAKE_IDENTITY_SHA256, + marginal_steps=[2, 4], + ) + + assert verify_calibration_score_artifact(raw + b" ")["valid"] is False + duplicate = raw.replace( + b'{\n "artifact_kind"', + b'{\n "schema_version": 1,\n "artifact_kind"', + 1, + ) + duplicate_report = verify_calibration_score_artifact(duplicate) + assert duplicate_report["valid"] is False + assert "duplicate JSON object key" in duplicate_report["errors"][0] + + document = json.loads(raw) + evidence = document["evidence"] + score_record = evidence["scores"] + score_bytes = bytearray(base64.b64decode(score_record["data_base64"])) + score_bytes[0] ^= 1 + score_record["data_base64"] = base64.b64encode(score_bytes).decode() + score_tamper = verify_calibration_score_artifact(_rehashed_document(document)) + assert score_tamper["valid"] is False + assert "calibration score SHA-256 mismatch" in score_tamper["errors"][0] + + document = json.loads(raw) + evidence = document["evidence"] + evidence["allocations"][0]["code_map_sha256"] = "0" * 64 + code_tamper = verify_calibration_score_artifact(_rehashed_document(document)) + assert code_tamper["valid"] is False + assert "code-map SHA-256" in code_tamper["errors"][0] + + document = json.loads(raw) + evidence = document["evidence"] + score_record = evidence["scores"] + score_bytes = bytearray(base64.b64decode(score_record["data_base64"])) + score_bytes[:8] = torch.tensor(float("nan"), dtype=torch.float64).numpy().tobytes() + score_record["data_base64"] = base64.b64encode(score_bytes).decode() + nonfinite = verify_calibration_score_artifact(_rehashed_document(document)) + assert nonfinite["valid"] is False + assert "finite and non-negative" in nonfinite["errors"][0] + + wrong_file = verify_calibration_score_artifact(raw, expected_file_sha256="0" * 64) + assert wrong_file["valid"] is False + assert "file SHA-256 mismatch" in wrong_file["errors"][0] + + +def test_artifact_builder_rejects_nonfinite_aggregate_and_bad_identity_hash() -> None: + aggregate = _tiny_aggregate() + broken = CalibrationAggregate( + d4=aggregate.d4.clone(), + d6=aggregate.d6.clone(), + d8=aggregate.d8.clone(), + family_sequence_counts=aggregate.family_sequence_counts, + ruler_category_sequence_counts=aggregate.ruler_category_sequence_counts, + sequence_score_manifest_sha256=aggregate.sequence_score_manifest_sha256, + source_contract=aggregate.source_contract, + ) + broken.d4[0] = math.inf + with pytest.raises(ValueError, match="finite"): + build_calibration_score_artifact( + broken, + geometry=TINY_GEOMETRY, + calibration_identity_sha256=FAKE_IDENTITY_SHA256, + marginal_steps=[2], + ) + with pytest.raises(ValueError, match="SHA-256"): + build_calibration_score_artifact( + aggregate, + geometry=TINY_GEOMETRY, + calibration_identity_sha256="not-a-hash", + marginal_steps=[2], + ) + + +def test_artifact_builder_rejects_inconsistent_and_nonfrozen_sequence_counts() -> None: + aggregate = _tiny_aggregate() + inconsistent = CalibrationAggregate( + d4=aggregate.d4, + d6=aggregate.d6, + d8=aggregate.d8, + family_sequence_counts=(("mbpp", 2), ("pg19", 2), ("ruler", 7)), + ruler_category_sequence_counts=aggregate.ruler_category_sequence_counts, + sequence_score_manifest_sha256=aggregate.sequence_score_manifest_sha256, + source_contract=aggregate.source_contract, + ) + with pytest.raises(ValueError, match="must equal"): + build_calibration_score_artifact( + inconsistent, + geometry=TINY_GEOMETRY, + calibration_identity_sha256=FAKE_IDENTITY_SHA256, + marginal_steps=[2], + ) + with pytest.raises(ValueError, match="MBPP=128"): + build_frozen_calibration_score_artifact( + aggregate, + calibration_identity_sha256=FAKE_IDENTITY_SHA256, + ) + + frozen = _synthetic_frozen_aggregate() + wrong_source = CalibrationAggregate( + d4=frozen.d4, + d6=frozen.d6, + d8=frozen.d8, + family_sequence_counts=frozen.family_sequence_counts, + ruler_category_sequence_counts=frozen.ruler_category_sequence_counts, + sequence_score_manifest_sha256=frozen.sequence_score_manifest_sha256, + source_contract=aggregate.source_contract, + ) + with pytest.raises(ValueError, match="exact Experiment 013 source contract"): + build_frozen_calibration_score_artifact( + wrong_source, + calibration_identity_sha256=FAKE_IDENTITY_SHA256, + ) + + +def test_real_geometry_allocates_both_frozen_exact_k_maps_without_large_data() -> None: + geometry = FROZEN_QWEN35_STATIC_Q468_GEOMETRY + rows = geometry.total_rows + row_axis = torch.arange(rows, dtype=torch.float64) + aggregate = CalibrationAggregate( + d4=4.0 + row_axis / rows, + d6=2.0 + row_axis / (2 * rows), + d8=1.0 + row_axis / (4 * rows), + family_sequence_counts=(("mbpp", 128), ("pg19", 16), ("ruler", 16)), + ruler_category_sequence_counts=( + ("retrieval", 4), + ("multi_hop_tracing", 4), + ("aggregation", 4), + ("question_answering", 4), + ), + sequence_score_manifest_sha256="d" * 64, + source_contract=FROZEN_SOURCE_TENSOR_CONTRACT, + ) + + maps = allocate_frozen_static_q468_code_maps(aggregate) + + assert set(maps) == { + FROZEN_STATIC_Q468_ABLATION_STEPS, + FROZEN_STATIC_Q468_PRIMARY_STEPS, + } + for steps, codes in maps.items(): + assert codes.shape == (rows,) + assert codes.dtype == torch.uint8 + assert int(codes.to(torch.int64).sum().item()) == steps + assert sum(int((codes == code).sum().item()) for code in range(3)) == rows From 406b33ae9476893f3de6066e85413215eed9d4ae Mon Sep 17 00:00:00 2001 From: HiddenTrojan <93521146+Labeeb2339@users.noreply.github.com> Date: Sun, 2 Aug 2026 11:41:21 +0800 Subject: [PATCH 08/24] Harden Experiment 013 identity promotion --- .gitleaks.toml | 8 + scripts/capture_static_q468_identity_input.py | 1675 +++++++++++++++++ scripts/resolve_static_q468_identity.py | 1306 +++++++++++-- ...test_capture_static_q468_identity_input.py | 870 +++++++++ tests/test_resolve_static_q468_identity.py | 437 ++++- 5 files changed, 4104 insertions(+), 192 deletions(-) create mode 100644 scripts/capture_static_q468_identity_input.py create mode 100644 tests/test_capture_static_q468_identity_input.py diff --git a/.gitleaks.toml b/.gitleaks.toml index 12f479c..47e3d2a 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -45,6 +45,14 @@ regexes = [ '''"[A-Za-z0-9_./-]+\.md"\s*:\s*"[0-9a-f]{64}"''', ] +[[allowlists]] +description = "Pinned public RULER Git blob object IDs" +targetRules = ["generic-api-key"] +regexTarget = "line" +regexes = [ + '''"scripts/[A-Za-z0-9_./-]+"\s*:\s*\(?"[0-9a-f]{40}"''', +] + [[allowlists]] description = "Dtype assertions on recurrent-state buffers" targetRules = ["generic-api-key"] diff --git a/scripts/capture_static_q468_identity_input.py b/scripts/capture_static_q468_identity_input.py new file mode 100644 index 0000000..f4ebc6f --- /dev/null +++ b/scripts/capture_static_q468_identity_input.py @@ -0,0 +1,1675 @@ +#!/usr/bin/env python3 +"""Capture resolver-compatible Experiment 013 identity inputs without model weights. + +Only the public calibration and Stage-A identity surfaces are implemented. Stage B +and Stage C are rejected before a source object, local receipt, dataset, tokenizer, +or output path is touched. The live source reads pinned dataset projections and +selected public rows, downloads tokenizer assets by an explicit allow-list, and +consumes separately prepared RULER generator receipts. It never imports a model +class or requests a weight file. +""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import hashlib +import importlib.metadata +import importlib.util +import json +import os +import re +import string +import sys +import tempfile +import unicodedata +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Final, Protocol + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +RESOLVER_PATH = REPOSITORY_ROOT / "scripts" / "resolve_static_q468_identity.py" +_RESOLVER_MODULE_NAME = "recurquant_experiment013_identity_resolver" +_RESOLVER_SPEC = importlib.util.spec_from_file_location(_RESOLVER_MODULE_NAME, RESOLVER_PATH) +if _RESOLVER_SPEC is None or _RESOLVER_SPEC.loader is None: # pragma: no cover + raise RuntimeError("cannot load the Experiment 013 identity resolver") +_existing_resolver = sys.modules.get(_RESOLVER_MODULE_NAME) +if _existing_resolver is not None: + existing_path = getattr(_existing_resolver, "__file__", None) + if existing_path is None or Path(existing_path).resolve() != RESOLVER_PATH.resolve(): + raise RuntimeError("resolver module name is already bound to another file") + resolver = _existing_resolver +else: + resolver = importlib.util.module_from_spec(_RESOLVER_SPEC) + sys.modules[_RESOLVER_MODULE_NAME] = resolver + try: + _RESOLVER_SPEC.loader.exec_module(resolver) + except BaseException: + sys.modules.pop(_RESOLVER_MODULE_NAME, None) + raise + +CAPTURE_VERSION: Final = 2 +RULER_RECEIPT_SCHEMA: Final = "recurquant.experiment013.ruler-receipt.v1" +RULER_SEQUENCE_NAMESPACE: Final = "recurquant.experiment013.ruler.sequence.v1" +PG19_TRAIN_SEGMENT_NAMESPACE: Final = "recurquant.experiment013.pg19.segment.v1\0" +PG19_VALIDATION_SEGMENT_NAMESPACE: Final = "recurquant.experiment013.pg19.validation-segment.v1\0" + +SOURCE_HEAD_ORDER: Final = ( + "primary_model", + "mbpp", + "pg19", + "ruler", + "humaneval_plus", + "evalplus", +) +EXPECTED_SOURCE_HEADS: Final = { + "primary_model": resolver.PRIMARY_MODEL_REVISION, + "mbpp": resolver.MBPP_REVISION, + "pg19": resolver.PG19_REVISION, + "ruler": resolver.RULER_REVISION, + "humaneval_plus": resolver.HUMANEVAL_PLUS_REVISION, + "evalplus": resolver.EVALPLUS_SOURCE_REVISION, +} + +RULER_CONFIGS_BY_CATEGORY: Final = { + "retrieval": ( + "niah_single_1", + "niah_single_2", + "niah_single_3", + "niah_multikey_1", + "niah_multikey_2", + "niah_multikey_3", + "niah_multivalue", + "niah_multiquery", + ), + "multi_hop_tracing": ("vt",), + "aggregation": ("cwe", "fwe"), + "question_answering": ("qa_1", "qa_2"), +} +RULER_ALL_CONFIGS: Final = tuple( + config + for category in resolver.RULER_CATEGORIES + for config in RULER_CONFIGS_BY_CATEGORY[category] +) +RULER_REQUIRED_OUTPUT_COUNTS: Final = { + "niah_single_1": 1, + "niah_single_2": 1, + "niah_single_3": 1, + "niah_multikey_1": 1, + "niah_multikey_2": 1, + "niah_multikey_3": 1, + "niah_multivalue": 4, + "niah_multiquery": 4, + "vt": 5, + "cwe": 10, + "fwe": 3, +} +RULER_REQUIRED_OUTPUT_SEPARATOR: Final = ", " + +# Git object IDs from the recursive tree at RULER_REVISION. Verifying Git's +# blob hash catches an unexpected raw response before it enters the formatter +# identity. Auxiliary corpora and package resources remain receipt-bound. +RULER_GENERATOR_GIT_BLOBS: Final = { + "scripts/synthetic.yaml": "29cfa5f60b49a7fa53f8dccbbd4f0c7c9e7834fa", + "scripts/data/prepare.py": "4d106c46b7faa1deb1540b9e04a8bb2b71c01b4b", + "scripts/data/template.py": "9bbf7b91382ddb20815315cccad92e50bd95bf7e", + "scripts/data/tokenizer.py": "5a2ddb504ce26da5b43c0f196629e152fca1460b", + "scripts/data/manifest_utils.py": "63153f8579e05cbde77006558e09f1990238bd8b", + "scripts/data/synthetic/constants.py": "e1a880a1d41c953e55236966b7eb7e84174e00cc", + "scripts/data/synthetic/niah.py": "729eddc260ef5a9aa0473557cd249abca232764a", + "scripts/data/synthetic/variable_tracking.py": ("bc5dab381f38e810e5050340d8dae29ae1cfc82a"), + "scripts/data/synthetic/common_words_extraction.py": ( + "af07a9bd76fbbb96910c61b790f1c8e8e944a901" + ), + "scripts/data/synthetic/freq_words_extraction.py": ("77ddcd383d698378d2049278657e2b3aad84e3e1"), + "scripts/data/synthetic/qa.py": "d71cf0355026ab9265dc7f4de14cb04159c62230", +} + +TOKENIZER_ASSET_NAMES: Final = ( + "tokenizer.json", + "tokenizer_config.json", + "vocab.json", + "merges.txt", + "added_tokens.json", + "special_tokens_map.json", + "chat_template.jinja", +) +FORBIDDEN_MODEL_FILE_RE: Final = re.compile( + r"(?:^|/)(?:model(?:[-.]|$)|pytorch_model|tf_model|flax_model|adapter_model)" + r"|\.(?:safetensors|bin|pt|pth|ckpt|onnx|gguf)$", + flags=re.IGNORECASE, +) + +MBPP_FORMATTER_SPEC: Final = { + "id": "recurquant.mbpp-prompt-code.v1", + "prompt": ( + "You are an expert Python programmer, and here is your task: {text}\\n" + "Your code should pass these tests:\\n\\n{tests}\\n[BEGIN]\\n" + ), + "prompt_add_special_tokens": True, + "target": "code", + "target_add_special_tokens": False, + "normalization": "CRLF_and_CR_to_LF", +} +PG19_FORMATTER_SPEC: Final = { + "id": "recurquant.pg19-token-slice.v1", + "canonical_id": "exact UTF-8 url field", + "add_special_tokens": False, + "calibration": { + "tokens": 2_304, + "namespace": PG19_TRAIN_SEGMENT_NAMESPACE, + }, + "stage_a": { + "tokens": 4_224, + "prefill": 4_096, + "continuation_tokens": 128, + "cache_exposed_predictions": 127, + "namespace": PG19_VALIDATION_SEGMENT_NAMESPACE, + }, +} +HUMANEVAL_FORMATTER_SPEC: Final = { + "id": "recurquant.humaneval-plus-prompt-solution.v1", + "canonical_id": "exact task_id field", + "prompt_field": "prompt", + "prompt_add_special_tokens": True, + "target_field": "canonical_solution", + "target_add_special_tokens": False, + "target_token_cap": 128, + "minimum_target_tokens": 2, + "cache_exposed_predictions": "target_length - 1", +} + +RULER_RECEIPT_FIELDS: Final = frozenset( + { + "schema", + "source_id", + "revision", + "category", + "config", + "configured_length", + "seed", + "sample_index", + "generator_reported_length", + "input", + "answer_prefix", + "outputs", + "auxiliary_files", + } +) +AUXILIARY_FILE_FIELDS: Final = frozenset({"name", "sha256", "size_bytes"}) + + +def ruler_receipt_filename(*, category: str, config: str, configured_length: int, seed: int) -> str: + """Return the frozen filename for one separately generated RULER receipt.""" + + return f"{category}__{config}__l{configured_length}__s{seed}.json" + + +def required_ruler_receipts() -> tuple[dict[str, Any], ...]: + """Return the 16 calibration and four Stage-A receipt identities.""" + + result: list[dict[str, Any]] = [] + for phase, schedule in ( + ("calibration", resolver.RULER_CALIBRATION_SCHEDULE), + ("stage_a", resolver.RULER_STAGE_A_SCHEDULE), + ): + for category, config, configured_length, seed in schedule: + result.append( + { + "phase": phase, + "category": category, + "config": config, + "configured_length": configured_length, + "seed": seed, + "sample_index": 0, + "filename": ruler_receipt_filename( + category=category, + config=config, + configured_length=configured_length, + seed=seed, + ), + } + ) + if len(result) != 20 or len({item["filename"] for item in result}) != 20: + raise RuntimeError("frozen RULER receipt inventory is not 20 unique files") + return tuple(result) + + +@dataclass(frozen=True, slots=True) +class ProjectionRow: + canonical_id: str + offset: int + + +@dataclass(frozen=True, slots=True) +class TokenizerMaterial: + tokenizer: Any + tokenizer_class: str + transformers_version: str + files: Mapping[str, bytes] + model_weights_loaded: bool = False + + +class CaptureSource(Protocol): + """Narrow, mockable source surface used by the deterministic capturer.""" + + def source_heads(self) -> Mapping[str, str]: ... + + def tokenizer_material(self) -> TokenizerMaterial: ... + + def mbpp_train_rows(self) -> Sequence[Mapping[str, Any]]: ... + + def pg19_projection(self, split: str) -> Sequence[ProjectionRow]: ... + + def pg19_row(self, split: str, *, offset: int, expected_url: str) -> Mapping[str, Any]: ... + + def ruler_generator_files(self) -> Mapping[str, bytes]: ... + + def ruler_receipt( + self, *, category: str, config: str, configured_length: int, seed: int + ) -> Mapping[str, Any]: ... + + def humaneval_projection(self) -> Sequence[ProjectionRow]: ... + + def humaneval_row(self, *, offset: int, expected_task_id: str) -> Mapping[str, Any]: ... + + +def canonical_json_bytes(value: object) -> bytes: + return resolver.canonical_json_bytes(value) + + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _git_blob_sha1(value: bytes) -> str: + header = f"blob {len(value)}\0".encode() + return hashlib.sha1(header + value, usedforsecurity=False).hexdigest() + + +def _strict_json(raw: bytes, *, context: str) -> dict[str, Any]: + def pairs_hook(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"{context} contains duplicate key {key!r}") + result[key] = value + return result + + try: + value = json.loads(raw.decode("utf-8"), object_pairs_hook=pairs_hook) + except UnicodeDecodeError as error: + raise ValueError(f"{context} must be UTF-8") from error + except json.JSONDecodeError as error: + raise ValueError(f"{context} must be strict JSON") from error + if not isinstance(value, dict): + raise ValueError(f"{context} must contain one JSON object") + return value + + +def _require_exact_fields( + value: Mapping[str, Any], expected: frozenset[str], *, context: str +) -> None: + actual = frozenset(value) + if actual != expected: + raise ValueError( + f"{context} fields drifted; " + f"missing={sorted(expected - actual)}, extra={sorted(actual - expected)}" + ) + + +def _require_string(value: object, *, context: str, allow_empty: bool = False) -> str: + if not isinstance(value, str) or (not allow_empty and not value): + raise ValueError(f"{context} must be a {'string' if allow_empty else 'non-empty string'}") + if value != unicodedata.normalize("NFC", value): + raise ValueError(f"{context} must be NFC text") + return value + + +def _require_int(value: object, *, context: str, minimum: int = 0) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise ValueError(f"{context} must be an integer >= {minimum}") + return value + + +def _require_sha256(value: object, *, context: str) -> str: + if ( + not isinstance(value, str) + or len(value) != 64 + or value != value.lower() + or any(character not in string.hexdigits for character in value) + ): + raise ValueError(f"{context} must be a lowercase SHA-256") + return value + + +def _json_safe(value: object, *, context: str) -> object: + try: + canonical_json_bytes(value) + except (TypeError, ValueError) as error: + raise ValueError(f"{context} is not canonical-JSON serializable") from error + return value + + +def _normalize_lf(value: str) -> str: + return value.replace("\r\n", "\n").replace("\r", "\n") + + +def _encode(tokenizer: Any, text: str, *, add_special_tokens: bool) -> tuple[int, ...]: + if not isinstance(text, str): + raise ValueError("tokenizer input must be text") + encoded = tokenizer.encode(text, add_special_tokens=add_special_tokens) + if not isinstance(encoded, Sequence) or isinstance(encoded, (str, bytes, bytearray)): + raise ValueError("tokenizer.encode must return an integer sequence") + result: list[int] = [] + for index, token_id in enumerate(encoded): + if isinstance(token_id, bool) or not isinstance(token_id, int) or token_id < 0: + raise ValueError(f"tokenizer token {index} is not a non-negative integer") + result.append(token_id) + return tuple(result) + + +def _token_hash(token_ids: Sequence[int]) -> str: + return resolver.sequence_token_ids_sha256(token_ids) + + +def _anchor_manifest_hash( + *, canonical_id: str, sequence_ids: Sequence[int], token_span: Mapping[str, int] +) -> str: + return resolver.identity_anchor_manifest_sha256( + canonical_id=canonical_id, + sequence_length=len(sequence_ids), + sequence_token_ids_sha256_value=_token_hash(sequence_ids), + token_span=token_span, + ) + + +def _base_record( + *, + phase: str, + family: str, + canonical_id: str, + config: str, + seed: int | None, + configured_length: int | None, + ruler_category: str | None, + generator_receipt_sha256: str | None, + source_payload: object, + formatted_payload: object, + prompt_ids: Sequence[int], + target_ids: Sequence[int], + tokenizer_manifest_sha256: str, +) -> dict[str, Any]: + sequence_ids = tuple(prompt_ids) + tuple(target_ids) + if not sequence_ids: + raise ValueError(f"{family} record {canonical_id!r} produced no tokens") + if phase == "stage_a" and len(target_ids) < 2: + raise ValueError( + f"Stage-A {family} continuation must contain at least two tokens " + "to expose one cache prediction" + ) + scored_stop = len(sequence_ids) + cache_exposed_start = scored_stop if phase == "calibration" else len(prompt_ids) + 1 + token_span = { + "prefill_start": 0, + "prefill_stop": len(prompt_ids), + "scored_start": len(prompt_ids), + "scored_stop": scored_stop, + "cache_exposed_start": cache_exposed_start, + "cache_exposed_stop": scored_stop, + } + namespace = { + ("calibration", "mbpp"): None, + ("calibration", "pg19"): resolver.PG19_TRAIN_NAMESPACE, + ("calibration", "ruler"): resolver.RULER_CALIBRATION_SELECTION_NAMESPACE, + ("stage_a", "pg19"): resolver.PG19_VALIDATION_NAMESPACE, + ("stage_a", "ruler"): resolver.RULER_STAGE_A_SELECTION_NAMESPACE, + ("stage_a", "humaneval_plus"): resolver.HUMANEVAL_AB_NAMESPACE, + }.get((phase, family)) + if family == "mbpp": + selection_hash = resolver.mbpp_selection_sha256(canonical_id) + elif namespace is not None: + selection_hash = resolver.selection_sha256(namespace, canonical_id) + else: + raise ValueError(f"no selection namespace for {phase}/{family}") + return { + "family": family, + "canonical_id": canonical_id, + "config": config, + "selection_rank": -1, + "selection_sha256": selection_hash, + "seed": seed, + "configured_length": configured_length, + "sequence_length": len(sequence_ids), + "ruler_category": ruler_category, + "generator_receipt_sha256": generator_receipt_sha256, + "source_content_sha256": sha256_bytes(canonical_json_bytes(source_payload)), + "formatted_content_sha256": sha256_bytes(canonical_json_bytes(formatted_payload)), + "prompt_token_ids_sha256": _token_hash(prompt_ids), + "target_token_ids_sha256": _token_hash(target_ids), + "sequence_token_ids_sha256": _token_hash(sequence_ids), + "tokenizer_manifest_sha256": tokenizer_manifest_sha256, + "token_span": token_span, + "anchor_manifest_sha256": _anchor_manifest_hash( + canonical_id=canonical_id, + sequence_ids=sequence_ids, + token_span=token_span, + ), + } + + +def _assign_sha_ranks(records: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + ordered = sorted( + records, + key=lambda row: (row["selection_sha256"], row["canonical_id"]), + ) + for rank, row in enumerate(ordered): + row["selection_rank"] = rank + row["identity_record_sha256"] = resolver.identity_record_sha256(row) + return ordered + + +def _tokenizer_contract(material: TokenizerMaterial) -> tuple[dict[str, Any], str]: + if material.model_weights_loaded is not False: + raise ValueError("tokenizer capture reports that model weights were loaded") + if material.transformers_version != resolver.TRANSFORMERS_VERSION: + raise ValueError("Transformers version drifted during tokenizer capture") + if not material.files: + raise ValueError("tokenizer capture returned no files") + files: list[dict[str, Any]] = [] + names: set[str] = set() + for raw_name, content in material.files.items(): + name = _require_string(raw_name, context="tokenizer file name") + if Path(name).name != name or name in names: + raise ValueError("tokenizer file names must be unique basenames") + if FORBIDDEN_MODEL_FILE_RE.search(name): + raise ValueError(f"model weight-like file is forbidden: {name}") + if not isinstance(content, bytes) or not content: + raise ValueError(f"tokenizer file {name!r} must contain bytes") + names.add(name) + files.append({"name": name, "sha256": sha256_bytes(content), "size_bytes": len(content)}) + files.sort(key=lambda item: item["name"]) + manifest_hash = sha256_bytes(canonical_json_bytes(files)) + return ( + { + "source_id": resolver.PRIMARY_MODEL_ID, + "revision": resolver.PRIMARY_MODEL_REVISION, + "class": _require_string(material.tokenizer_class, context="tokenizer class"), + "transformers_version": resolver.TRANSFORMERS_VERSION, + "files": files, + }, + manifest_hash, + ) + + +def _validate_heads(heads: Mapping[str, str], *, context: str) -> dict[str, str]: + if set(heads) != set(SOURCE_HEAD_ORDER): + raise ValueError(f"{context} source-head fields drifted") + normalized = {key: str(heads[key]) for key in SOURCE_HEAD_ORDER} + if normalized != EXPECTED_SOURCE_HEADS: + drift = { + key: {"expected": EXPECTED_SOURCE_HEADS[key], "actual": normalized[key]} + for key in SOURCE_HEAD_ORDER + if normalized[key] != EXPECTED_SOURCE_HEADS[key] + } + raise ValueError(f"{context} source HEAD is not the pinned revision: {drift}") + return normalized + + +def _canonical_mbpp_row(row: Mapping[str, Any]) -> dict[str, Any]: + task_id = _require_int(row.get("task_id"), context="MBPP task_id", minimum=1) + text = _normalize_lf(_require_string(row.get("text"), context="MBPP text")) + code = _normalize_lf(_require_string(row.get("code"), context="MBPP code")) + tests = row.get("test_list") + challenges = row.get("challenge_test_list") + if isinstance(tests, (str, bytes)) or not isinstance(tests, Sequence): + raise ValueError("MBPP test_list must be a string sequence") + if isinstance(challenges, (str, bytes)) or not isinstance(challenges, Sequence): + raise ValueError("MBPP challenge_test_list must be a string sequence") + + def strings(values: Sequence[Any], name: str) -> list[str]: + result: list[str] = [] + for value in values: + result.append(_normalize_lf(_require_string(value, context=name))) + return result + + return { + "task_id": task_id, + "text": text, + "code": code, + "test_list": strings(tests, "MBPP test_list item"), + "test_setup_code": _normalize_lf( + _require_string( + row.get("test_setup_code"), + context="MBPP test_setup_code", + allow_empty=True, + ) + ), + "challenge_test_list": strings(challenges, "MBPP challenge item"), + } + + +def _capture_mbpp( + source: CaptureSource, + *, + phase: str, + tokenizer: Any, + tokenizer_manifest_sha256: str, +) -> tuple[list[dict[str, Any]], str]: + _selected_ids, population_hash = resolver.mbpp_calibration_identity() + if phase == "stage_a": + return [], population_hash + raw_rows = source.mbpp_train_rows() + canonical_rows = [_canonical_mbpp_row(row) for row in raw_rows] + by_id: dict[int, dict[str, Any]] = {} + for row in canonical_rows: + task_id = int(row["task_id"]) + if task_id in by_id: + raise ValueError(f"duplicate MBPP task_id {task_id}") + by_id[task_id] = row + expected_population = set(range(601, 975)) + if set(by_id) != expected_population: + raise ValueError("MBPP train population must be exactly task IDs 601..974") + selected_ids, _ = resolver.mbpp_calibration_identity() + records: list[dict[str, Any]] = [] + for task_id_text in selected_ids: + row = by_id[int(task_id_text)] + tests = "\n".join(row["test_list"]) + prompt = ( + f"You are an expert Python programmer, and here is your task: {row['text']}\n" + f"Your code should pass these tests:\n\n{tests}\n[BEGIN]\n" + ) + code = str(row["code"]) + prompt_ids = _encode(tokenizer, prompt, add_special_tokens=True) + target_ids = _encode(tokenizer, code, add_special_tokens=False) + if not prompt_ids or not target_ids: + raise ValueError(f"MBPP task {task_id_text} produced an empty token side") + records.append( + _base_record( + phase=phase, + family="mbpp", + canonical_id=task_id_text, + config=resolver.MBPP_CONFIG, + seed=None, + configured_length=None, + ruler_category=None, + generator_receipt_sha256=None, + source_payload=row, + formatted_payload={"prompt": prompt, "target": code}, + prompt_ids=prompt_ids, + target_ids=target_ids, + tokenizer_manifest_sha256=tokenizer_manifest_sha256, + ) + ) + return _assign_sha_ranks(records), population_hash + + +def _validate_projection( + rows: Sequence[ProjectionRow], *, context: str, exact_count: int | None = None +) -> tuple[ProjectionRow, ...]: + if exact_count is not None and len(rows) != exact_count: + raise ValueError(f"{context} must contain exactly {exact_count} identities") + if not rows: + raise ValueError(f"{context} cannot be empty") + seen_ids: set[str] = set() + seen_offsets: set[int] = set() + normalized: list[ProjectionRow] = [] + for index, item in enumerate(rows): + if not isinstance(item, ProjectionRow): + raise ValueError(f"{context}[{index}] is not a ProjectionRow") + canonical_id = _require_string( + item.canonical_id, context=f"{context}[{index}].canonical_id" + ) + offset = _require_int(item.offset, context=f"{context}[{index}].offset") + if canonical_id in seen_ids or offset in seen_offsets: + raise ValueError(f"{context} contains a duplicate identity or offset") + seen_ids.add(canonical_id) + seen_offsets.add(offset) + normalized.append(ProjectionRow(canonical_id, offset)) + if sorted(seen_offsets) != list(range(len(normalized))): + raise ValueError(f"{context} offsets must be contiguous from zero") + return tuple(sorted(normalized, key=lambda item: item.offset)) + + +def _segment_start(*, namespace: str, canonical_id: str, token_count: int, width: int) -> int: + if token_count < width: + raise ValueError("segment source is shorter than the requested width") + digest = hashlib.sha256(namespace.encode() + canonical_id.encode("utf-8")).digest() + return int.from_bytes(digest[:8], "big") % (token_count - width + 1) + + +def _capture_pg19( + source: CaptureSource, + *, + phase: str, + tokenizer: Any, + tokenizer_manifest_sha256: str, +) -> tuple[list[dict[str, Any]], str, str]: + split = "train" if phase == "calibration" else "validation" + required = 16 if phase == "calibration" else 4 + width = 2_304 if phase == "calibration" else 4_224 + segment_namespace = ( + PG19_TRAIN_SEGMENT_NAMESPACE + if phase == "calibration" + else PG19_VALIDATION_SEGMENT_NAMESPACE + ) + selection_namespace = ( + resolver.PG19_TRAIN_NAMESPACE + if phase == "calibration" + else resolver.PG19_VALIDATION_NAMESPACE + ) + projection = _validate_projection( + source.pg19_projection(split), + context=f"PG19 {split} URL projection", + exact_count=13_684 if split == "train" else 50, + ) + identity_manifest_hash = sha256_bytes( + canonical_json_bytes([item.canonical_id for item in projection]) + ) + ranked = sorted( + projection, + key=lambda item: ( + resolver.selection_sha256(selection_namespace, item.canonical_id), + item.canonical_id, + ), + ) + accepted: list[dict[str, Any]] = [] + for item in ranked: + raw = source.pg19_row(split, offset=item.offset, expected_url=item.canonical_id) + if set(raw) < {"url", "text"}: + raise ValueError("PG19 selected row is missing url or text") + url = _require_string(raw["url"], context="PG19 row url") + if url != item.canonical_id: + raise ValueError("PG19 selected row URL does not match its projection") + text = _require_string(raw["text"], context="PG19 row text", allow_empty=True) + full_ids = _encode(tokenizer, text, add_special_tokens=False) + if len(full_ids) < width: + continue + start = _segment_start( + namespace=segment_namespace, + canonical_id=url, + token_count=len(full_ids), + width=width, + ) + selected_ids = full_ids[start : start + width] + if phase == "calibration": + prompt_ids = selected_ids + target_ids: tuple[int, ...] = () + else: + prompt_ids = selected_ids[:4_096] + target_ids = selected_ids[4_096:] + source_payload = _json_safe(dict(raw), context="PG19 selected row") + formatted_payload = { + "url": url, + "source_token_count": len(full_ids), + "source_token_start": start, + "source_token_stop": start + width, + "selected_token_ids": list(selected_ids), + } + accepted.append( + _base_record( + phase=phase, + family="pg19", + canonical_id=url, + config="default", + seed=None, + configured_length=None, + ruler_category=None, + generator_receipt_sha256=None, + source_payload=source_payload, + formatted_payload=formatted_payload, + prompt_ids=prompt_ids, + target_ids=target_ids, + tokenizer_manifest_sha256=tokenizer_manifest_sha256, + ) + ) + if len(accepted) == required: + break + if len(accepted) != required: + raise ValueError( + f"PG19 {split} has only {len(accepted)} eligible rows; {required} required" + ) + return _assign_sha_ranks(accepted), identity_manifest_hash, split + + +def _top_level_yaml_keys(raw: bytes) -> tuple[str, ...]: + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as error: + raise ValueError("RULER synthetic.yaml must be UTF-8") from error + keys = [] + for line in text.splitlines(): + match = re.fullmatch(r"([A-Za-z0-9_]+):", line) + if match: + keys.append(match.group(1)) + return tuple(keys) + + +def _ruler_generator_manifest(files: Mapping[str, bytes]) -> list[dict[str, Any]]: + if set(files) != set(RULER_GENERATOR_GIT_BLOBS): + raise ValueError("RULER generator file set drifted from the pinned capture contract") + manifest: list[dict[str, Any]] = [] + for path in sorted(files): + content = files[path] + if not isinstance(content, bytes) or not content: + raise ValueError(f"RULER generator file {path!r} is empty") + actual_blob = _git_blob_sha1(content) + if actual_blob != RULER_GENERATOR_GIT_BLOBS[path]: + raise ValueError(f"RULER generator Git blob drifted for {path}") + manifest.append( + { + "path": path, + "git_blob_sha1": actual_blob, + "sha256": sha256_bytes(content), + "size_bytes": len(content), + } + ) + config_keys = _top_level_yaml_keys(files["scripts/synthetic.yaml"]) + if set(config_keys) != set(RULER_ALL_CONFIGS) or len(config_keys) != len(RULER_ALL_CONFIGS): + raise ValueError("pinned RULER synthetic config inventory drifted") + return manifest + + +def _normalize_auxiliary_files(value: object, *, context: str) -> list[dict[str, Any]]: + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise ValueError(f"{context} must be an array") + if not value: + raise ValueError(f"{context} cannot be empty; external resources must be bound") + result: list[dict[str, Any]] = [] + names: set[str] = set() + for index, raw in enumerate(value): + if not isinstance(raw, Mapping): + raise ValueError(f"{context}[{index}] must be an object") + _require_exact_fields(raw, AUXILIARY_FILE_FIELDS, context=f"{context}[{index}]") + name = _require_string(raw["name"], context=f"{context}[{index}].name") + if name in names: + raise ValueError(f"{context} contains duplicate file name {name!r}") + names.add(name) + result.append( + { + "name": name, + "sha256": _require_sha256(raw["sha256"], context=f"{context}[{index}].sha256"), + "size_bytes": _require_int( + raw["size_bytes"], + context=f"{context}[{index}].size_bytes", + minimum=1, + ), + } + ) + return sorted(result, key=lambda item: item["name"]) + + +def _ruler_canonical_id(*, category: str, config: str, configured_length: int, seed: int) -> str: + return ( + f"{RULER_SEQUENCE_NAMESPACE}:" + f"{resolver.RULER_REVISION}:{category}:{config}:" + f"length={configured_length}:seed={seed}:sample=0" + ) + + +def _normalize_ruler_receipt( + value: Mapping[str, Any], + *, + category: str, + config: str, + configured_length: int, + seed: int, +) -> dict[str, Any]: + _require_exact_fields(value, RULER_RECEIPT_FIELDS, context="RULER receipt") + expected = { + "schema": RULER_RECEIPT_SCHEMA, + "source_id": resolver.RULER_SOURCE_ID, + "revision": resolver.RULER_REVISION, + "category": category, + "config": config, + "configured_length": configured_length, + "seed": seed, + "sample_index": 0, + } + for field, expected_value in expected.items(): + if value[field] != expected_value: + raise ValueError(f"RULER receipt {field} drifted") + generator_length = _require_int( + value["generator_reported_length"], + context="RULER receipt generator_reported_length", + minimum=1, + ) + if generator_length > configured_length: + raise ValueError("RULER generator-reported length exceeds configured length") + input_text = _require_string(value["input"], context="RULER receipt input") + answer_prefix = _require_string( + value["answer_prefix"], + context="RULER receipt answer_prefix", + allow_empty=True, + ) + outputs = value["outputs"] + if isinstance(outputs, (str, bytes)) or not isinstance(outputs, Sequence) or not outputs: + raise ValueError("RULER receipt outputs must be a non-empty string array") + normalized_outputs = [_require_string(item, context="RULER receipt output") for item in outputs] + required_count = RULER_REQUIRED_OUTPUT_COUNTS.get(config) + if required_count is not None: + if len(normalized_outputs) != required_count: + raise ValueError( + f"RULER receipt {config} must contain exactly {required_count} required outputs" + ) + if len(set(normalized_outputs)) != len(normalized_outputs): + raise ValueError(f"RULER receipt {config} required outputs must be unique") + auxiliary_files = _normalize_auxiliary_files( + value["auxiliary_files"], context="RULER receipt auxiliary_files" + ) + return { + **expected, + "generator_reported_length": generator_length, + "input": input_text, + "answer_prefix": answer_prefix, + "outputs": normalized_outputs, + "auxiliary_files": auxiliary_files, + } + + +def _ruler_stage_a_target(*, category: str, config: str, outputs: Sequence[str]) -> tuple[str, str]: + """Freeze a teacher-forced target without changing RULER reference semantics.""" + + if category == "question_answering": + if config not in {"qa_1", "qa_2"}: + raise ValueError("RULER QA target received a non-QA configuration") + return outputs[0], "first_pinned_alternative_reference_v1" + if config not in RULER_REQUIRED_OUTPUT_COUNTS: + raise ValueError("RULER required-output target received an unknown configuration") + return ( + RULER_REQUIRED_OUTPUT_SEPARATOR.join(outputs), + "all_required_outputs_comma_space_v1", + ) + + +def _capture_ruler( + source: CaptureSource, + *, + phase: str, + tokenizer: Any, + tokenizer_manifest_sha256: str, +) -> tuple[list[dict[str, Any]], str, str]: + generator_manifest = _ruler_generator_manifest(source.ruler_generator_files()) + schedule = ( + resolver.RULER_CALIBRATION_SCHEDULE + if phase == "calibration" + else resolver.RULER_STAGE_A_SCHEDULE + ) + records: list[dict[str, Any]] = [] + auxiliary_manifests: list[dict[str, Any]] = [] + selected_identities: list[dict[str, Any]] = [] + for category, config, configured_length, seed in schedule: + receipt = _normalize_ruler_receipt( + source.ruler_receipt( + category=category, + config=config, + configured_length=configured_length, + seed=seed, + ), + category=category, + config=config, + configured_length=configured_length, + seed=seed, + ) + receipt_hash = sha256_bytes(canonical_json_bytes(receipt)) + prompt = receipt["input"] + receipt["answer_prefix"] + prompt_ids = _encode(tokenizer, prompt, add_special_tokens=False) + if phase == "calibration": + target_text = "" + target_semantics = None + target_ids = () + else: + target_text, target_semantics = _ruler_stage_a_target( + category=category, + config=config, + outputs=receipt["outputs"], + ) + target_ids = _encode(tokenizer, target_text, add_special_tokens=False) + if not prompt_ids: + raise ValueError("RULER receipt produced an empty prompt") + if phase == "stage_a" and not target_ids: + raise ValueError("Stage-A RULER receipt produced an empty answer") + if len(prompt_ids) + len(target_ids) > configured_length: + raise ValueError("RULER actual sequence exceeds configured length") + if receipt["generator_reported_length"] < len(prompt_ids): + raise ValueError("RULER generator receipt is shorter than the tokenized prompt") + canonical_id = _ruler_canonical_id( + category=category, + config=config, + configured_length=configured_length, + seed=seed, + ) + records.append( + _base_record( + phase=phase, + family="ruler", + canonical_id=canonical_id, + config=config, + seed=seed, + configured_length=configured_length, + ruler_category=category, + generator_receipt_sha256=receipt_hash, + source_payload=receipt, + formatted_payload={ + "prompt": prompt, + "target": target_text, + "target_semantics": target_semantics, + "official_output_indices": ( + None + if phase == "calibration" + else ( + [0] + if category == "question_answering" + else list(range(len(receipt["outputs"]))) + ) + ), + }, + prompt_ids=prompt_ids, + target_ids=target_ids, + tokenizer_manifest_sha256=tokenizer_manifest_sha256, + ) + ) + auxiliary_manifests.append( + { + "canonical_id": canonical_id, + "files": receipt["auxiliary_files"], + } + ) + selected_identities.append( + { + "canonical_id": canonical_id, + "category": category, + "config": config, + "configured_length": configured_length, + "seed": seed, + } + ) + inventory = { + "revision": resolver.RULER_REVISION, + "configs_by_category": RULER_CONFIGS_BY_CATEGORY, + "calibration_schedule": resolver.RULER_CALIBRATION_SCHEDULE, + "stage_a_schedule": resolver.RULER_STAGE_A_SCHEDULE, + "selected": selected_identities, + } + identity_manifest_hash = sha256_bytes(canonical_json_bytes(inventory)) + formatter = { + "id": "recurquant.ruler-teacher-forced-target.v2", + "capture_version": CAPTURE_VERSION, + "prompt": "input + answer_prefix", + "prompt_add_special_tokens": False, + "stage_a_target": { + "required_reference_tasks": { + "categories": ["retrieval", "multi_hop_tracing", "aggregation"], + "serialization": ( + "comma-space join of every official required output in source order" + ), + "separator": RULER_REQUIRED_OUTPUT_SEPARATOR, + }, + "alternative_reference_tasks": { + "category": "question_answering", + "serialization": "first pinned official alternative in source order", + }, + }, + "target_add_special_tokens": False, + "minimum_stage_a_target_tokens": 2, + "cache_exposed_predictions": "target_length - 1", + "generator_files": generator_manifest, + "auxiliary_receipts": auxiliary_manifests, + } + formatter_hash = sha256_bytes(canonical_json_bytes(formatter)) + return _assign_sha_ranks(records), identity_manifest_hash, formatter_hash + + +def _capture_humaneval( + source: CaptureSource, + *, + phase: str, + tokenizer: Any, + tokenizer_manifest_sha256: str, +) -> tuple[list[dict[str, Any]], str]: + projection = _validate_projection( + source.humaneval_projection(), + context="HumanEval+ task_id projection", + exact_count=164, + ) + identity_manifest_hash = sha256_bytes( + canonical_json_bytes([item.canonical_id for item in projection]) + ) + if phase == "calibration": + return [], identity_manifest_hash + ranked = sorted( + projection, + key=lambda item: ( + resolver.selection_sha256(resolver.HUMANEVAL_AB_NAMESPACE, item.canonical_id), + item.canonical_id, + ), + ) + records: list[dict[str, Any]] = [] + for item in ranked[:4]: + row = source.humaneval_row(offset=item.offset, expected_task_id=item.canonical_id) + task_id = _require_string(row.get("task_id"), context="HumanEval+ task_id") + if task_id != item.canonical_id: + raise ValueError("HumanEval+ row does not match its task_id projection") + prompt = _require_string(row.get("prompt"), context="HumanEval+ prompt") + solution = _require_string( + row.get("canonical_solution"), context="HumanEval+ canonical_solution" + ) + prompt_ids = _encode(tokenizer, prompt, add_special_tokens=True) + target_ids = _encode(tokenizer, solution, add_special_tokens=False)[:128] + if not prompt_ids or not target_ids: + raise ValueError(f"HumanEval+ task {task_id!r} produced an empty token side") + records.append( + _base_record( + phase=phase, + family="humaneval_plus", + canonical_id=task_id, + config="default", + seed=None, + configured_length=None, + ruler_category=None, + generator_receipt_sha256=None, + source_payload=_json_safe(dict(row), context="HumanEval+ row"), + formatted_payload={"prompt": prompt, "target": solution}, + prompt_ids=prompt_ids, + target_ids=target_ids, + tokenizer_manifest_sha256=tokenizer_manifest_sha256, + ) + ) + return _assign_sha_ranks(records), identity_manifest_hash + + +def _dataset_contracts( + *, + mbpp_manifest_hash: str, + pg19_manifest_hash: str, + pg19_split: str, + ruler_manifest_hash: str, + ruler_formatter_hash: str, + humaneval_manifest_hash: str, +) -> list[dict[str, Any]]: + return [ + { + "key": "mbpp", + "dataset_id": resolver.MBPP_DATASET_ID, + "config": resolver.MBPP_CONFIG, + "revision": resolver.MBPP_REVISION, + "split": "train", + "canonical_id_field": "task_id", + "canonical_id_manifest_sha256": mbpp_manifest_hash, + "formatter_id": MBPP_FORMATTER_SPEC["id"], + "formatter_sha256": sha256_bytes(canonical_json_bytes(MBPP_FORMATTER_SPEC)), + }, + { + "key": "pg19", + "dataset_id": resolver.PG19_DATASET_ID, + "config": "default", + "revision": resolver.PG19_REVISION, + "split": pg19_split, + "canonical_id_field": "url", + "canonical_id_manifest_sha256": pg19_manifest_hash, + "formatter_id": PG19_FORMATTER_SPEC["id"], + "formatter_sha256": sha256_bytes(canonical_json_bytes(PG19_FORMATTER_SPEC)), + }, + { + "key": "ruler", + "dataset_id": resolver.RULER_SOURCE_ID, + "config": "official-generator", + "revision": resolver.RULER_REVISION, + "split": "generated", + "canonical_id_field": "configuration_id", + "canonical_id_manifest_sha256": ruler_manifest_hash, + "formatter_id": "recurquant.ruler-official-generated-record.v1", + "formatter_sha256": ruler_formatter_hash, + }, + { + "key": "humaneval_plus", + "dataset_id": resolver.HUMANEVAL_PLUS_DATASET_ID, + "config": "default", + "revision": resolver.HUMANEVAL_PLUS_REVISION, + "split": "test", + "canonical_id_field": "task_id", + "canonical_id_manifest_sha256": humaneval_manifest_hash, + "formatter_id": HUMANEVAL_FORMATTER_SPEC["id"], + "formatter_sha256": sha256_bytes(canonical_json_bytes(HUMANEVAL_FORMATTER_SPEC)), + }, + ] + + +def _normalize_calibration_binding(value: object) -> dict[str, str]: + if not isinstance(value, bytes): + raise ValueError("Stage-A calibration binding must be a verified artifact byte string") + verified = resolver.deserialize_stage_a_calibration_binding_artifact(value) + return dict(verified.binding) + + +def capture_identity_input( + *, + phase: str, + source: CaptureSource, + calibration_binding: bytes | None = None, +) -> dict[str, Any]: + """Capture one deterministic calibration or Stage-A resolver input.""" + + if phase in resolver.PROTECTED_STAGES: + raise PermissionError(f"{phase} is protected; capture v2 refuses it before source access") + if phase not in resolver.ALLOWED_PHASES: + raise ValueError(f"unsupported identity phase: {phase!r}") + if phase == "stage_a" and calibration_binding is None: + raise ValueError("Stage A requires a frozen calibration binding") + if phase == "calibration" and calibration_binding is not None: + raise ValueError("calibration capture forbids a Stage-A binding") + + before = _validate_heads(source.source_heads(), context="pre-capture") + material = source.tokenizer_material() + tokenizer_contract, tokenizer_manifest_hash = _tokenizer_contract(material) + mbpp_records, mbpp_manifest_hash = _capture_mbpp( + source, + phase=phase, + tokenizer=material.tokenizer, + tokenizer_manifest_sha256=tokenizer_manifest_hash, + ) + pg19_records, pg19_manifest_hash, pg19_split = _capture_pg19( + source, + phase=phase, + tokenizer=material.tokenizer, + tokenizer_manifest_sha256=tokenizer_manifest_hash, + ) + ruler_records, ruler_manifest_hash, ruler_formatter_hash = _capture_ruler( + source, + phase=phase, + tokenizer=material.tokenizer, + tokenizer_manifest_sha256=tokenizer_manifest_hash, + ) + humaneval_records, humaneval_manifest_hash = _capture_humaneval( + source, + phase=phase, + tokenizer=material.tokenizer, + tokenizer_manifest_sha256=tokenizer_manifest_hash, + ) + after = _validate_heads(source.source_heads(), context="post-capture") + if after != before: # Defensive even though both were pinned. + raise ValueError("source HEAD changed during capture") + + result: dict[str, Any] = { + "schema": resolver.INPUT_SCHEMA, + "phase": phase, + "datasets": _dataset_contracts( + mbpp_manifest_hash=mbpp_manifest_hash, + pg19_manifest_hash=pg19_manifest_hash, + pg19_split=pg19_split, + ruler_manifest_hash=ruler_manifest_hash, + ruler_formatter_hash=ruler_formatter_hash, + humaneval_manifest_hash=humaneval_manifest_hash, + ), + "tokenizer": tokenizer_contract, + "records": [ + *mbpp_records, + *pg19_records, + *ruler_records, + *humaneval_records, + ], + "model_weights_loaded": False, + } + if phase == "stage_a": + result["calibration_binding"] = _normalize_calibration_binding(calibration_binding) + expected_revisions = dict(resolver.FROZEN_DATASET_REVISIONS) + resolver.build_candidate( + result, + expected_revisions=expected_revisions, + calibration_binding_artifact=calibration_binding, + ) + return result + + +def atomic_write_no_overwrite(path: Path, payload: bytes) -> None: + """Atomically publish *payload* while refusing an existing destination.""" + + resolved = path.resolve() + resolved.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{resolved.name}.", suffix=".tmp", dir=resolved.parent + ) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + try: + os.link(temporary, resolved) + except FileExistsError as error: + raise FileExistsError(f"refusing to overwrite existing capture: {resolved}") from error + temporary.unlink() + finally: + if temporary.exists(): + temporary.unlink() + + +class LiveCaptureSource: + """Pinned, read-only network source; RULER generated rows come from receipts.""" + + def __init__(self, *, cache_dir: Path, ruler_receipt_dir: Path) -> None: + self.cache_dir = cache_dir.resolve() + self.ruler_receipt_dir = ruler_receipt_dir.resolve() + + @staticmethod + def _github_head(repo_id: str) -> str: + repo_request = urllib.request.Request( + f"https://api.github.com/repos/{repo_id}", + headers={ + "Accept": "application/vnd.github+json", + "User-Agent": "RecurQuant-Experiment-013-identity-capture", + }, + ) + try: + with urllib.request.urlopen(repo_request, timeout=30) as response: + metadata = json.load(response) + branch = _require_string( + metadata.get("default_branch"), context=f"{repo_id} default branch" + ) + commit_request = urllib.request.Request( + f"https://api.github.com/repos/{repo_id}/commits/" + f"{urllib.parse.quote(branch, safe='')}", + headers={ + "Accept": "application/vnd.github+json", + "User-Agent": "RecurQuant-Experiment-013-identity-capture", + }, + ) + with urllib.request.urlopen(commit_request, timeout=30) as response: + commit = json.load(response) + except (OSError, urllib.error.HTTPError, json.JSONDecodeError) as error: + raise RuntimeError(f"cannot resolve GitHub HEAD for {repo_id}") from error + return _require_string(commit.get("sha"), context=f"{repo_id} HEAD") + + def source_heads(self) -> Mapping[str, str]: + try: + from huggingface_hub import HfApi + except ModuleNotFoundError as error: # pragma: no cover - dependency guard + raise RuntimeError("live capture requires huggingface-hub") from error + api = HfApi() + return { + "primary_model": str(api.model_info(resolver.PRIMARY_MODEL_ID).sha), + "mbpp": str(api.dataset_info(resolver.MBPP_DATASET_ID).sha), + "pg19": str(api.dataset_info(resolver.PG19_DATASET_ID).sha), + "ruler": self._github_head(resolver.RULER_SOURCE_ID), + "humaneval_plus": str(api.dataset_info(resolver.HUMANEVAL_PLUS_DATASET_ID).sha), + "evalplus": self._github_head(resolver.EVALPLUS_SOURCE_ID), + } + + def tokenizer_material(self) -> TokenizerMaterial: + try: + from huggingface_hub import HfApi, hf_hub_download + from transformers import AutoTokenizer + except ModuleNotFoundError as error: # pragma: no cover - dependency guard + raise RuntimeError("live capture requires huggingface-hub and Transformers") from error + api = HfApi() + available = set( + api.list_repo_files(resolver.PRIMARY_MODEL_ID, revision=resolver.PRIMARY_MODEL_REVISION) + ) + selected = [name for name in TOKENIZER_ASSET_NAMES if name in available] + if "tokenizer.json" not in selected or "tokenizer_config.json" not in selected: + raise ValueError("pinned model revision lacks mandatory tokenizer assets") + if any(FORBIDDEN_MODEL_FILE_RE.search(name) for name in selected): + raise ValueError("tokenizer allow-list unexpectedly includes a model file") + paths = [ + Path( + hf_hub_download( + repo_id=resolver.PRIMARY_MODEL_ID, + filename=name, + revision=resolver.PRIMARY_MODEL_REVISION, + cache_dir=self.cache_dir, + ) + ) + for name in selected + ] + files = {path.name: path.read_bytes() for path in paths} + if set(files) != set(selected): + raise ValueError("downloaded tokenizer file inventory drifted") + # A shared Hub snapshot can contain files fetched by an earlier run. + # Load only from a new directory populated with the exact authenticated + # allow-list so no unbound sibling can affect tokenizer construction. + with tempfile.TemporaryDirectory(prefix="recurquant-exp013-tokenizer-") as temporary: + isolated = Path(temporary) + for name, data in files.items(): + (isolated / name).write_bytes(data) + tokenizer = AutoTokenizer.from_pretrained( + isolated, + local_files_only=True, + trust_remote_code=False, + ) + isolated_inventory = { + path.relative_to(isolated).as_posix() + for path in isolated.rglob("*") + if path.is_file() + } + if isolated_inventory != set(selected): + raise ValueError("tokenizer construction changed the isolated file inventory") + return TokenizerMaterial( + tokenizer=tokenizer, + tokenizer_class=tokenizer.__class__.__name__, + transformers_version=importlib.metadata.version("transformers"), + files=files, + model_weights_loaded=False, + ) + + def mbpp_train_rows(self) -> Sequence[Mapping[str, Any]]: + try: + from datasets import load_dataset + except ModuleNotFoundError as error: # pragma: no cover - dependency guard + raise RuntimeError("live capture requires the datasets extra") from error + rows = load_dataset( + resolver.MBPP_DATASET_ID, + resolver.MBPP_CONFIG, + revision=resolver.MBPP_REVISION, + split="train", + streaming=True, + ) + return tuple(dict(row) for row in rows) + + @staticmethod + def _active_parquet_aliases( + *, dataset_id: str, revision: str, config: str, split: str + ) -> tuple[str, ...]: + """Resolve only Dataset Viewer's active parquet conversion at *revision*. + + Dataset repositories can retain stale or sibling parquet files that are + not members of the active builder split. The `/parquet` manifest is the + authority for the active conversion; its `x-revision` must bind it to the + pinned source commit before any alias is opened. + """ + + query = urllib.parse.urlencode({"dataset": dataset_id}) + request = urllib.request.Request( + f"https://datasets-server.huggingface.co/parquet?{query}", + headers={"User-Agent": "RecurQuant-Experiment-013-identity-capture"}, + ) + try: + with urllib.request.urlopen(request, timeout=60) as response: + source_revision = response.headers.get("x-revision") + payload = json.load(response) + except (OSError, urllib.error.HTTPError, json.JSONDecodeError) as error: + raise RuntimeError( + f"cannot resolve active parquet manifest for {dataset_id}" + ) from error + if source_revision != revision: + raise ValueError("Dataset Viewer parquet x-revision is not pinned") + raw_entries = payload.get("parquet_files") + if not isinstance(raw_entries, list): + raise ValueError("Dataset Viewer parquet manifest is malformed") + aliases: list[str] = [] + names: set[str] = set() + for raw in raw_entries: + if not isinstance(raw, Mapping): + raise ValueError("Dataset Viewer parquet entry is malformed") + if ( + raw.get("dataset") != dataset_id + or raw.get("config") != config + or raw.get("split") != split + ): + continue + filename = _require_string(raw.get("filename"), context="active parquet filename") + if Path(filename).name != filename or filename in names: + raise ValueError("active parquet filenames must be unique basenames") + _require_int(raw.get("size"), context="active parquet size", minimum=1) + url = _require_string(raw.get("url"), context="active parquet URL") + expected_prefix = ( + f"https://huggingface.co/datasets/{dataset_id}/resolve/refs%2Fconvert%2Fparquet/" + ) + if not url.startswith(expected_prefix): + raise ValueError("active parquet URL does not match its alias identity") + relative = url.removeprefix(expected_prefix) + parts = relative.split("/") + if ( + len(parts) != 3 + or parts[0] != config + or parts[1] not in {split, f"partial-{split}"} + or parts[2] != filename + ): + raise ValueError("active parquet URL does not match its alias identity") + names.add(filename) + aliases.append(f"datasets/{dataset_id}@~parquet/{relative}") + if not aliases: + raise ValueError(f"no active {dataset_id} {config}/{split} parquet files") + return tuple(sorted(aliases)) + + @staticmethod + def _parquet_projection( + *, dataset_id: str, revision: str, config: str, split: str, column: str + ) -> tuple[ProjectionRow, ...]: + try: + import pyarrow.parquet as pq + from huggingface_hub import HfFileSystem + except ModuleNotFoundError as error: # pragma: no cover - dependency guard + raise RuntimeError("ID-only projection requires pyarrow and huggingface-hub") from error + aliases = LiveCaptureSource._active_parquet_aliases( + dataset_id=dataset_id, + revision=revision, + config=config, + split=split, + ) + + def read_column(alias: str) -> list[Any]: + fs = HfFileSystem() + with fs.open(alias, "rb") as handle: + table = pq.ParquetFile(handle).read(columns=[column]) + return table.column(column).to_pylist() + + worker_count = min(4, len(aliases)) + with concurrent.futures.ThreadPoolExecutor(max_workers=worker_count) as executor: + shards = tuple(executor.map(read_column, aliases)) + result: list[ProjectionRow] = [] + offset = 0 + for values in shards: + for value in values: + result.append( + ProjectionRow( + _require_string(value, context=f"{dataset_id} {column}"), + offset, + ) + ) + offset += 1 + return tuple(result) + + @staticmethod + def _viewer_row( + *, + dataset_id: str, + config: str, + split: str, + offset: int, + expected_revision: str, + ) -> Mapping[str, Any]: + query = urllib.parse.urlencode( + { + "dataset": dataset_id, + "config": config, + "split": split, + "offset": offset, + "length": 1, + } + ) + request = urllib.request.Request( + f"https://datasets-server.huggingface.co/rows?{query}", + headers={"User-Agent": "RecurQuant-Experiment-013-identity-capture"}, + ) + try: + with urllib.request.urlopen(request, timeout=60) as response: + revision = response.headers.get("x-revision") + payload = json.load(response) + except (OSError, urllib.error.HTTPError, json.JSONDecodeError) as error: + raise RuntimeError(f"Dataset Viewer row request failed for {dataset_id}") from error + if revision != expected_revision: + raise ValueError("Dataset Viewer x-revision does not match the pinned commit") + rows = payload.get("rows") + if not isinstance(rows, list) or len(rows) != 1: + raise ValueError("Dataset Viewer did not return exactly one row") + item = rows[0] + if not isinstance(item, Mapping) or item.get("row_idx") != offset: + raise ValueError("Dataset Viewer row offset drifted") + truncated = item.get("truncated_cells") + if truncated not in (None, []): + raise ValueError("Dataset Viewer truncated a selected source row") + row = item.get("row") + if not isinstance(row, Mapping): + raise ValueError("Dataset Viewer selected row is malformed") + return dict(row) + + def pg19_projection(self, split: str) -> Sequence[ProjectionRow]: + return self._parquet_projection( + dataset_id=resolver.PG19_DATASET_ID, + revision=resolver.PG19_REVISION, + config="default", + split=split, + column="url", + ) + + def pg19_row(self, split: str, *, offset: int, expected_url: str) -> Mapping[str, Any]: + row = self._viewer_row( + dataset_id=resolver.PG19_DATASET_ID, + config="default", + split=split, + offset=offset, + expected_revision=resolver.PG19_REVISION, + ) + if row.get("url") != expected_url: + raise ValueError("Dataset Viewer PG19 URL does not match the pinned projection") + return row + + def ruler_generator_files(self) -> Mapping[str, bytes]: + base = ( + "https://raw.githubusercontent.com/" + f"{resolver.RULER_SOURCE_ID}/{resolver.RULER_REVISION}/" + ) + files: dict[str, bytes] = {} + for path in RULER_GENERATOR_GIT_BLOBS: + request = urllib.request.Request( + base + path, + headers={"User-Agent": "RecurQuant-Experiment-013-identity-capture"}, + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + files[path] = response.read() + except (OSError, urllib.error.HTTPError) as error: + raise RuntimeError(f"cannot fetch pinned RULER source file {path}") from error + return files + + @staticmethod + def _receipt_filename(*, category: str, config: str, configured_length: int, seed: int) -> str: + return ruler_receipt_filename( + category=category, + config=config, + configured_length=configured_length, + seed=seed, + ) + + def ruler_receipt( + self, *, category: str, config: str, configured_length: int, seed: int + ) -> Mapping[str, Any]: + path = self.ruler_receipt_dir / self._receipt_filename( + category=category, + config=config, + configured_length=configured_length, + seed=seed, + ) + if not path.is_file(): + raise FileNotFoundError( + f"missing audited RULER receipt; generation is intentionally separate: {path}" + ) + return _strict_json(path.read_bytes(), context=f"RULER receipt {path.name}") + + def humaneval_projection(self) -> Sequence[ProjectionRow]: + return self._parquet_projection( + dataset_id=resolver.HUMANEVAL_PLUS_DATASET_ID, + revision=resolver.HUMANEVAL_PLUS_REVISION, + config="default", + split="test", + column="task_id", + ) + + def humaneval_row(self, *, offset: int, expected_task_id: str) -> Mapping[str, Any]: + row = self._viewer_row( + dataset_id=resolver.HUMANEVAL_PLUS_DATASET_ID, + config="default", + split="test", + offset=offset, + expected_revision=resolver.HUMANEVAL_PLUS_REVISION, + ) + if row.get("task_id") != expected_task_id: + raise ValueError("Dataset Viewer HumanEval+ ID does not match projection") + return row + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Capture a calibration or Stage-A Experiment 013 identity input. " + "No model weights are requested or loaded." + ) + ) + parser.add_argument( + "--phase", + required=True, + choices=("calibration", "stage_a", "stage_b", "stage_c"), + ) + parser.add_argument("--output", type=Path) + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--cache-dir", type=Path, default=Path(".cache/exp013-identity")) + parser.add_argument("--ruler-receipt-dir", type=Path) + parser.add_argument("--calibration-binding", type=Path) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + if args.phase in resolver.PROTECTED_STAGES: + raise PermissionError( + f"{args.phase} is protected; capture v2 refuses it before file or source access" + ) + if args.ruler_receipt_dir is None: + raise ValueError("--ruler-receipt-dir is required") + if args.dry_run and args.output is not None: + raise ValueError("--dry-run forbids --output") + if not args.dry_run and args.output is None: + raise ValueError("capture requires --output or --dry-run") + binding: bytes | None = None + if args.phase == "stage_a": + if args.calibration_binding is None: + raise ValueError("Stage A requires --calibration-binding") + binding = args.calibration_binding.read_bytes() + elif args.calibration_binding is not None: + raise ValueError("--calibration-binding is valid only for Stage A") + source = LiveCaptureSource( + cache_dir=args.cache_dir, + ruler_receipt_dir=args.ruler_receipt_dir, + ) + captured = capture_identity_input( + phase=args.phase, + source=source, + calibration_binding=binding, + ) + payload = canonical_json_bytes(captured) + digest = sha256_bytes(payload) + if args.dry_run: + print(digest) + return 0 + atomic_write_no_overwrite(args.output, payload) + print(digest) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except ( + FileExistsError, + FileNotFoundError, + PermissionError, + RuntimeError, + ValueError, + ) as error: + print(f"error: {error}", file=sys.stderr) + raise SystemExit(2) from error diff --git a/scripts/resolve_static_q468_identity.py b/scripts/resolve_static_q468_identity.py index aa65996..fa5f68f 100644 --- a/scripts/resolve_static_q468_identity.py +++ b/scripts/resolve_static_q468_identity.py @@ -15,6 +15,8 @@ from __future__ import annotations import argparse +import base64 +import binascii import hashlib import json import os @@ -27,11 +29,11 @@ from pathlib import Path from typing import Any, Final -INPUT_SCHEMA: Final = "recurquant.experiment013.identity-input.v1" -CANDIDATE_SCHEMA: Final = "recurquant.experiment013.identity-candidate.v1" -FROZEN_SCHEMA: Final = "recurquant.experiment013.identity-frozen.v1" +INPUT_SCHEMA: Final = "recurquant.experiment013.identity-input.v2" +CANDIDATE_SCHEMA: Final = "recurquant.experiment013.identity-candidate.v2" +FROZEN_SCHEMA: Final = "recurquant.experiment013.identity-frozen.v2" ARTIFACT_KIND: Final = "recurquant_static_rht_q468_identity" -RESOLVER_VERSION: Final = 1 +RESOLVER_VERSION: Final = 2 PRIMARY_MODEL_ID: Final = "Qwen/Qwen3.5-0.8B-Base" PRIMARY_MODEL_REVISION: Final = "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68" @@ -67,25 +69,16 @@ } PG19_TRAIN_NAMESPACE: Final = "recurquant.experiment013.pg19.train.v1\0" -PG19_VALIDATION_NAMESPACE: Final = ( - "recurquant.experiment013.pg19.validation.v1\0" -) +PG19_VALIDATION_NAMESPACE: Final = "recurquant.experiment013.pg19.validation.v1\0" PG19_TEST_NAMESPACE: Final = "recurquant.experiment013.pg19.test.v1\0" -HUMANEVAL_AB_NAMESPACE: Final = ( - "recurquant.experiment013.humaneval-plus.stage-a-b.v1\0" -) -HUMANEVAL_C_NAMESPACE: Final = ( - "recurquant.experiment013.humaneval-plus.stage-c.v1\0" -) -CALIBRATION_SPLIT_NAMESPACE: Final = ( - "recurquant.experiment013.calibration-split.v1\0" -) +HUMANEVAL_AB_NAMESPACE: Final = "recurquant.experiment013.humaneval-plus.stage-a-b.v1\0" +HUMANEVAL_C_NAMESPACE: Final = "recurquant.experiment013.humaneval-plus.stage-c.v1\0" +CALIBRATION_SPLIT_NAMESPACE: Final = "recurquant.experiment013.calibration-split.v1\0" +IDENTITY_RECORD_NAMESPACE: Final = "recurquant.experiment013.identity-record.v1\0" RULER_CALIBRATION_SELECTION_NAMESPACE: Final = ( "recurquant.experiment013.ruler.calibration-sequence.v1\0" ) -RULER_STAGE_A_SELECTION_NAMESPACE: Final = ( - "recurquant.experiment013.ruler.stage-a-sequence.v1\0" -) +RULER_STAGE_A_SELECTION_NAMESPACE: Final = "recurquant.experiment013.ruler.stage-a-sequence.v1\0" RULER_CATEGORIES: Final = ( "retrieval", "multi_hop_tracing", @@ -174,23 +167,67 @@ "formatted_content_sha256", "prompt_token_ids_sha256", "target_token_ids_sha256", + "sequence_token_ids_sha256", "tokenizer_manifest_sha256", "token_span", "anchor_manifest_sha256", + "identity_record_sha256", } ) +IDENTITY_RECORD_PAYLOAD_FIELDS: Final = RECORD_FIELDS - {"identity_record_sha256"} TOKEN_SPAN_FIELDS: Final = frozenset( - {"prefill_start", "prefill_stop", "scored_start", "scored_stop"} + { + "prefill_start", + "prefill_stop", + "scored_start", + "scored_stop", + "cache_exposed_start", + "cache_exposed_stop", + } ) CALIBRATION_BINDING_FIELDS: Final = frozenset( { - "identity_file_sha256", - "canonical_identity_sha256", - "static_k29334_code_map_sha256", - "static_k27030_code_map_sha256", - "split_half_policy_manifest_sha256", + "calibration_identity_file_sha256", + "calibration_score_artifact_file_sha256", + "split_half_stability_artifact_file_sha256", + "static_k27030_policy_file_sha256", + "static_k29334_policy_file_sha256", + } +) +FROZEN_EVIDENCE_FIELDS: Final = frozenset( + { + "schema_version", + "artifact_kind", + "identity_schema", + "resolver_version", + "status", + "phase", + "identity_only", + "claim_boundary", + "source_manifest_sha256", + "model_contracts", + "datasets", + "upstream_tool_contracts", + "tokenizer", + "records", + "record_count", + "content_manifest_sha256", + "selection", + "calibration_split_half", + "calibration_binding", + "protected_identity", + "promotion_required", + "promotion", } ) +CANDIDATE_EVIDENCE_FIELDS: Final = FROZEN_EVIDENCE_FIELDS - {"promotion"} +FROZEN_RECORD_FIELDS: Final = RECORD_FIELDS | { + "anchor_positions", + "anchor_positions_sha256", +} +STAGE_A_BINDING_ARTIFACT_KIND: Final = "recurquant_experiment013_stage_a_calibration_binding" +STAGE_A_BINDING_ARTIFACT_SCHEMA_VERSION: Final = 2 +STAGE_A_BINDING_ARTIFACT_REVISION: Final = "experiment-013-stage-a-calibration-binding-v2" def canonical_json_bytes(value: object) -> bytes: @@ -234,9 +271,7 @@ def require_exact_revision(value: object, *, context: str) -> str: or value != value.lower() or not all(character in string.hexdigits for character in value) ): - raise ValueError( - f"{context} must be an immutable lowercase 40- or 64-hex revision" - ) + raise ValueError(f"{context} must be an immutable lowercase 40- or 64-hex revision") return value @@ -276,6 +311,57 @@ def require_exact_fields( raise ValueError(f"{context} fields drifted; missing={missing}, extra={extra}") +def sequence_token_ids_sha256(token_ids: Sequence[int]) -> str: + """Hash the exact ordered token-ID sequence using resolver canonical JSON.""" + + if isinstance(token_ids, (str, bytes, bytearray)) or not isinstance(token_ids, Sequence): + raise ValueError("sequence token IDs must be an integer sequence") + normalized = [ + require_int(token_id, context=f"sequence token IDs[{index}]") + for index, token_id in enumerate(token_ids) + ] + return sha256_bytes(canonical_json_bytes(normalized)) + + +def identity_anchor_manifest_sha256( + *, + canonical_id: str, + sequence_length: int, + sequence_token_ids_sha256_value: str, + token_span: Mapping[str, Any], +) -> str: + """Recompute the exact capture anchor-manifest commitment.""" + + normalized_id = require_string(canonical_id, context="anchor canonical_id") + length = require_int(sequence_length, context="anchor sequence_length", minimum=1) + token_hash = require_sha256( + sequence_token_ids_sha256_value, + context="anchor sequence_token_ids_sha256", + ) + require_exact_fields(token_span, TOKEN_SPAN_FIELDS, context="anchor token_span") + normalized_span = { + name: require_int(token_span[name], context=f"anchor token_span.{name}") + for name in sorted(TOKEN_SPAN_FIELDS) + } + manifest = { + "canonical_id": normalized_id, + "positions": list(anchor_positions(length)), + "sequence_token_ids_sha256": token_hash, + "token_span": normalized_span, + } + return sha256_bytes(canonical_json_bytes(manifest)) + + +def identity_record_sha256(record: Mapping[str, Any]) -> str: + """Hash the exact capture record payload, excluding its self-hash.""" + + missing = IDENTITY_RECORD_PAYLOAD_FIELDS - set(record) + if missing: + raise ValueError(f"identity record payload is missing fields: {sorted(missing)}") + payload = {name: record[name] for name in sorted(IDENTITY_RECORD_PAYLOAD_FIELDS)} + return sha256_bytes(IDENTITY_RECORD_NAMESPACE.encode("utf-8") + canonical_json_bytes(payload)) + + def selection_sha256(namespace: str, canonical_id: str) -> str: return sha256_bytes(namespace.encode("utf-8") + canonical_id.encode("utf-8")) @@ -322,9 +408,7 @@ def mbpp_calibration_identity() -> tuple[tuple[str, ...], str]: return selected, population_hash -def _validate_sha_rank_order( - rows: Sequence[Mapping[str, Any]], *, context: str -) -> None: +def _validate_sha_rank_order(rows: Sequence[Mapping[str, Any]], *, context: str) -> None: ranked = sorted( rows, key=lambda row: (str(row["selection_sha256"]), str(row["canonical_id"])), @@ -366,9 +450,7 @@ def _validate_dataset_contracts( key = require_string(item["key"], context=f"datasets[{index}].key") if key not in DATASET_KEYS or key in normalized: raise ValueError(f"dataset key is unknown or duplicated: {key}") - revision = require_exact_revision( - item["revision"], context=f"datasets[{index}].revision" - ) + revision = require_exact_revision(item["revision"], context=f"datasets[{index}].revision") if revision != expected_revisions[key]: raise ValueError(f"{key} dataset revision does not match the CLI contract") contract = { @@ -380,9 +462,7 @@ def _validate_dataset_contracts( item["config"], context=f"datasets[{index}].config", allow_empty=True ), "revision": revision, - "split": require_string( - item["split"], context=f"datasets[{index}].split" - ), + "split": require_string(item["split"], context=f"datasets[{index}].split"), "canonical_id_field": require_string( item["canonical_id_field"], context=f"datasets[{index}].canonical_id_field", @@ -401,12 +481,10 @@ def _validate_dataset_contracts( } if contract["canonical_id_field"] != FROZEN_CANONICAL_ID_FIELDS[key]: raise ValueError( - f"{key} canonical ID field must be " - f"{FROZEN_CANONICAL_ID_FIELDS[key]!r}" + f"{key} canonical ID field must be {FROZEN_CANONICAL_ID_FIELDS[key]!r}" ) if key == "mbpp" and ( - contract["dataset_id"] != MBPP_DATASET_ID - or contract["config"] != MBPP_CONFIG + contract["dataset_id"] != MBPP_DATASET_ID or contract["config"] != MBPP_CONFIG ): raise ValueError("MBPP identity must match the frozen v0.2 source") if key == "mbpp": @@ -442,9 +520,7 @@ def _validate_tokenizer(value: object) -> dict[str, Any]: names: set[str] = set() for index, raw in enumerate(raw_files): item = require_mapping(raw, context=f"tokenizer.files[{index}]") - require_exact_fields( - item, TOKENIZER_FILE_FIELDS, context=f"tokenizer.files[{index}]" - ) + require_exact_fields(item, TOKENIZER_FILE_FIELDS, context=f"tokenizer.files[{index}]") name = require_string(item["name"], context=f"tokenizer.files[{index}].name") if Path(name).name != name or name in names: raise ValueError("tokenizer file names must be unique basenames") @@ -502,17 +578,11 @@ def _normalize_record( family = require_string(item["family"], context=f"records[{index}].family") if family not in DATASET_KEYS: raise ValueError(f"records[{index}].family is unknown") - canonical_id = require_string( - item["canonical_id"], context=f"records[{index}].canonical_id" - ) - config = require_string( - item["config"], context=f"records[{index}].config", allow_empty=True - ) + canonical_id = require_string(item["canonical_id"], context=f"records[{index}].canonical_id") + config = require_string(item["config"], context=f"records[{index}].config", allow_empty=True) rank = require_int(item["selection_rank"], context=f"records[{index}].selection_rank") seed_value = item["seed"] - seed = None if seed_value is None else require_int( - seed_value, context=f"records[{index}].seed" - ) + seed = None if seed_value is None else require_int(seed_value, context=f"records[{index}].seed") configured_value = item["configured_length"] configured_length = ( None @@ -544,20 +614,15 @@ def _normalize_record( if family == "ruler": expected_category = RULER_CONFIG_CATEGORY.get(config) if expected_category is None or ruler_category != expected_category: - raise ValueError( - f"records[{index}] RULER config/category binding drifted" - ) + raise ValueError(f"records[{index}] RULER config/category binding drifted") if configured_length is None or generator_receipt_sha256 is None: raise ValueError( f"records[{index}] RULER configured length and generator receipt are required" ) elif any( - value is not None - for value in (configured_length, ruler_category, generator_receipt_sha256) + value is not None for value in (configured_length, ruler_category, generator_receipt_sha256) ): - raise ValueError( - f"records[{index}] non-RULER rows cannot carry RULER-only fields" - ) + raise ValueError(f"records[{index}] non-RULER rows cannot carry RULER-only fields") sequence_length = require_int( item["sequence_length"], context=f"records[{index}].sequence_length", @@ -577,7 +642,14 @@ def _normalize_record( require_exact_fields(span, TOKEN_SPAN_FIELDS, context=f"records[{index}].token_span") normalized_span = { name: require_int(span[name], context=f"records[{index}].token_span.{name}") - for name in ("prefill_start", "prefill_stop", "scored_start", "scored_stop") + for name in ( + "prefill_start", + "prefill_stop", + "scored_start", + "scored_stop", + "cache_exposed_start", + "cache_exposed_stop", + ) } if ( normalized_span["prefill_start"] != 0 @@ -587,12 +659,50 @@ def _normalize_record( or normalized_span["scored_stop"] != sequence_length ): raise ValueError(f"records[{index}] token span is not contiguous and canonical") + if phase == "calibration": + if ( + normalized_span["cache_exposed_start"] != normalized_span["scored_stop"] + or normalized_span["cache_exposed_stop"] != normalized_span["scored_stop"] + ): + raise ValueError( + f"records[{index}] calibration cache-exposed prediction span " + "must be empty at continuation stop" + ) + else: + continuation_tokens = normalized_span["scored_stop"] - normalized_span["scored_start"] + if continuation_tokens < 2: + raise ValueError( + f"records[{index}] Stage-A continuation must contain at least two " + "tokens to expose one cache prediction" + ) + if ( + normalized_span["cache_exposed_start"] != normalized_span["scored_start"] + 1 + or normalized_span["cache_exposed_stop"] != normalized_span["scored_stop"] + ): + raise ValueError( + f"records[{index}] Stage-A cache-exposed prediction span must exclude " + "the first continuation token" + ) if configured_length is not None and sequence_length > configured_length: - raise ValueError( - f"records[{index}] actual sequence exceeds the RULER configured length" - ) + raise ValueError(f"records[{index}] actual sequence exceeds the RULER configured length") positions = anchor_positions(sequence_length) - return { + sequence_hash = require_sha256( + item["sequence_token_ids_sha256"], + context=f"records[{index}].sequence_token_ids_sha256", + ) + recorded_anchor_hash = require_sha256( + item["anchor_manifest_sha256"], + context=f"records[{index}].anchor_manifest_sha256", + ) + computed_anchor_hash = identity_anchor_manifest_sha256( + canonical_id=canonical_id, + sequence_length=sequence_length, + sequence_token_ids_sha256_value=sequence_hash, + token_span=normalized_span, + ) + if recorded_anchor_hash != computed_anchor_hash: + raise ValueError(f"records[{index}] anchor manifest SHA-256 drifted") + normalized = { "family": family, "canonical_id": canonical_id, "config": config, @@ -619,15 +729,21 @@ def _normalize_record( item["target_token_ids_sha256"], context=f"records[{index}].target_token_ids_sha256", ), + "sequence_token_ids_sha256": sequence_hash, "tokenizer_manifest_sha256": tokenizer_hash, "token_span": normalized_span, - "anchor_manifest_sha256": require_sha256( - item["anchor_manifest_sha256"], - context=f"records[{index}].anchor_manifest_sha256", + "anchor_manifest_sha256": recorded_anchor_hash, + "identity_record_sha256": require_sha256( + item["identity_record_sha256"], + context=f"records[{index}].identity_record_sha256", ), "anchor_positions": list(positions), "anchor_positions_sha256": sha256_bytes(canonical_json_bytes(positions)), } + computed_record_hash = identity_record_sha256(normalized) + if normalized["identity_record_sha256"] != computed_record_hash: + raise ValueError(f"records[{index}] identity record SHA-256 drifted") + return normalized def _record_sort_key(record: Mapping[str, Any]) -> tuple[Any, ...]: @@ -639,9 +755,7 @@ def _record_sort_key(record: Mapping[str, Any]) -> tuple[Any, ...]: "" if record["ruler_category"] is None else str(record["ruler_category"]), str(record["config"]), -1 if record["seed"] is None else int(record["seed"]), - -1 - if record["configured_length"] is None - else int(record["configured_length"]), + -1 if record["configured_length"] is None else int(record["configured_length"]), int(record["sequence_length"]), ) @@ -655,9 +769,7 @@ def _validate_calibration_records(records: Sequence[Mapping[str, Any]]) -> None: if {family: len(rows) for family, rows in grouped.items()} != expected_counts: raise ValueError("calibration record counts must be MBPP=128, PG19=16, RULER=16") expected_mbpp_ids, _population_hash = mbpp_calibration_identity() - actual_mbpp = sorted( - grouped["mbpp"], key=lambda row: int(row["selection_rank"]) - ) + actual_mbpp = sorted(grouped["mbpp"], key=lambda row: int(row["selection_rank"])) if tuple(str(row["canonical_id"]) for row in actual_mbpp) != expected_mbpp_ids: raise ValueError("MBPP rows do not match the frozen v0.2 calibration identity") if sorted(int(row["selection_rank"]) for row in grouped["pg19"]) != list(range(16)): @@ -672,11 +784,11 @@ def _validate_calibration_records(records: Sequence[Mapping[str, Any]]) -> None: "prefill_stop": 2_304, "scored_start": 2_304, "scored_stop": 2_304, + "cache_exposed_start": 2_304, + "cache_exposed_stop": 2_304, }: raise ValueError("calibration PG19 span must cover exactly 2,304 tokens") - if sorted(int(row["selection_rank"]) for row in grouped["ruler"]) != list( - range(16) - ): + if sorted(int(row["selection_rank"]) for row in grouped["ruler"]) != list(range(16)): raise ValueError("calibration RULER ranks must be exactly 0..15") _validate_sha_rank_order(grouped["ruler"], context="calibration RULER") actual_ruler_schedule = { @@ -702,6 +814,8 @@ def _validate_calibration_records(records: Sequence[Mapping[str, Any]]) -> None: "prefill_stop": row["sequence_length"], "scored_start": row["sequence_length"], "scored_stop": row["sequence_length"], + "cache_exposed_start": row["sequence_length"], + "cache_exposed_stop": row["sequence_length"], }: raise ValueError("calibration RULER must anchor the actual prompt tokens only") for row in grouped["mbpp"]: @@ -737,17 +851,20 @@ def _validate_stage_a_records(records: Sequence[Mapping[str, Any]]) -> None: for row in grouped["pg19"]: span = row["token_span"] if row["seed"] is not None or row["sequence_length"] != 4_224: - raise ValueError("Stage-A PG19 must use 4,096 prefill plus 128 scored tokens") + raise ValueError("Stage-A PG19 must use 4,096 prefill plus 128 continuation tokens") if span != { "prefill_start": 0, "prefill_stop": 4_096, "scored_start": 4_096, "scored_stop": 4_224, + "cache_exposed_start": 4_097, + "cache_exposed_stop": 4_224, }: - raise ValueError("Stage-A PG19 token span drifted") - if sorted(int(row["selection_rank"]) for row in grouped["ruler"]) != list( - range(4) - ): + raise ValueError( + "Stage-A PG19 must bind 128 continuation tokens and exactly " + "127 cache-exposed predictions" + ) + if sorted(int(row["selection_rank"]) for row in grouped["ruler"]) != list(range(4)): raise ValueError("Stage-A RULER ranks must be exactly 0..3") _validate_sha_rank_order(grouped["ruler"], context="Stage-A RULER") actual_ruler_schedule = { @@ -763,13 +880,21 @@ def _validate_stage_a_records(records: Sequence[Mapping[str, Any]]) -> None: raise ValueError("Stage-A RULER rows differ from the frozen category representatives") for row in grouped["ruler"]: span = row["token_span"] - if span["scored_stop"] <= span["scored_start"]: - raise ValueError("Stage-A RULER answer span cannot be empty") + continuation = span["scored_stop"] - span["scored_start"] + exposed = span["cache_exposed_stop"] - span["cache_exposed_start"] + if continuation < 2 or exposed != continuation - 1: + raise ValueError( + "Stage-A RULER continuation must expose target_length - 1 cache predictions" + ) for row in grouped["humaneval_plus"]: span = row["token_span"] - scored = span["scored_stop"] - span["scored_start"] - if row["seed"] is not None or not 1 <= scored <= 128: - raise ValueError("Stage-A HumanEval+ must score 1..128 solution tokens") + continuation = span["scored_stop"] - span["scored_start"] + exposed = span["cache_exposed_stop"] - span["cache_exposed_start"] + if row["seed"] is not None or not 2 <= continuation <= 128 or exposed != continuation - 1: + raise ValueError( + "Stage-A HumanEval+ must bind 2..128 continuation tokens and expose " + "target_length - 1 cache predictions" + ) if row["sequence_length"] != span["scored_stop"]: raise ValueError("Stage-A HumanEval+ sequence length must equal span stop") identities = [ @@ -825,9 +950,7 @@ def _split_half_manifest(records: Sequence[Mapping[str, Any]]) -> dict[str, Any] def _validate_calibration_binding(value: object) -> dict[str, str]: binding = require_mapping(value, context="calibration_binding") - require_exact_fields( - binding, CALIBRATION_BINDING_FIELDS, context="calibration_binding" - ) + require_exact_fields(binding, CALIBRATION_BINDING_FIELDS, context="calibration_binding") return { key: require_sha256(binding[key], context=f"calibration_binding.{key}") for key in sorted(CALIBRATION_BINDING_FIELDS) @@ -835,7 +958,10 @@ def _validate_calibration_binding(value: object) -> dict[str, str]: def build_candidate( - source: Mapping[str, Any], *, expected_revisions: Mapping[str, str] + source: Mapping[str, Any], + *, + expected_revisions: Mapping[str, str], + calibration_binding_artifact: bytes | None = None, ) -> dict[str, Any]: """Validate metadata and return a deterministic candidate artifact.""" @@ -855,10 +981,19 @@ def build_candidate( raise ValueError("identity input schema drifted") if phase not in ALLOWED_PHASES: if phase in PROTECTED_STAGES: - raise PermissionError(f"{phase} is protected and unavailable in resolver v1") + raise PermissionError(f"{phase} is protected and unavailable in resolver v2") raise ValueError(f"unsupported identity phase: {phase!r}") if source["model_weights_loaded"] is not False: raise ValueError("identity resolution must occur before model weights") + expected_calibration_binding: dict[str, str] | None = None + if phase == "stage_a": + if not isinstance(calibration_binding_artifact, bytes): + raise ValueError("Stage A requires a verified calibration binding artifact") + expected_calibration_binding = dict( + deserialize_stage_a_calibration_binding_artifact(calibration_binding_artifact).binding + ) + elif calibration_binding_artifact is not None: + raise ValueError("calibration resolution forbids a Stage-A binding artifact") if set(expected_revisions) != set(DATASET_KEYS): raise ValueError("all four dataset revisions are mandatory") revisions = { @@ -888,11 +1023,13 @@ def build_candidate( _validate_stage_a_records(records) split_half = None calibration_binding = _validate_calibration_binding(source["calibration_binding"]) + if calibration_binding != expected_calibration_binding: + raise ValueError("Stage-A input calibration binding differs from the verified artifact") content_manifest_hash = sha256_bytes(canonical_json_bytes(records)) source_hash = sha256_bytes(canonical_json_bytes(source)) evidence: dict[str, Any] = { - "schema_version": 1, + "schema_version": 2, "artifact_kind": ARTIFACT_KIND, "identity_schema": CANDIDATE_SCHEMA, "resolver_version": RESOLVER_VERSION, @@ -953,32 +1090,145 @@ def validate_candidate_artifact(artifact: Mapping[str, Any]) -> None: if set(artifact) != {"canonical_evidence_sha256", "evidence"}: raise ValueError("candidate wrapper fields drifted") evidence = require_mapping(artifact.get("evidence"), context="candidate evidence") - if artifact.get("canonical_evidence_sha256") != sha256_bytes( - canonical_json_bytes(evidence) - ): + require_exact_fields( + evidence, + CANDIDATE_EVIDENCE_FIELDS, + context="candidate evidence", + ) + if artifact.get("canonical_evidence_sha256") != sha256_bytes(canonical_json_bytes(evidence)): raise ValueError("candidate canonical evidence SHA-256 drifted") - if ( - evidence.get("identity_schema") != CANDIDATE_SCHEMA - or evidence.get("status") != "candidate" - or evidence.get("phase") not in ALLOWED_PHASES - or evidence.get("identity_only") is not True - or evidence.get("claim_boundary") != CLAIM_BOUNDARY - or evidence.get("promotion_required") is not True - ): - raise ValueError("candidate identity contract drifted") - models = require_mapping(evidence.get("model_contracts"), context="model contracts") - if models.get("weights_loaded") is not False: + phase = evidence["phase"] + exact_scalars = { + "schema_version": 2, + "artifact_kind": ARTIFACT_KIND, + "identity_schema": CANDIDATE_SCHEMA, + "resolver_version": RESOLVER_VERSION, + "status": "candidate", + "identity_only": True, + "claim_boundary": CLAIM_BOUNDARY, + "promotion_required": True, + } + for name, expected in exact_scalars.items(): + if evidence[name] != expected: + raise ValueError(f"candidate {name} drifted") + if phase not in ALLOWED_PHASES: + raise ValueError("candidate phase drifted") + require_sha256( + evidence["source_manifest_sha256"], + context="candidate source manifest SHA-256", + ) + + models = require_mapping(evidence["model_contracts"], context="model contracts") + require_exact_fields( + models, + frozenset({"primary", "conditional_scale_check", "weights_loaded"}), + context="model contracts", + ) + primary = require_mapping(models["primary"], context="primary model contract") + require_exact_fields( + primary, + frozenset({"id", "revision"}), + context="primary model contract", + ) + conditional = require_mapping( + models["conditional_scale_check"], + context="conditional model contract", + ) + require_exact_fields( + conditional, + frozenset({"id", "revision", "cold_start_peak_hbm_limit_bytes"}), + context="conditional model contract", + ) + if dict(primary) != {"id": PRIMARY_MODEL_ID, "revision": PRIMARY_MODEL_REVISION}: + raise ValueError("candidate primary model contract drifted") + if dict(conditional) != { + "id": CONDITIONAL_MODEL_ID, + "revision": CONDITIONAL_MODEL_REVISION, + "cold_start_peak_hbm_limit_bytes": 8_053_063_680, + }: + raise ValueError("candidate conditional model contract drifted") + if models["weights_loaded"] is not False: raise ValueError("candidate claims model weights were loaded") - records = require_sequence(evidence.get("records"), context="candidate records") - if evidence.get("record_count") != len(records): + + datasets = _validate_dataset_contracts( + evidence["datasets"], + expected_revisions=FROZEN_DATASET_REVISIONS, + ) + if list(datasets) != evidence["datasets"]: + raise ValueError("candidate dataset contracts are not canonical") + tokenizer = require_mapping(evidence["tokenizer"], context="candidate tokenizer") + require_exact_fields( + tokenizer, + TOKENIZER_FIELDS | {"file_manifest_sha256"}, + context="candidate tokenizer", + ) + normalized_tokenizer = _validate_tokenizer({name: tokenizer[name] for name in TOKENIZER_FIELDS}) + if dict(tokenizer) != normalized_tokenizer: + raise ValueError("candidate tokenizer contract or file manifest drifted") + tokenizer_manifest_sha256 = normalized_tokenizer["file_manifest_sha256"] + + raw_records = require_sequence(evidence["records"], context="candidate records") + records: list[dict[str, Any]] = [] + for index, raw_record in enumerate(raw_records): + record = require_mapping(raw_record, context=f"candidate records[{index}]") + require_exact_fields( + record, + FROZEN_RECORD_FIELDS, + context=f"candidate records[{index}]", + ) + normalized = _normalize_record( + {name: record[name] for name in RECORD_FIELDS}, + index=index, + phase=str(phase), + tokenizer_hash=tokenizer_manifest_sha256, + ) + if dict(record) != normalized: + raise ValueError(f"candidate records[{index}] is not canonical") + records.append(normalized) + if records != sorted(records, key=_record_sort_key): + raise ValueError("candidate records are not in canonical resolver order") + if phase == "calibration": + _validate_calibration_records(records) + expected_record_count = 160 + else: + _validate_stage_a_records(records) + expected_record_count = 12 + if evidence["record_count"] != len(records) or len(records) != expected_record_count: raise ValueError("candidate record count drifted") - if evidence.get("content_manifest_sha256") != sha256_bytes( - canonical_json_bytes(records) - ): + if evidence["content_manifest_sha256"] != sha256_bytes(canonical_json_bytes(records)): raise ValueError("candidate content manifest SHA-256 drifted") - protected = require_mapping( - evidence.get("protected_identity"), context="protected identity" - ) + + expected_selection = { + "pg19_train_namespace": PG19_TRAIN_NAMESPACE, + "pg19_validation_namespace": PG19_VALIDATION_NAMESPACE, + "pg19_test_namespace": PG19_TEST_NAMESPACE, + "humaneval_plus_stage_a_b_namespace": HUMANEVAL_AB_NAMESPACE, + "humaneval_plus_stage_c_namespace": HUMANEVAL_C_NAMESPACE, + } + if evidence["selection"] != expected_selection: + raise ValueError("candidate selection namespaces drifted") + expected_upstream = { + "ruler_generator": {"id": RULER_SOURCE_ID, "revision": RULER_REVISION}, + "evalplus_formatter": { + "id": EVALPLUS_SOURCE_ID, + "revision": EVALPLUS_SOURCE_REVISION, + }, + } + if evidence["upstream_tool_contracts"] != expected_upstream: + raise ValueError("candidate upstream tool contracts drifted") + if phase == "calibration": + if evidence["calibration_binding"] is not None: + raise ValueError("calibration candidate cannot carry a Stage-A binding") + if evidence["calibration_split_half"] != _split_half_manifest(records): + raise ValueError("candidate calibration split assignments drifted") + else: + if evidence["calibration_split_half"] is not None: + raise ValueError("Stage-A candidate cannot carry a calibration split") + binding = _validate_calibration_binding(evidence["calibration_binding"]) + if dict(evidence["calibration_binding"]) != binding: + raise ValueError("candidate calibration binding is not canonical") + + protected = require_mapping(evidence["protected_identity"], context="protected identity") if protected != { "stage_b_read": False, "stage_c_read": False, @@ -988,21 +1238,38 @@ def validate_candidate_artifact(artifact: Mapping[str, Any]) -> None: def promote_candidate( - candidate: Mapping[str, Any], *, candidate_file_sha256: str + candidate: Mapping[str, Any], + *, + candidate_file_sha256: str, + calibration_binding_artifact: bytes | None = None, ) -> dict[str, Any]: """Create a deterministic frozen identity from an authenticated candidate.""" validate_candidate_artifact(candidate) - require_sha256(candidate_file_sha256, context="candidate file SHA-256") + expected_candidate_file_sha256 = require_sha256( + candidate_file_sha256, + context="candidate file SHA-256", + ) + if expected_candidate_file_sha256 != sha256_bytes(canonical_json_bytes(candidate)): + raise ValueError("candidate file bytes are not canonical resolver JSON") + candidate_phase = candidate["evidence"]["phase"] + if candidate_phase == "stage_a": + if not isinstance(calibration_binding_artifact, bytes): + raise ValueError("Stage-A promotion requires a verified calibration binding artifact") + verified_binding = deserialize_stage_a_calibration_binding_artifact( + calibration_binding_artifact + ).binding + if candidate["evidence"]["calibration_binding"] != verified_binding: + raise ValueError("Stage-A candidate differs from the verified calibration binding") + elif calibration_binding_artifact is not None: + raise ValueError("calibration promotion forbids a Stage-A binding artifact") evidence = deepcopy(dict(candidate["evidence"])) evidence["identity_schema"] = FROZEN_SCHEMA evidence["status"] = "frozen" evidence["promotion_required"] = False evidence["promotion"] = { - "candidate_file_sha256": candidate_file_sha256, - "candidate_canonical_evidence_sha256": candidate[ - "canonical_evidence_sha256" - ], + "candidate_file_sha256": expected_candidate_file_sha256, + "candidate_canonical_evidence_sha256": candidate["canonical_evidence_sha256"], "explicit": True, } return { @@ -1011,6 +1278,755 @@ def promote_candidate( } +class FrozenCalibrationIdentityArtifact: + """Strictly verified frozen calibration identity and its binding commitments.""" + + __slots__ = ( + "file_sha256", + "canonical_evidence_sha256", + "records", + "assignment", + "assignment_sha256", + "tokenizer_manifest_sha256", + ) + + def __init__( + self, + *, + file_sha256: str, + canonical_evidence_sha256: str, + records: tuple[dict[str, Any], ...], + assignment: tuple[dict[str, Any], ...], + assignment_sha256: str, + tokenizer_manifest_sha256: str, + ) -> None: + self.file_sha256 = file_sha256 + self.canonical_evidence_sha256 = canonical_evidence_sha256 + self.records = records + self.assignment = assignment + self.assignment_sha256 = assignment_sha256 + self.tokenizer_manifest_sha256 = tokenizer_manifest_sha256 + + +class FrozenStageAIdentityArtifact: + """Strictly verified frozen Stage-A identity and five-file calibration binding.""" + + __slots__ = ( + "file_sha256", + "canonical_evidence_sha256", + "records", + "tokenizer_manifest_sha256", + "calibration_binding", + ) + + def __init__( + self, + *, + file_sha256: str, + canonical_evidence_sha256: str, + records: tuple[dict[str, Any], ...], + tokenizer_manifest_sha256: str, + calibration_binding: dict[str, str], + ) -> None: + self.file_sha256 = file_sha256 + self.canonical_evidence_sha256 = canonical_evidence_sha256 + self.records = records + self.tokenizer_manifest_sha256 = tokenizer_manifest_sha256 + self.calibration_binding = calibration_binding + + +class StageACalibrationBindingArtifact: + """Verified five-field Stage-A binding and authenticated dependency hashes.""" + + __slots__ = ( + "binding", + "dependency_file_sha256", + "canonical_evidence_sha256", + "file_sha256", + ) + + def __init__( + self, + *, + binding: dict[str, str], + dependency_file_sha256: dict[str, str], + canonical_evidence_sha256: str, + file_sha256: str, + ) -> None: + self.binding = binding + self.dependency_file_sha256 = dependency_file_sha256 + self.canonical_evidence_sha256 = canonical_evidence_sha256 + self.file_sha256 = file_sha256 + + +def deserialize_frozen_calibration_identity_artifact( + data: bytes, + *, + expected_file_sha256: str | None = None, +) -> FrozenCalibrationIdentityArtifact: + """Decode and independently recompute the complete frozen calibration identity.""" + + if not isinstance(data, bytes): + raise TypeError("frozen identity artifact must be bytes") + file_sha256 = sha256_bytes(data) + if expected_file_sha256 is not None: + expected = require_sha256( + expected_file_sha256, + context="expected frozen identity file SHA-256", + ) + if file_sha256 != expected: + raise ValueError("frozen identity file SHA-256 mismatch") + root = _json_without_duplicate_keys(data, context="frozen identity artifact") + require_exact_fields( + root, + frozenset({"canonical_evidence_sha256", "evidence"}), + context="frozen identity wrapper", + ) + if canonical_json_bytes(root) != data: + raise ValueError("frozen identity bytes are not canonical resolver JSON") + evidence = require_mapping(root["evidence"], context="frozen identity evidence") + require_exact_fields( + evidence, + FROZEN_EVIDENCE_FIELDS, + context="frozen identity evidence", + ) + canonical_evidence_sha256 = require_sha256( + root["canonical_evidence_sha256"], + context="frozen identity canonical evidence SHA-256", + ) + if canonical_evidence_sha256 != sha256_bytes(canonical_json_bytes(evidence)): + raise ValueError("frozen identity canonical evidence SHA-256 drifted") + exact_scalars = { + "schema_version": 2, + "artifact_kind": ARTIFACT_KIND, + "identity_schema": FROZEN_SCHEMA, + "resolver_version": RESOLVER_VERSION, + "status": "frozen", + "phase": "calibration", + "identity_only": True, + "claim_boundary": CLAIM_BOUNDARY, + "calibration_binding": None, + "promotion_required": False, + } + for name, expected in exact_scalars.items(): + if evidence[name] != expected: + raise ValueError(f"frozen identity {name} drifted") + require_sha256( + evidence["source_manifest_sha256"], + context="frozen identity source manifest SHA-256", + ) + + models = require_mapping(evidence["model_contracts"], context="model contracts") + require_exact_fields( + models, + frozenset({"primary", "conditional_scale_check", "weights_loaded"}), + context="model contracts", + ) + primary = require_mapping(models["primary"], context="primary model contract") + require_exact_fields( + primary, + frozenset({"id", "revision"}), + context="primary model contract", + ) + conditional = require_mapping( + models["conditional_scale_check"], + context="conditional model contract", + ) + require_exact_fields( + conditional, + frozenset({"id", "revision", "cold_start_peak_hbm_limit_bytes"}), + context="conditional model contract", + ) + if primary != {"id": PRIMARY_MODEL_ID, "revision": PRIMARY_MODEL_REVISION}: + raise ValueError("primary model contract drifted") + if conditional != { + "id": CONDITIONAL_MODEL_ID, + "revision": CONDITIONAL_MODEL_REVISION, + "cold_start_peak_hbm_limit_bytes": 8_053_063_680, + }: + raise ValueError("conditional model contract drifted") + if models["weights_loaded"] is not False: + raise ValueError("frozen calibration identity claims model weights were loaded") + + datasets = _validate_dataset_contracts( + evidence["datasets"], + expected_revisions=FROZEN_DATASET_REVISIONS, + ) + if list(datasets) != evidence["datasets"]: + raise ValueError("frozen dataset contracts are not canonical") + tokenizer = require_mapping(evidence["tokenizer"], context="frozen tokenizer") + require_exact_fields( + tokenizer, + TOKENIZER_FIELDS | {"file_manifest_sha256"}, + context="frozen tokenizer", + ) + tokenizer_without_manifest = {name: tokenizer[name] for name in TOKENIZER_FIELDS} + normalized_tokenizer = _validate_tokenizer(tokenizer_without_manifest) + if dict(tokenizer) != normalized_tokenizer: + raise ValueError("frozen tokenizer contract or file manifest drifted") + tokenizer_manifest_sha256 = normalized_tokenizer["file_manifest_sha256"] + + raw_records = require_sequence(evidence["records"], context="frozen records") + records: list[dict[str, Any]] = [] + for index, raw_record in enumerate(raw_records): + record = require_mapping(raw_record, context=f"frozen records[{index}]") + require_exact_fields( + record, + FROZEN_RECORD_FIELDS, + context=f"frozen records[{index}]", + ) + capture_record = {name: record[name] for name in RECORD_FIELDS} + normalized = _normalize_record( + capture_record, + index=index, + phase="calibration", + tokenizer_hash=tokenizer_manifest_sha256, + ) + if dict(record) != normalized: + raise ValueError(f"frozen records[{index}] is not canonical") + records.append(normalized) + if records != sorted(records, key=_record_sort_key): + raise ValueError("frozen calibration records are not in canonical resolver order") + _validate_calibration_records(records) + if evidence["record_count"] != len(records) or len(records) != 160: + raise ValueError("frozen calibration record count drifted") + content_manifest_sha256 = require_sha256( + evidence["content_manifest_sha256"], + context="frozen content manifest SHA-256", + ) + if content_manifest_sha256 != sha256_bytes(canonical_json_bytes(records)): + raise ValueError("frozen content manifest SHA-256 drifted") + + expected_selection = { + "pg19_train_namespace": PG19_TRAIN_NAMESPACE, + "pg19_validation_namespace": PG19_VALIDATION_NAMESPACE, + "pg19_test_namespace": PG19_TEST_NAMESPACE, + "humaneval_plus_stage_a_b_namespace": HUMANEVAL_AB_NAMESPACE, + "humaneval_plus_stage_c_namespace": HUMANEVAL_C_NAMESPACE, + } + if evidence["selection"] != expected_selection: + raise ValueError("frozen selection namespaces drifted") + expected_upstream = { + "ruler_generator": {"id": RULER_SOURCE_ID, "revision": RULER_REVISION}, + "evalplus_formatter": { + "id": EVALPLUS_SOURCE_ID, + "revision": EVALPLUS_SOURCE_REVISION, + }, + } + if evidence["upstream_tool_contracts"] != expected_upstream: + raise ValueError("frozen upstream tool contracts drifted") + expected_protected = { + "stage_b_read": False, + "stage_c_read": False, + "ordinary_tests_may_read_protected_content": False, + } + if evidence["protected_identity"] != expected_protected: + raise ValueError("frozen protected identity boundary drifted") + + split = require_mapping( + evidence["calibration_split_half"], + context="frozen calibration split", + ) + expected_split = _split_half_manifest(records) + if dict(split) != expected_split: + raise ValueError("frozen calibration split assignments drifted") + + promotion = require_mapping(evidence["promotion"], context="frozen promotion") + require_exact_fields( + promotion, + frozenset( + { + "candidate_file_sha256", + "candidate_canonical_evidence_sha256", + "explicit", + } + ), + context="frozen promotion", + ) + if promotion["explicit"] is not True: + raise ValueError("frozen identity was not explicitly promoted") + candidate_evidence = deepcopy(dict(evidence)) + candidate_evidence.pop("promotion") + candidate_evidence["identity_schema"] = CANDIDATE_SCHEMA + candidate_evidence["status"] = "candidate" + candidate_evidence["promotion_required"] = True + candidate_canonical_sha256 = sha256_bytes(canonical_json_bytes(candidate_evidence)) + if ( + require_sha256( + promotion["candidate_canonical_evidence_sha256"], + context="promoted candidate canonical evidence SHA-256", + ) + != candidate_canonical_sha256 + ): + raise ValueError("promoted candidate canonical evidence SHA-256 drifted") + candidate_document = { + "canonical_evidence_sha256": candidate_canonical_sha256, + "evidence": candidate_evidence, + } + candidate_file_sha256 = sha256_bytes(canonical_json_bytes(candidate_document)) + if ( + require_sha256( + promotion["candidate_file_sha256"], + context="promoted candidate file SHA-256", + ) + != candidate_file_sha256 + ): + raise ValueError("promoted candidate file SHA-256 drifted") + + assignments = tuple( + dict(item) + for item in require_sequence( + expected_split["assignment"], + context="frozen split assignment", + ) + ) + return FrozenCalibrationIdentityArtifact( + file_sha256=file_sha256, + canonical_evidence_sha256=canonical_evidence_sha256, + records=tuple(records), + assignment=assignments, + assignment_sha256=str(expected_split["assignment_sha256"]), + tokenizer_manifest_sha256=str(tokenizer_manifest_sha256), + ) + + +def deserialize_frozen_stage_a_identity_artifact( + data: bytes, + *, + calibration_binding_artifact: bytes, + expected_file_sha256: str | None = None, +) -> FrozenStageAIdentityArtifact: + """Decode Stage A and reauthenticate both its promotion and calibration chain.""" + + if not isinstance(data, bytes): + raise TypeError("frozen Stage-A identity artifact must be bytes") + if not isinstance(calibration_binding_artifact, bytes): + raise TypeError("Stage-A calibration binding artifact must be bytes") + file_sha256 = sha256_bytes(data) + if expected_file_sha256 is not None: + expected = require_sha256( + expected_file_sha256, + context="expected frozen Stage-A identity file SHA-256", + ) + if file_sha256 != expected: + raise ValueError("frozen Stage-A identity file SHA-256 mismatch") + root = _json_without_duplicate_keys(data, context="frozen Stage-A identity artifact") + require_exact_fields( + root, + frozenset({"canonical_evidence_sha256", "evidence"}), + context="frozen Stage-A identity wrapper", + ) + if canonical_json_bytes(root) != data: + raise ValueError("frozen Stage-A identity bytes are not canonical resolver JSON") + evidence = require_mapping(root["evidence"], context="frozen Stage-A identity evidence") + require_exact_fields( + evidence, + FROZEN_EVIDENCE_FIELDS, + context="frozen Stage-A identity evidence", + ) + canonical_evidence_sha256 = require_sha256( + root["canonical_evidence_sha256"], + context="frozen Stage-A canonical evidence SHA-256", + ) + if canonical_evidence_sha256 != sha256_bytes(canonical_json_bytes(evidence)): + raise ValueError("frozen Stage-A canonical evidence SHA-256 drifted") + if ( + evidence["identity_schema"] != FROZEN_SCHEMA + or evidence["status"] != "frozen" + or evidence["phase"] != "stage_a" + or evidence["promotion_required"] is not False + ): + raise ValueError("frozen Stage-A identity contract drifted") + + promotion = require_mapping(evidence["promotion"], context="frozen Stage-A promotion") + require_exact_fields( + promotion, + frozenset( + { + "candidate_file_sha256", + "candidate_canonical_evidence_sha256", + "explicit", + } + ), + context="frozen Stage-A promotion", + ) + if promotion["explicit"] is not True: + raise ValueError("frozen Stage-A identity was not explicitly promoted") + candidate_evidence = deepcopy(dict(evidence)) + candidate_evidence.pop("promotion") + candidate_evidence["identity_schema"] = CANDIDATE_SCHEMA + candidate_evidence["status"] = "candidate" + candidate_evidence["promotion_required"] = True + candidate_canonical_sha256 = sha256_bytes(canonical_json_bytes(candidate_evidence)) + candidate_document = { + "canonical_evidence_sha256": candidate_canonical_sha256, + "evidence": candidate_evidence, + } + validate_candidate_artifact(candidate_document) + if ( + require_sha256( + promotion["candidate_canonical_evidence_sha256"], + context="promoted Stage-A candidate canonical evidence SHA-256", + ) + != candidate_canonical_sha256 + ): + raise ValueError("promoted Stage-A candidate canonical evidence SHA-256 drifted") + candidate_file_sha256 = sha256_bytes(canonical_json_bytes(candidate_document)) + if ( + require_sha256( + promotion["candidate_file_sha256"], + context="promoted Stage-A candidate file SHA-256", + ) + != candidate_file_sha256 + ): + raise ValueError("promoted Stage-A candidate file SHA-256 drifted") + + verified_binding = deserialize_stage_a_calibration_binding_artifact( + calibration_binding_artifact + ).binding + if candidate_evidence["calibration_binding"] != verified_binding: + raise ValueError("frozen Stage-A identity differs from the verified calibration binding") + records = tuple(dict(record) for record in candidate_evidence["records"]) + tokenizer_manifest_sha256 = require_sha256( + candidate_evidence["tokenizer"]["file_manifest_sha256"], + context="frozen Stage-A tokenizer manifest SHA-256", + ) + return FrozenStageAIdentityArtifact( + file_sha256=file_sha256, + canonical_evidence_sha256=canonical_evidence_sha256, + records=records, + tokenizer_manifest_sha256=tokenizer_manifest_sha256, + calibration_binding=dict(verified_binding), + ) + + +def _canonical_b64(data: bytes, *, context: str) -> str: + if not isinstance(data, bytes): + raise TypeError(f"{context} must be bytes") + return base64.b64encode(data).decode("ascii") + + +def _decode_canonical_b64(value: object, *, context: str) -> bytes: + if not isinstance(value, str): + raise ValueError(f"{context} must be base64 text") + try: + decoded = base64.b64decode(value, validate=True) + except (ValueError, binascii.Error) as error: + raise ValueError(f"{context} is invalid base64") from error + if base64.b64encode(decoded).decode("ascii") != value: + raise ValueError(f"{context} is not canonical base64") + return decoded + + +def _identity_half_record_manifests( + identity: FrozenCalibrationIdentityArtifact, +) -> dict[str, str]: + from recurquant.static_q468_calibration import ( + calibration_identity_record_manifest_sha256, + ) + + records_by_identity = { + ( + record["family"], + record["ruler_category"], + record["config"], + record["canonical_id"], + record["seed"], + record["configured_length"], + record["sequence_length"], + ): record + for record in identity.records + } + halves: dict[str, list[dict[str, Any]]] = {"a": [], "b": []} + seen: set[tuple[object, ...]] = set() + for index, assignment in enumerate(identity.assignment): + group = assignment["group"] + if not isinstance(group, str): + raise ValueError(f"identity assignment {index} group is invalid") + family = "ruler" if group.startswith("ruler:") else group + key = ( + family, + assignment["ruler_category"], + assignment["config"], + assignment["canonical_id"], + assignment["seed"], + assignment["configured_length"], + assignment["sequence_length"], + ) + if key not in records_by_identity or key in seen: + raise ValueError( + "resolver split assignment does not bijectively cover identity records" + ) + half = assignment["half"] + if half not in halves: + raise ValueError("resolver split assignment half drifted") + seen.add(key) + halves[str(half)].append(records_by_identity[key]) + if seen != set(records_by_identity): + raise ValueError("resolver split assignments do not cover all calibration records") + return { + half: calibration_identity_record_manifest_sha256(records) + for half, records in halves.items() + } + + +def _derive_stage_a_calibration_binding( + *, + frozen_identity_artifact: bytes, + calibration_score_artifact: bytes, + split_half_stability_artifact: bytes, + static_k27030_policy_artifact: bytes, + static_k29334_policy_artifact: bytes, +) -> tuple[dict[str, str], dict[str, str]]: + from recurquant.static_q468 import ( + FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + FROZEN_STATIC_Q468_ABLATION_STEPS, + FROZEN_STATIC_Q468_PRIMARY_STEPS, + STATIC_Q468_ABLATION_METHOD, + STATIC_Q468_PRIMARY_METHOD, + deserialize_static_rht_q468_policy, + ) + from recurquant.static_q468_calibration import ( + CALIBRATION_SCORE_ARTIFACT_KIND, + calibration_identity_record_manifest_sha256, + deserialize_calibration_score_artifact, + deserialize_frozen_split_half_stability_artifact, + ) + + identity = deserialize_frozen_calibration_identity_artifact(frozen_identity_artifact) + scores = deserialize_calibration_score_artifact(calibration_score_artifact) + if scores.artifact_kind != CALIBRATION_SCORE_ARTIFACT_KIND: + raise ValueError("Stage-A binding requires the official frozen score artifact") + expected_identity_manifest = calibration_identity_record_manifest_sha256(identity.records) + if scores.calibration_identity_sha256 != identity.file_sha256: + raise ValueError("score artifact is not bound to the frozen identity file") + if scores.aggregate.identity_record_manifest_sha256 != expected_identity_manifest: + raise ValueError("score identity-record manifest differs from the frozen identity") + split = deserialize_frozen_split_half_stability_artifact( + split_half_stability_artifact, + expected_identity_file_sha256=identity.file_sha256, + expected_canonical_identity_sha256=identity.canonical_evidence_sha256, + expected_resolver_assignment_sha256=identity.assignment_sha256, + ) + if ( + split.identity_file_sha256 != identity.file_sha256 + or split.canonical_identity_sha256 != identity.canonical_evidence_sha256 + or split.resolver_assignment_sha256 != identity.assignment_sha256 + ): + raise ValueError("split-half artifact identity binding drifted") + if ( + split.full_sequence_score_manifest_sha256 != scores.aggregate.sequence_score_manifest_sha256 + or split.full_calibration_scores_sha256 != scores.calibration_scores_sha256 + ): + raise ValueError("split-half artifact differs from the official full score artifact") + half_manifests = _identity_half_record_manifests(identity) + if ( + split.half_a_aggregate.identity_record_manifest_sha256 != half_manifests["a"] + or split.half_b_aggregate.identity_record_manifest_sha256 != half_manifests["b"] + ): + raise ValueError("split-half identity-record manifests differ from resolver assignment") + + policy27030 = deserialize_static_rht_q468_policy(static_k27030_policy_artifact) + policy29334 = deserialize_static_rht_q468_policy(static_k29334_policy_artifact) + expected_policy_contracts = ( + ( + policy27030, + STATIC_Q468_ABLATION_METHOD, + FROZEN_STATIC_Q468_ABLATION_STEPS, + ), + ( + policy29334, + STATIC_Q468_PRIMARY_METHOD, + FROZEN_STATIC_Q468_PRIMARY_STEPS, + ), + ) + allocations = {steps: (codes, digest) for steps, codes, digest in scores.allocations} + for policy, method_id, steps in expected_policy_contracts: + if ( + policy.method_id != method_id + or policy.marginal_steps != steps + or policy.geometry != FROZEN_QWEN35_STATIC_Q468_GEOMETRY + ): + raise ValueError(f"policy {method_id} does not satisfy its reserved exact-K contract") + if policy.identity_artifact_sha256 != identity.file_sha256: + raise ValueError(f"policy {method_id} is not bound to the frozen identity file") + if policy.tokenizer_manifest_sha256 != identity.tokenizer_manifest_sha256: + raise ValueError(f"policy {method_id} tokenizer manifest differs from identity") + if ( + policy.calibration_manifest_sha256 != scores.aggregate.sequence_score_manifest_sha256 + or policy.calibration_scores_sha256 != scores.calibration_scores_sha256 + ): + raise ValueError(f"policy {method_id} differs from official calibration scores") + if steps not in allocations: + raise ValueError(f"official score artifact is missing exact K{steps}") + allocation_codes, allocation_hash = allocations[steps] + if policy.code_map_sha256 != allocation_hash or not __import__("torch").equal( + policy.precision_codes().reshape(-1).to("cpu"), + allocation_codes, + ): + raise ValueError(f"policy {method_id} code map differs from exact allocation") + if policy27030.source_commit != policy29334.source_commit: + raise ValueError("K27030 and K29334 policies must share one source commit") + + binding = { + "calibration_identity_file_sha256": identity.file_sha256, + "calibration_score_artifact_file_sha256": scores.file_sha256, + "split_half_stability_artifact_file_sha256": split.file_sha256, + "static_k27030_policy_file_sha256": sha256_bytes(static_k27030_policy_artifact), + "static_k29334_policy_file_sha256": sha256_bytes(static_k29334_policy_artifact), + } + dependency_hashes = { + "calibration_score_artifact": scores.file_sha256, + "frozen_identity_artifact": identity.file_sha256, + "split_half_stability_artifact": split.file_sha256, + "static_k27030_policy_artifact": sha256_bytes(static_k27030_policy_artifact), + "static_k29334_policy_artifact": sha256_bytes(static_k29334_policy_artifact), + } + return binding, dependency_hashes + + +def build_stage_a_calibration_binding_artifact( + *, + frozen_identity_artifact: bytes, + calibration_score_artifact: bytes, + split_half_stability_artifact: bytes, + static_k27030_policy_artifact: bytes, + static_k29334_policy_artifact: bytes, +) -> bytes: + """Build the five-field Stage-A binding only from fully verified dependencies.""" + + dependencies = { + "calibration_score_artifact": calibration_score_artifact, + "frozen_identity_artifact": frozen_identity_artifact, + "split_half_stability_artifact": split_half_stability_artifact, + "static_k27030_policy_artifact": static_k27030_policy_artifact, + "static_k29334_policy_artifact": static_k29334_policy_artifact, + } + binding, dependency_hashes = _derive_stage_a_calibration_binding( + frozen_identity_artifact=frozen_identity_artifact, + calibration_score_artifact=calibration_score_artifact, + split_half_stability_artifact=split_half_stability_artifact, + static_k27030_policy_artifact=static_k27030_policy_artifact, + static_k29334_policy_artifact=static_k29334_policy_artifact, + ) + evidence = { + "artifact_revision": STAGE_A_BINDING_ARTIFACT_REVISION, + "binding": binding, + "dependencies_base64": { + name: _canonical_b64(value, context=name) + for name, value in sorted(dependencies.items()) + }, + "dependency_file_sha256": dependency_hashes, + } + document = { + "artifact_kind": STAGE_A_BINDING_ARTIFACT_KIND, + "canonical_evidence_sha256": sha256_bytes(canonical_json_bytes(evidence)), + "evidence": evidence, + "schema_version": STAGE_A_BINDING_ARTIFACT_SCHEMA_VERSION, + } + return canonical_json_bytes(document) + + +def deserialize_stage_a_calibration_binding_artifact( + data: bytes, + *, + expected_file_sha256: str | None = None, +) -> StageACalibrationBindingArtifact: + """Strictly reverify every embedded dependency of a Stage-A binding artifact.""" + + if not isinstance(data, bytes): + raise TypeError("Stage-A calibration binding artifact must be bytes") + file_sha256 = sha256_bytes(data) + if expected_file_sha256 is not None and file_sha256 != require_sha256( + expected_file_sha256, + context="expected Stage-A binding file SHA-256", + ): + raise ValueError("Stage-A calibration binding file SHA-256 mismatch") + root = _json_without_duplicate_keys(data, context="Stage-A calibration binding") + require_exact_fields( + root, + frozenset( + { + "artifact_kind", + "canonical_evidence_sha256", + "evidence", + "schema_version", + } + ), + context="Stage-A calibration binding wrapper", + ) + if canonical_json_bytes(root) != data: + raise ValueError("Stage-A calibration binding bytes are not canonical JSON") + if ( + root["artifact_kind"] != STAGE_A_BINDING_ARTIFACT_KIND + or root["schema_version"] != STAGE_A_BINDING_ARTIFACT_SCHEMA_VERSION + ): + raise ValueError("Stage-A calibration binding kind or schema drifted") + evidence = require_mapping(root["evidence"], context="Stage-A binding evidence") + require_exact_fields( + evidence, + frozenset( + { + "artifact_revision", + "binding", + "dependencies_base64", + "dependency_file_sha256", + } + ), + context="Stage-A binding evidence", + ) + if evidence["artifact_revision"] != STAGE_A_BINDING_ARTIFACT_REVISION: + raise ValueError("Stage-A calibration binding revision drifted") + canonical_evidence_sha256 = require_sha256( + root["canonical_evidence_sha256"], + context="Stage-A binding canonical evidence SHA-256", + ) + if canonical_evidence_sha256 != sha256_bytes(canonical_json_bytes(evidence)): + raise ValueError("Stage-A binding canonical evidence SHA-256 drifted") + encoded_dependencies = require_mapping( + evidence["dependencies_base64"], + context="Stage-A binding dependencies", + ) + dependency_names = frozenset( + { + "calibration_score_artifact", + "frozen_identity_artifact", + "split_half_stability_artifact", + "static_k27030_policy_artifact", + "static_k29334_policy_artifact", + } + ) + require_exact_fields( + encoded_dependencies, + dependency_names, + context="Stage-A binding dependencies", + ) + dependencies = { + name: _decode_canonical_b64( + encoded_dependencies[name], + context=f"Stage-A dependency {name}", + ) + for name in sorted(dependency_names) + } + binding, dependency_hashes = _derive_stage_a_calibration_binding( + frozen_identity_artifact=dependencies["frozen_identity_artifact"], + calibration_score_artifact=dependencies["calibration_score_artifact"], + split_half_stability_artifact=dependencies["split_half_stability_artifact"], + static_k27030_policy_artifact=dependencies["static_k27030_policy_artifact"], + static_k29334_policy_artifact=dependencies["static_k29334_policy_artifact"], + ) + if evidence["binding"] != binding: + raise ValueError("Stage-A calibration binding fields drifted") + if evidence["dependency_file_sha256"] != dependency_hashes: + raise ValueError("Stage-A calibration dependency hashes drifted") + return StageACalibrationBindingArtifact( + binding=binding, + dependency_file_sha256=dependency_hashes, + canonical_evidence_sha256=canonical_evidence_sha256, + file_sha256=file_sha256, + ) + + def validate_quarantine_output(path: Path) -> None: resolved = path.resolve() if not any("quarantine" in part.lower() for part in resolved.parts[:-1]): @@ -1028,9 +2044,12 @@ def validate_promotion_output(path: Path) -> None: def atomic_write(path: Path, payload: bytes) -> None: - path.parent.mkdir(parents=True, exist_ok=True) + """Atomically publish one new file without a check-then-replace race.""" + + resolved = path.resolve() + resolved.parent.mkdir(parents=True, exist_ok=True) descriptor, temporary_name = tempfile.mkstemp( - prefix=f".{path.name}.", suffix=".tmp", dir=path.parent + prefix=f".{resolved.name}.", suffix=".tmp", dir=resolved.parent ) temporary = Path(temporary_name) try: @@ -1038,7 +2057,13 @@ def atomic_write(path: Path, payload: bytes) -> None: handle.write(payload) handle.flush() os.fsync(handle.fileno()) - os.replace(temporary, path) + try: + os.link(temporary, resolved) + except FileExistsError as error: + raise FileExistsError( + f"refusing to overwrite existing identity file: {resolved}" + ) from error + temporary.unlink() finally: if temporary.exists(): temporary.unlink() @@ -1058,6 +2083,7 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.add_argument("--dry-run", action="store_true") parser.add_argument("--promote", action="store_true") parser.add_argument("--expected-candidate-sha256") + parser.add_argument("--calibration-binding", type=Path) parser.add_argument("--mbpp-revision") parser.add_argument("--pg19-revision") parser.add_argument("--ruler-revision") @@ -1068,7 +2094,7 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: def _reject_protected_before_input(phase: str) -> None: if phase in PROTECTED_STAGES: raise PermissionError( - f"{phase} is protected; resolver v1 refuses it before reading --input" + f"{phase} is protected; resolver v2 refuses it before reading --input" ) @@ -1078,6 +2104,13 @@ def main(argv: Sequence[str] | None = None) -> int: if args.promote: if args.dry_run or args.output is None: raise ValueError("promotion requires --output and forbids --dry-run") + calibration_binding_artifact: bytes | None = None + if args.phase == "stage_a": + if args.calibration_binding is None: + raise ValueError("Stage-A promotion requires --calibration-binding") + calibration_binding_artifact = args.calibration_binding.read_bytes() + elif args.calibration_binding is not None: + raise ValueError("calibration promotion forbids --calibration-binding") expected_hash = require_sha256( args.expected_candidate_sha256, context="--expected-candidate-sha256", @@ -1087,9 +2120,15 @@ def main(argv: Sequence[str] | None = None) -> int: if actual_hash != expected_hash: raise ValueError("candidate file SHA-256 does not match explicit promotion hash") candidate = _json_without_duplicate_keys(raw, context="candidate artifact") + if raw != canonical_json_bytes(candidate): + raise ValueError("candidate file bytes are not canonical resolver JSON") if candidate.get("evidence", {}).get("phase") != args.phase: raise ValueError("candidate phase does not match --phase") - frozen = promote_candidate(candidate, candidate_file_sha256=actual_hash) + frozen = promote_candidate( + candidate, + candidate_file_sha256=actual_hash, + calibration_binding_artifact=calibration_binding_artifact, + ) validate_promotion_output(args.output) atomic_write(args.output, canonical_json_bytes(frozen)) print(sha256_bytes(canonical_json_bytes(frozen))) @@ -1108,7 +2147,18 @@ def main(argv: Sequence[str] | None = None) -> int: source = _json_without_duplicate_keys(args.input.read_bytes(), context="identity input") if source.get("phase") != args.phase: raise ValueError("input phase does not match --phase") - candidate = build_candidate(source, expected_revisions=revisions) # type: ignore[arg-type] + calibration_binding_artifact: bytes | None = None + if args.phase == "stage_a": + if args.calibration_binding is None: + raise ValueError("Stage A requires --calibration-binding") + calibration_binding_artifact = args.calibration_binding.read_bytes() + elif args.calibration_binding is not None: + raise ValueError("--calibration-binding is valid only for Stage A") + candidate = build_candidate( + source, + expected_revisions=revisions, # type: ignore[arg-type] + calibration_binding_artifact=calibration_binding_artifact, + ) payload = canonical_json_bytes(candidate) digest = sha256_bytes(payload) if args.dry_run: diff --git a/tests/test_capture_static_q468_identity_input.py b/tests/test_capture_static_q468_identity_input.py new file mode 100644 index 0000000..328057c --- /dev/null +++ b/tests/test_capture_static_q468_identity_input.py @@ -0,0 +1,870 @@ +from __future__ import annotations + +import base64 +import copy +import importlib.util +import io +import json +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest +import torch + +from recurquant.static_q468 import ( + FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + FROZEN_STATIC_Q468_ABLATION_STEPS, + FROZEN_STATIC_Q468_PRIMARY_STEPS, + STATIC_Q468_ABLATION_METHOD, + STATIC_Q468_PRIMARY_METHOD, + build_static_rht_q468_policy, + serialize_static_rht_q468_policy, +) +from recurquant.static_q468_calibration import ( + FROZEN_SOURCE_TENSOR_CONTRACT, + CalibrationAggregate, + build_frozen_calibration_score_artifact, + build_frozen_split_half_stability_artifact, + calibration_identity_record_manifest_sha256, + deserialize_calibration_score_artifact, +) + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +SCRIPT_PATH = REPOSITORY_ROOT / "scripts" / "capture_static_q468_identity_input.py" +SPEC = importlib.util.spec_from_file_location("capture_static_q468_identity_input", SCRIPT_PATH) +assert SPEC is not None and SPEC.loader is not None +capture = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = capture +SPEC.loader.exec_module(capture) +resolver = capture.resolver +FIXTURE_BINDING_ARTIFACT = b"verified-fixture-binding-artifact" + + +def test_capture_script_imports_in_direct_cli_process() -> None: + completed = subprocess.run( + [sys.executable, str(SCRIPT_PATH), "--help"], + cwd=REPOSITORY_ROOT, + capture_output=True, + check=False, + text=True, + ) + + assert completed.returncode == 0, completed.stderr + assert "Capture a calibration or Stage-A Experiment 013 identity input" in ( + completed.stdout.replace("\n", " ") + ) + + +def _hash(label: str) -> str: + return capture.sha256_bytes(label.encode()) + + +class FakeTokenizer: + def encode(self, text: str, *, add_special_tokens: bool) -> list[int]: + prefix = [1] if add_special_tokens else [] + return [*prefix, *(2 + (ord(character) % 251) for character in text)] + + +def _fake_generator_files() -> dict[str, bytes]: + config_yaml = "".join( + f"{config}:\n task: fixture\n" for config in capture.RULER_ALL_CONFIGS + ).encode() + return { + path: ( + config_yaml + if path == "scripts/synthetic.yaml" + else f"fixture source for {path}\n".encode() + ) + for path in capture.RULER_GENERATOR_GIT_BLOBS + } + + +@pytest.fixture(autouse=True) +def _bind_fixture_generator_blobs(monkeypatch: pytest.MonkeyPatch) -> None: + files = _fake_generator_files() + monkeypatch.setattr( + capture, + "RULER_GENERATOR_GIT_BLOBS", + {path: capture._git_blob_sha1(content) for path, content in files.items()}, + ) + strict_binding_decoder = resolver.deserialize_stage_a_calibration_binding_artifact + + def decode_binding(data: bytes) -> object: + if data == FIXTURE_BINDING_ARTIFACT: + return SimpleNamespace( + binding={ + key: _hash(f"binding-{key}") + for key in sorted(resolver.CALIBRATION_BINDING_FIELDS) + } + ) + return strict_binding_decoder(data) + + monkeypatch.setattr( + resolver, + "deserialize_stage_a_calibration_binding_artifact", + decode_binding, + ) + + +class FakeSource: + def __init__(self) -> None: + self.accesses: list[str] = [] + self.head_calls = 0 + self.drift_after_capture = False + self.short_pg19_ids: set[str] = set() + self.extra_tokenizer_files: dict[str, bytes] = {} + self.generator_files = _fake_generator_files() + self.receipt_mutator: Any = None + + def source_heads(self) -> dict[str, str]: + self.accesses.append("source_heads") + self.head_calls += 1 + heads = dict(capture.EXPECTED_SOURCE_HEADS) + if self.drift_after_capture and self.head_calls > 1: + heads["pg19"] = "f" * 40 + return heads + + def tokenizer_material(self) -> Any: + self.accesses.append("tokenizer_material") + return capture.TokenizerMaterial( + tokenizer=FakeTokenizer(), + tokenizer_class="FixtureTokenizer", + transformers_version=resolver.TRANSFORMERS_VERSION, + files={ + "tokenizer.json": b"fixture-tokenizer", + "tokenizer_config.json": b"fixture-tokenizer-config", + **self.extra_tokenizer_files, + }, + model_weights_loaded=False, + ) + + def mbpp_train_rows(self) -> tuple[dict[str, Any], ...]: + self.accesses.append("mbpp_train_rows") + return tuple( + { + "task_id": task_id, + "text": f"Return {task_id}.", + "code": f"def answer():\n return {task_id}\n", + "test_list": [f"assert answer() == {task_id}"], + "test_setup_code": "", + "challenge_test_list": [], + } + for task_id in range(601, 975) + ) + + def pg19_projection(self, split: str) -> tuple[Any, ...]: + self.accesses.append(f"pg19_projection:{split}") + count = 13_684 if split == "train" else 50 + return tuple( + capture.ProjectionRow(f"https://pg19.example/{split}/{offset}", offset) + for offset in range(count) + ) + + def pg19_text(self, split: str, url: str) -> str: + if url in self.short_pg19_ids: + width = 2_300 if split == "train" else 4_200 + else: + width = 2_420 if split == "train" else 4_340 + return chr(65 + (int(url.rsplit("/", 1)[1]) % 20)) * width + + def pg19_row(self, split: str, *, offset: int, expected_url: str) -> dict[str, Any]: + self.accesses.append(f"pg19_row:{split}:{offset}") + return {"url": expected_url, "text": self.pg19_text(split, expected_url)} + + def ruler_generator_files(self) -> dict[str, bytes]: + self.accesses.append("ruler_generator_files") + return dict(self.generator_files) + + def ruler_receipt( + self, *, category: str, config: str, configured_length: int, seed: int + ) -> dict[str, Any]: + self.accesses.append(f"ruler_receipt:{category}:{config}:{configured_length}:{seed}") + prompt = f"RULER {category} {config} {configured_length} {seed}." + output_count = capture.RULER_REQUIRED_OUTPUT_COUNTS.get(config, 2) + receipt: dict[str, Any] = { + "schema": capture.RULER_RECEIPT_SCHEMA, + "source_id": resolver.RULER_SOURCE_ID, + "revision": resolver.RULER_REVISION, + "category": category, + "config": config, + "configured_length": configured_length, + "seed": seed, + "sample_index": 0, + "generator_reported_length": len(prompt) + 32, + "input": prompt, + "answer_prefix": " Answer:", + "outputs": [f"result-{config}-{seed}-{index}" for index in range(output_count)], + "auxiliary_files": [ + { + "name": f"fixture/{config}.txt", + "sha256": _hash(f"aux-{config}"), + "size_bytes": 100 + len(config), + } + ], + } + if self.receipt_mutator is not None: + self.receipt_mutator(receipt) + return receipt + + def humaneval_projection(self) -> tuple[Any, ...]: + self.accesses.append("humaneval_projection") + return tuple(capture.ProjectionRow(f"HumanEval/{offset}", offset) for offset in range(164)) + + def humaneval_row(self, *, offset: int, expected_task_id: str) -> dict[str, Any]: + self.accesses.append(f"humaneval_row:{offset}") + return { + "task_id": expected_task_id, + "prompt": f"def task_{offset}(x):\n", + "canonical_solution": " return x\n" * 20, + "entry_point": f"task_{offset}", + "test": f"assert task_{offset}(1) == 1", + } + + +def _binding() -> bytes: + return FIXTURE_BINDING_ARTIFACT + + +def _frozen_aggregate( + *, + half: bool, + identity_manifest_sha256: str, + sequence_manifest_sha256: str, +) -> CalibrationAggregate: + rows = FROZEN_QWEN35_STATIC_Q468_GEOMETRY.total_rows + row_axis = torch.arange(rows, dtype=torch.float64) + broad_counts = ( + (("mbpp", 64), ("pg19", 8), ("ruler", 8)) + if half + else (("mbpp", 128), ("pg19", 16), ("ruler", 16)) + ) + ruler_count = 2 if half else 4 + return CalibrationAggregate( + d4=4.0 + row_axis / rows, + d6=2.0 + row_axis / (2 * rows), + d8=1.0 + row_axis / (4 * rows), + family_sequence_counts=broad_counts, + ruler_category_sequence_counts=tuple( + (category, ruler_count) for category in resolver.RULER_CATEGORIES + ), + sequence_score_manifest_sha256=sequence_manifest_sha256, + source_contract=FROZEN_SOURCE_TENSOR_CONTRACT, + identity_record_manifest_sha256=identity_manifest_sha256, + ) + + +def test_calibration_capture_is_deterministic_and_resolver_compatible() -> None: + first = capture.capture_identity_input(phase="calibration", source=FakeSource()) + second = capture.capture_identity_input(phase="calibration", source=FakeSource()) + + assert capture.canonical_json_bytes(first) == capture.canonical_json_bytes(second) + candidate = resolver.build_candidate( + first, expected_revisions=resolver.FROZEN_DATASET_REVISIONS + ) + assert candidate["evidence"]["record_count"] == 160 + counts = { + family: sum(row["family"] == family for row in first["records"]) + for family in resolver.DATASET_KEYS + } + assert counts == {"mbpp": 128, "pg19": 16, "ruler": 16, "humaneval_plus": 0} + assert first["model_weights_loaded"] is False + assert all( + row["ruler_category"] is None + and row["configured_length"] is None + and row["generator_receipt_sha256"] is None + for row in first["records"] + if row["family"] != "ruler" + ) + assert all( + row["token_span"]["cache_exposed_start"] + == row["token_span"]["scored_stop"] + == row["token_span"]["cache_exposed_stop"] + for row in first["records"] + ) + + +def test_frozen_calibration_identity_decoder_recomputes_capture_lineage() -> None: + captured = capture.capture_identity_input(phase="calibration", source=FakeSource()) + candidate = resolver.build_candidate( + captured, + expected_revisions=resolver.FROZEN_DATASET_REVISIONS, + ) + candidate_bytes = resolver.canonical_json_bytes(candidate) + frozen = resolver.promote_candidate( + candidate, + candidate_file_sha256=resolver.sha256_bytes(candidate_bytes), + ) + frozen_bytes = resolver.canonical_json_bytes(frozen) + + decoded = resolver.deserialize_frozen_calibration_identity_artifact(frozen_bytes) + + assert decoded.file_sha256 == resolver.sha256_bytes(frozen_bytes) + assert decoded.canonical_evidence_sha256 == frozen["canonical_evidence_sha256"] + assert len(decoded.records) == 160 + assert len(decoded.assignment) == 160 + assert ( + decoded.assignment_sha256 + == frozen["evidence"]["calibration_split_half"]["assignment_sha256"] + ) + assert all( + row["identity_record_sha256"] == resolver.identity_record_sha256(row) + for row in decoded.records + ) + + tampered = copy.deepcopy(frozen) + tampered["evidence"]["records"][0]["anchor_positions"][0] += 1 + tampered["evidence"]["content_manifest_sha256"] = resolver.sha256_bytes( + resolver.canonical_json_bytes(tampered["evidence"]["records"]) + ) + tampered["canonical_evidence_sha256"] = resolver.sha256_bytes( + resolver.canonical_json_bytes(tampered["evidence"]) + ) + with pytest.raises(ValueError, match="not canonical"): + resolver.deserialize_frozen_calibration_identity_artifact( + resolver.canonical_json_bytes(tampered) + ) + + +def test_stage_a_binding_is_derived_from_identity_scores_split_and_policies() -> None: + captured = capture.capture_identity_input(phase="calibration", source=FakeSource()) + candidate = resolver.build_candidate( + captured, + expected_revisions=resolver.FROZEN_DATASET_REVISIONS, + ) + candidate_bytes = resolver.canonical_json_bytes(candidate) + frozen = resolver.promote_candidate( + candidate, + candidate_file_sha256=resolver.sha256_bytes(candidate_bytes), + ) + identity_bytes = resolver.canonical_json_bytes(frozen) + identity = resolver.deserialize_frozen_calibration_identity_artifact(identity_bytes) + full_identity_manifest = calibration_identity_record_manifest_sha256(identity.records) + half_identity_manifests = resolver._identity_half_record_manifests(identity) + full_sequence_manifest = "c" * 64 + full_aggregate = _frozen_aggregate( + half=False, + identity_manifest_sha256=full_identity_manifest, + sequence_manifest_sha256=full_sequence_manifest, + ) + score_bytes = build_frozen_calibration_score_artifact( + full_aggregate, + calibration_identity_sha256=identity.file_sha256, + ) + score = deserialize_calibration_score_artifact(score_bytes) + split_bytes = build_frozen_split_half_stability_artifact( + _frozen_aggregate( + half=True, + identity_manifest_sha256=half_identity_manifests["a"], + sequence_manifest_sha256="a" * 64, + ), + _frozen_aggregate( + half=True, + identity_manifest_sha256=half_identity_manifests["b"], + sequence_manifest_sha256="b" * 64, + ), + identity_file_sha256=identity.file_sha256, + canonical_identity_sha256=identity.canonical_evidence_sha256, + resolver_assignment_sha256=identity.assignment_sha256, + full_sequence_score_manifest_sha256=full_sequence_manifest, + full_calibration_scores_sha256=score.calibration_scores_sha256, + ) + policy_arguments = { + "d4": full_aggregate.d4, + "d6": full_aggregate.d6, + "d8": full_aggregate.d8, + "geometry": FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + "calibration_manifest_sha256": full_sequence_manifest, + "identity_artifact_sha256": identity.file_sha256, + "tokenizer_manifest_sha256": identity.tokenizer_manifest_sha256, + "source_commit": "f" * 40, + "calibration_scores_sha256": score.calibration_scores_sha256, + } + policy27030_bytes = serialize_static_rht_q468_policy( + build_static_rht_q468_policy( + **policy_arguments, + marginal_steps=FROZEN_STATIC_Q468_ABLATION_STEPS, + method_id=STATIC_Q468_ABLATION_METHOD, + ) + ) + policy29334_bytes = serialize_static_rht_q468_policy( + build_static_rht_q468_policy( + **policy_arguments, + marginal_steps=FROZEN_STATIC_Q468_PRIMARY_STEPS, + method_id=STATIC_Q468_PRIMARY_METHOD, + ) + ) + binding_bytes = resolver.build_stage_a_calibration_binding_artifact( + frozen_identity_artifact=identity_bytes, + calibration_score_artifact=score_bytes, + split_half_stability_artifact=split_bytes, + static_k27030_policy_artifact=policy27030_bytes, + static_k29334_policy_artifact=policy29334_bytes, + ) + + verified = resolver.deserialize_stage_a_calibration_binding_artifact(binding_bytes) + + assert capture._normalize_calibration_binding(binding_bytes) == verified.binding + assert verified.binding == { + "calibration_identity_file_sha256": identity.file_sha256, + "calibration_score_artifact_file_sha256": resolver.sha256_bytes(score_bytes), + "split_half_stability_artifact_file_sha256": resolver.sha256_bytes(split_bytes), + "static_k27030_policy_file_sha256": resolver.sha256_bytes(policy27030_bytes), + "static_k29334_policy_file_sha256": resolver.sha256_bytes(policy29334_bytes), + } + + tampered = json.loads(binding_bytes) + encoded_policy = tampered["evidence"]["dependencies_base64"]["static_k29334_policy_artifact"] + policy_payload = bytearray(base64.b64decode(encoded_policy)) + policy_payload[0] ^= 1 + tampered["evidence"]["dependencies_base64"]["static_k29334_policy_artifact"] = base64.b64encode( + policy_payload + ).decode("ascii") + tampered["canonical_evidence_sha256"] = resolver.sha256_bytes( + resolver.canonical_json_bytes(tampered["evidence"]) + ) + with pytest.raises(ValueError): + resolver.deserialize_stage_a_calibration_binding_artifact( + resolver.canonical_json_bytes(tampered) + ) + + +def test_stage_a_capture_uses_exact_schedules_and_token_caps() -> None: + captured = capture.capture_identity_input( + phase="stage_a", source=FakeSource(), calibration_binding=_binding() + ) + candidate = resolver.build_candidate( + captured, + expected_revisions=resolver.FROZEN_DATASET_REVISIONS, + calibration_binding_artifact=_binding(), + ) + assert candidate["evidence"]["record_count"] == 12 + ruler_rows = [row for row in captured["records"] if row["family"] == "ruler"] + assert { + ( + row["ruler_category"], + row["config"], + row["configured_length"], + row["seed"], + ) + for row in ruler_rows + } == set(resolver.RULER_STAGE_A_SCHEDULE) + assert all(row["sequence_length"] < row["configured_length"] for row in ruler_rows) + assert all( + row["token_span"]["cache_exposed_stop"] - row["token_span"]["cache_exposed_start"] + == row["token_span"]["scored_stop"] - row["token_span"]["scored_start"] - 1 + for row in ruler_rows + ) + human_rows = [row for row in captured["records"] if row["family"] == "humaneval_plus"] + assert len(human_rows) == 4 + assert all( + row["token_span"]["scored_stop"] - row["token_span"]["scored_start"] == 128 + for row in human_rows + ) + assert all( + row["token_span"]["cache_exposed_stop"] - row["token_span"]["cache_exposed_start"] == 127 + for row in human_rows + ) + + +def test_ruler_stage_a_target_includes_all_required_outputs_and_selects_one_qa_alternative() -> ( + None +): + required, required_semantics = capture._ruler_stage_a_target( + category="retrieval", + config="niah_multiquery", + outputs=("11", "22", "33", "44"), + ) + alternative, alternative_semantics = capture._ruler_stage_a_target( + category="question_answering", + config="qa_1", + outputs=("first answer", "alternate answer"), + ) + + assert required == "11, 22, 33, 44" + assert required_semantics == "all_required_outputs_comma_space_v1" + assert alternative == "first answer" + assert alternative_semantics == "first_pinned_alternative_reference_v1" + + +def test_ruler_receipt_required_output_cardinality_and_uniqueness_fail_closed() -> None: + source = FakeSource() + receipt = source.ruler_receipt( + category="retrieval", + config="niah_multiquery", + configured_length=4_096, + seed=2_339, + ) + receipt["outputs"] = ["only-one"] + with pytest.raises(ValueError, match="exactly 4 required outputs"): + capture._normalize_ruler_receipt( + receipt, + category="retrieval", + config="niah_multiquery", + configured_length=4_096, + seed=2_339, + ) + + receipt["outputs"] = ["same"] * 4 + with pytest.raises(ValueError, match="must be unique"): + capture._normalize_ruler_receipt( + receipt, + category="retrieval", + config="niah_multiquery", + configured_length=4_096, + seed=2_339, + ) + + +def test_pg19_ranks_all_ids_before_text_and_skips_ineligible_rows() -> None: + source = FakeSource() + projection = source.pg19_projection("train") + source.accesses.clear() + ranked = sorted( + projection, + key=lambda item: ( + resolver.selection_sha256(resolver.PG19_TRAIN_NAMESPACE, item.canonical_id), + item.canonical_id, + ), + ) + source.short_pg19_ids.update(item.canonical_id for item in ranked[:2]) + captured = capture.capture_identity_input(phase="calibration", source=source) + + projection_index = source.accesses.index("pg19_projection:train") + first_text_index = next( + index for index, value in enumerate(source.accesses) if value.startswith("pg19_row:") + ) + assert projection_index < first_text_index + accessed_offsets = [ + int(value.rsplit(":", 1)[1]) + for value in source.accesses + if value.startswith("pg19_row:train:") + ] + assert accessed_offsets == [item.offset for item in ranked[:18]] + selected = [row for row in captured["records"] if row["family"] == "pg19"] + assert {row["canonical_id"] for row in selected}.isdisjoint(source.short_pg19_ids) + + +def test_pg19_stage_a_uses_frozen_hashed_4224_token_slice() -> None: + source = FakeSource() + captured = capture.capture_identity_input( + phase="stage_a", source=source, calibration_binding=_binding() + ) + tokenizer = FakeTokenizer() + rows = [row for row in captured["records"] if row["family"] == "pg19"] + for row in rows: + url = row["canonical_id"] + full_ids = tokenizer.encode(source.pg19_text("validation", url), add_special_tokens=False) + start = capture._segment_start( + namespace=capture.PG19_VALIDATION_SEGMENT_NAMESPACE, + canonical_id=url, + token_count=len(full_ids), + width=4_224, + ) + selected = full_ids[start : start + 4_224] + assert row["prompt_token_ids_sha256"] == capture._token_hash(selected[:4_096]) + assert row["target_token_ids_sha256"] == capture._token_hash(selected[4_096:]) + assert row["token_span"] == { + "prefill_start": 0, + "prefill_stop": 4_096, + "scored_start": 4_096, + "scored_stop": 4_224, + "cache_exposed_start": 4_097, + "cache_exposed_stop": 4_224, + } + + +def test_stage_a_ruler_requires_two_continuation_tokens() -> None: + source = FakeSource() + source.receipt_mutator = lambda receipt: ( + receipt.update({"outputs": ["x"]}) if receipt["config"] == "qa_1" else None + ) + + with pytest.raises(ValueError, match="continuation must contain at least two tokens"): + capture.capture_identity_input( + phase="stage_a", source=source, calibration_binding=_binding() + ) + + +def test_stage_a_humaneval_requires_two_continuation_tokens() -> None: + source = FakeSource() + original = source.humaneval_row + + def one_token_solution(*, offset: int, expected_task_id: str) -> dict[str, Any]: + row = original(offset=offset, expected_task_id=expected_task_id) + row["canonical_solution"] = "x" + return row + + source.humaneval_row = one_token_solution # type: ignore[method-assign] + + with pytest.raises(ValueError, match="continuation must contain at least two tokens"): + capture.capture_identity_input( + phase="stage_a", source=source, calibration_binding=_binding() + ) + + +@pytest.mark.parametrize("phase", ["stage_b", "stage_c"]) +def test_protected_phases_fail_before_any_source_access(phase: str) -> None: + source = FakeSource() + + with pytest.raises(PermissionError, match="before source access"): + capture.capture_identity_input(phase=phase, source=source) + + assert source.accesses == [] + + +def test_cli_rejects_protected_phase_before_paths_are_read(tmp_path: Path) -> None: + output = tmp_path / "must-not-exist.json" + + with pytest.raises(PermissionError, match="before file or source access"): + capture.main( + [ + "--phase", + "stage_b", + "--ruler-receipt-dir", + str(tmp_path / "missing-receipts"), + "--output", + str(output), + ] + ) + + assert not output.exists() + + +def test_source_head_drift_fails_after_capture() -> None: + source = FakeSource() + source.drift_after_capture = True + + with pytest.raises(ValueError, match="post-capture source HEAD"): + capture.capture_identity_input(phase="calibration", source=source) + + +def test_tokenizer_weight_file_is_rejected_before_dataset_content() -> None: + source = FakeSource() + source.extra_tokenizer_files["model.safetensors"] = b"not-a-real-weight" + + with pytest.raises(ValueError, match="model weight-like file is forbidden"): + capture.capture_identity_input(phase="calibration", source=source) + + assert not any(value == "mbpp_train_rows" for value in source.accesses) + + +def test_live_tokenizer_load_uses_only_authenticated_files_from_an_isolated_directory( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + snapshot = tmp_path / "shared-snapshot" + snapshot.mkdir() + expected = { + "tokenizer.json": b'{"version":"fixture"}', + "tokenizer_config.json": b'{"tokenizer_class":"Fixture"}', + } + for name, data in expected.items(): + (snapshot / name).write_bytes(data) + (snapshot / "tokenizer.model").write_bytes(b"unbound-stray-snapshot-file") + observed: dict[str, object] = {} + + class FakeApi: + @staticmethod + def list_repo_files(_repo_id: str, *, revision: str) -> list[str]: + assert revision == resolver.PRIMARY_MODEL_REVISION + return [*expected, "tokenizer.model", "model.safetensors"] + + def fake_download(*, repo_id: str, filename: str, revision: str, cache_dir: Path) -> str: + assert repo_id == resolver.PRIMARY_MODEL_ID + assert revision == resolver.PRIMARY_MODEL_REVISION + assert cache_dir == (tmp_path / "cache").resolve() + return str(snapshot / filename) + + class IsolatedTokenizer: + pass + + class FakeAutoTokenizer: + @classmethod + def from_pretrained(cls, path: Path, **kwargs: object) -> IsolatedTokenizer: + isolated = Path(path) + observed["path"] = isolated + observed["files"] = sorted( + item.relative_to(isolated).as_posix() + for item in isolated.rglob("*") + if item.is_file() + ) + observed["kwargs"] = kwargs + return IsolatedTokenizer() + + monkeypatch.setitem( + sys.modules, + "huggingface_hub", + SimpleNamespace(HfApi=FakeApi, hf_hub_download=fake_download), + ) + monkeypatch.setitem( + sys.modules, + "transformers", + SimpleNamespace(AutoTokenizer=FakeAutoTokenizer), + ) + source = capture.LiveCaptureSource( + cache_dir=tmp_path / "cache", + ruler_receipt_dir=tmp_path / "receipts", + ) + + material = source.tokenizer_material() + + assert material.files == expected + assert observed["files"] == sorted(expected) + assert observed["path"] != snapshot + assert observed["kwargs"] == { + "local_files_only": True, + "trust_remote_code": False, + } + assert not Path(observed["path"]).exists() + + +def test_ruler_receipt_category_drift_fails_closed() -> None: + source = FakeSource() + source.receipt_mutator = lambda receipt: receipt.update({"category": "retrieval"}) + + with pytest.raises(ValueError, match="receipt category drifted"): + capture.capture_identity_input(phase="calibration", source=source) + + +def test_ruler_generator_source_tamper_is_rejected() -> None: + source = FakeSource() + path = next(path for path in source.generator_files if path != "scripts/synthetic.yaml") + source.generator_files[path] += b"tamper" + + with pytest.raises(ValueError, match="generator Git blob drifted"): + capture.capture_identity_input(phase="calibration", source=source) + + +def test_duplicate_projection_identity_fails_closed() -> None: + source = FakeSource() + original = source.pg19_projection + + def duplicated(split: str) -> tuple[Any, ...]: + rows = list(original(split)) + rows[1] = capture.ProjectionRow(rows[0].canonical_id, rows[1].offset) + return tuple(rows) + + source.pg19_projection = duplicated # type: ignore[method-assign] + + with pytest.raises(ValueError, match="duplicate identity or offset"): + capture.capture_identity_input(phase="calibration", source=source) + + +def test_atomic_write_is_canonical_and_never_overwrites(tmp_path: Path) -> None: + path = tmp_path / "identity-input.json" + first = capture.canonical_json_bytes({"value": 1}) + second = capture.canonical_json_bytes({"value": 2}) + + capture.atomic_write_no_overwrite(path, first) + assert path.read_bytes() == first + with pytest.raises(FileExistsError, match="refusing to overwrite"): + capture.atomic_write_no_overwrite(path, second) + assert path.read_bytes() == first + assert not list(tmp_path.glob("*.tmp")) + + +def test_duplicate_json_keys_are_rejected() -> None: + with pytest.raises(ValueError, match="duplicate key"): + capture._strict_json(b'{"a":1,"a":2}', context="fixture") + + +def test_active_parquet_manifest_excludes_stale_repo_tree_files( + monkeypatch: pytest.MonkeyPatch, +) -> None: + payload = { + "parquet_files": [ + { + "dataset": resolver.PG19_DATASET_ID, + "config": "default", + "split": "train", + "url": ( + "https://huggingface.co/datasets/emozilla/pg19/resolve/" + "refs%2Fconvert%2Fparquet/default/partial-train/0001.parquet" + ), + "filename": "0001.parquet", + "size": 200, + }, + { + "dataset": resolver.PG19_DATASET_ID, + "config": "default", + "split": "train", + "url": ( + "https://huggingface.co/datasets/emozilla/pg19/resolve/" + "refs%2Fconvert%2Fparquet/default/partial-train/0000.parquet" + ), + "filename": "0000.parquet", + "size": 100, + }, + ], + "repo_tree_files": [ + "data/train-00022-of-00023-stale-sibling.parquet", + ], + } + + class Response(io.BytesIO): + headers = {"x-revision": resolver.PG19_REVISION} + + def __enter__(self) -> Response: + return self + + def __exit__(self, *_args: object) -> None: + self.close() + + def fake_urlopen(_request: object, timeout: int) -> Response: + assert timeout == 60 + return Response(json.dumps(payload).encode()) + + monkeypatch.setattr(capture.urllib.request, "urlopen", fake_urlopen) + aliases = capture.LiveCaptureSource._active_parquet_aliases( + dataset_id=resolver.PG19_DATASET_ID, + revision=resolver.PG19_REVISION, + config="default", + split="train", + ) + + assert aliases == ( + "datasets/emozilla/pg19@~parquet/default/partial-train/0000.parquet", + "datasets/emozilla/pg19@~parquet/default/partial-train/0001.parquet", + ) + assert all("stale" not in alias for alias in aliases) + + +def test_calibration_binding_requires_verified_artifact_and_is_normalized() -> None: + binding = _binding() + captured = capture.capture_identity_input( + phase="stage_a", source=FakeSource(), calibration_binding=binding + ) + + assert captured["calibration_binding"] == { + key: _hash(f"binding-{key}") for key in sorted(resolver.CALIBRATION_BINDING_FIELDS) + } + with pytest.raises(ValueError, match="verified artifact byte string"): + capture.capture_identity_input( + phase="stage_a", + source=FakeSource(), + calibration_binding={ + key: _hash(f"binding-{key}") for key in resolver.CALIBRATION_BINDING_FIELDS + }, # type: ignore[arg-type] + ) + + +def test_capture_output_contains_no_raw_model_or_weight_claim() -> None: + captured = capture.capture_identity_input(phase="calibration", source=FakeSource()) + serialized = capture.canonical_json_bytes(copy.deepcopy(captured)) + + assert b"model.safetensors" not in serialized + assert captured["model_weights_loaded"] is False + + +def test_required_ruler_receipt_inventory_is_exact_and_unique() -> None: + receipts = capture.required_ruler_receipts() + + assert len(receipts) == 20 + assert len({item["filename"] for item in receipts}) == 20 + assert sum(item["phase"] == "calibration" for item in receipts) == 16 + assert sum(item["phase"] == "stage_a" for item in receipts) == 4 + assert receipts[0]["filename"] == ("retrieval__niah_multiquery__l2048__s12339.json") + assert receipts[-1]["filename"] == ("question_answering__qa_1__l4096__s2339.json") diff --git a/tests/test_resolve_static_q468_identity.py b/tests/test_resolve_static_q468_identity.py index 700a368..a4779e4 100644 --- a/tests/test_resolve_static_q468_identity.py +++ b/tests/test_resolve_static_q468_identity.py @@ -3,8 +3,11 @@ import copy import importlib.util import json +import sys from pathlib import Path +from types import SimpleNamespace from typing import Any +from unittest.mock import patch import pytest @@ -13,6 +16,7 @@ SPEC = importlib.util.spec_from_file_location("resolve_static_q468_identity", SCRIPT_PATH) assert SPEC is not None and SPEC.loader is not None resolver = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = resolver SPEC.loader.exec_module(resolver) REVISIONS = { @@ -21,6 +25,14 @@ "ruler": resolver.RULER_REVISION, "humaneval_plus": resolver.HUMANEVAL_PLUS_REVISION, } +FIXTURE_BINDING_ARTIFACT = b"verified-stage-a-binding-fixture" +FIXTURE_BINDING = { + "calibration_identity_file_sha256": resolver.sha256_bytes(b"calibration-file"), + "calibration_score_artifact_file_sha256": resolver.sha256_bytes(b"calibration-scores"), + "split_half_stability_artifact_file_sha256": resolver.sha256_bytes(b"split-half"), + "static_k27030_policy_file_sha256": resolver.sha256_bytes(b"k27030-policy"), + "static_k29334_policy_file_sha256": resolver.sha256_bytes(b"k29334-policy"), +} def _hash(label: str) -> str: @@ -117,7 +129,16 @@ def _record( "humaneval_plus": resolver.HUMANEVAL_AB_NAMESPACE, }[family] label = f"{family}-{canonical_id}-{config}-{seed}-{sequence_length}" - return { + sequence_hash = _hash(f"sequence-tokens-{label}") + token_span = { + "prefill_start": 0, + "prefill_stop": prefill_stop, + "scored_start": prefill_stop, + "scored_stop": scored_stop, + "cache_exposed_start": prefill_stop + 1, + "cache_exposed_stop": scored_stop, + } + record = { "family": family, "canonical_id": canonical_id, "config": config, @@ -134,15 +155,28 @@ def _record( "formatted_content_sha256": _hash(f"formatted-{label}"), "prompt_token_ids_sha256": _hash(f"prompt-tokens-{label}"), "target_token_ids_sha256": _hash(f"target-tokens-{label}"), + "sequence_token_ids_sha256": sequence_hash, "tokenizer_manifest_sha256": _tokenizer_manifest_hash(), - "token_span": { - "prefill_start": 0, - "prefill_stop": prefill_stop, - "scored_start": prefill_stop, - "scored_stop": scored_stop, - }, - "anchor_manifest_sha256": _hash(f"anchors-{label}"), + "token_span": token_span, + "anchor_manifest_sha256": resolver.identity_anchor_manifest_sha256( + canonical_id=canonical_id, + sequence_length=sequence_length, + sequence_token_ids_sha256_value=sequence_hash, + token_span=token_span, + ), } + record["identity_record_sha256"] = resolver.identity_record_sha256(record) + return record + + +def _refresh_record_lineage(record: dict[str, Any]) -> None: + record["anchor_manifest_sha256"] = resolver.identity_anchor_manifest_sha256( + canonical_id=record["canonical_id"], + sequence_length=record["sequence_length"], + sequence_token_ids_sha256_value=record["sequence_token_ids_sha256"], + token_span=record["token_span"], + ) + record["identity_record_sha256"] = resolver.identity_record_sha256(record) def _stage_a_source() -> dict[str, Any]: @@ -201,6 +235,7 @@ def _stage_a_source() -> dict[str, Any]: ) for rank, row in enumerate(ranked): row["selection_rank"] = rank + _refresh_record_lineage(row) return { "schema": resolver.INPUT_SCHEMA, "phase": "stage_a", @@ -208,16 +243,26 @@ def _stage_a_source() -> dict[str, Any]: "tokenizer": _tokenizer(), "records": list(reversed(records)), "model_weights_loaded": False, - "calibration_binding": { - "identity_file_sha256": _hash("calibration-file"), - "canonical_identity_sha256": _hash("calibration-identity"), - "static_k29334_code_map_sha256": _hash("k29334"), - "static_k27030_code_map_sha256": _hash("k27030"), - "split_half_policy_manifest_sha256": _hash("split-half-policies"), - }, + "calibration_binding": dict(FIXTURE_BINDING), } +def _build_candidate(source: dict[str, Any]) -> dict[str, Any]: + if source.get("phase") != "stage_a": + return resolver.build_candidate(source, expected_revisions=REVISIONS) + verified = SimpleNamespace(binding=dict(FIXTURE_BINDING)) + with patch.object( + resolver, + "deserialize_stage_a_calibration_binding_artifact", + return_value=verified, + ): + return resolver.build_candidate( + source, + expected_revisions=REVISIONS, + calibration_binding_artifact=FIXTURE_BINDING_ARTIFACT, + ) + + def _write_json(path: Path, value: object) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(resolver.canonical_json_bytes(value)) @@ -225,8 +270,8 @@ def _write_json(path: Path, value: object) -> None: def test_stage_a_candidate_is_deterministic_and_complete() -> None: source = _stage_a_source() - first = resolver.build_candidate(source, expected_revisions=REVISIONS) - second = resolver.build_candidate(copy.deepcopy(source), expected_revisions=REVISIONS) + first = _build_candidate(source) + second = _build_candidate(copy.deepcopy(source)) assert first == second resolver.validate_candidate_artifact(first) @@ -249,12 +294,22 @@ def test_stage_a_candidate_is_deterministic_and_complete() -> None: ) +def test_stage_a_candidate_requires_and_matches_a_verified_binding_artifact() -> None: + source = _stage_a_source() + with pytest.raises(ValueError, match="requires a verified calibration binding"): + resolver.build_candidate(source, expected_revisions=REVISIONS) + + source["calibration_binding"]["static_k29334_policy_file_sha256"] = "0" * 64 + with pytest.raises(ValueError, match="differs from the verified artifact"): + _build_candidate(source) + + def test_raw_content_and_unknown_fields_fail_closed() -> None: source = _stage_a_source() source["records"][0]["prompt"] = "raw protected text" with pytest.raises(ValueError, match="fields drifted"): - resolver.build_candidate(source, expected_revisions=REVISIONS) + _build_candidate(source) @pytest.mark.parametrize( @@ -265,9 +320,7 @@ def test_raw_content_and_unknown_fields_fail_closed() -> None: "tokenizer revision", ), ( - lambda source: source["records"][0].update( - {"tokenizer_manifest_sha256": "0" * 64} - ), + lambda source: source["records"][0].update({"tokenizer_manifest_sha256": "0" * 64}), "tokenizer manifest binding", ), ( @@ -275,20 +328,22 @@ def test_raw_content_and_unknown_fields_fail_closed() -> None: "selection SHA-256", ), ( - lambda source: source["datasets"][1].update( - {"canonical_id_field": "book_id"} - ), + lambda source: source["datasets"][1].update({"canonical_id_field": "book_id"}), "pg19 canonical ID field", ), ( lambda source: source.update({"model_weights_loaded": True}), "before model weights", ), + ( + lambda source: source["records"][0]["token_span"].update({"scored_start": 4_095}), + "contiguous", + ), ( lambda source: source["records"][0]["token_span"].update( - {"scored_start": 4_095} + {"cache_exposed_start": source["records"][0]["token_span"]["scored_start"]} ), - "contiguous", + "exclude the first continuation token", ), ], ) @@ -297,7 +352,7 @@ def test_identity_contract_drift_fails_closed(mutation: Any, message: str) -> No mutation(source) with pytest.raises(ValueError, match=message): - resolver.build_candidate(source, expected_revisions=REVISIONS) + _build_candidate(source) def test_dataset_revision_must_match_explicit_cli_contract() -> None: @@ -305,7 +360,7 @@ def test_dataset_revision_must_match_explicit_cli_contract() -> None: source["datasets"][1]["revision"] = "9" * 40 with pytest.raises(ValueError, match="does not match the CLI contract"): - resolver.build_candidate(source, expected_revisions=REVISIONS) + _build_candidate(source) def test_ruler_category_config_and_actual_length_are_independently_bound() -> None: @@ -313,57 +368,117 @@ def test_ruler_category_config_and_actual_length_are_independently_bound() -> No ruler = next(row for row in source["records"] if row["family"] == "ruler") ruler["ruler_category"] = "aggregation" with pytest.raises(ValueError, match="config/category binding"): - resolver.build_candidate(source, expected_revisions=REVISIONS) + _build_candidate(source) source = _stage_a_source() ruler = next(row for row in source["records"] if row["family"] == "ruler") ruler["configured_length"] = 4_095 with pytest.raises(ValueError, match="exceeds the RULER configured length"): - resolver.build_candidate(source, expected_revisions=REVISIONS) + _build_candidate(source) source = _stage_a_source() pg19 = next(row for row in source["records"] if row["family"] == "pg19") pg19["ruler_category"] = "retrieval" with pytest.raises(ValueError, match="non-RULER rows"): - resolver.build_candidate(source, expected_revisions=REVISIONS) + _build_candidate(source) + + +def test_stage_a_requires_two_continuation_tokens_for_one_cache_prediction() -> None: + source = _stage_a_source() + row = source["records"][0] + stop = row["token_span"]["scored_stop"] + row["token_span"].update( + { + "prefill_stop": stop - 1, + "scored_start": stop - 1, + "cache_exposed_start": stop, + "cache_exposed_stop": stop, + } + ) + + with pytest.raises(ValueError, match="continuation must contain at least two"): + _build_candidate(source) + + +def test_calibration_cache_exposure_is_empty_at_continuation_stop() -> None: + source = _stage_a_source() + row = copy.deepcopy(next(item for item in source["records"] if item["family"] == "pg19")) + row["selection_sha256"] = resolver.selection_sha256( + resolver.PG19_TRAIN_NAMESPACE, row["canonical_id"] + ) + stop = row["token_span"]["scored_stop"] + row["token_span"].update({"cache_exposed_start": stop, "cache_exposed_stop": stop}) + _refresh_record_lineage(row) + + normalized = resolver._normalize_record( + row, + index=0, + phase="calibration", + tokenizer_hash=_tokenizer_manifest_hash(), + ) + assert normalized["token_span"]["cache_exposed_start"] == stop + assert normalized["token_span"]["cache_exposed_stop"] == stop + + row["token_span"]["cache_exposed_start"] = stop - 1 + with pytest.raises(ValueError, match="calibration cache-exposed prediction span"): + resolver._normalize_record( + row, + index=0, + phase="calibration", + tokenizer_hash=_tokenizer_manifest_hash(), + ) def test_dry_run_writes_nothing(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: source_path = tmp_path / "source.json" _write_json(source_path, _stage_a_source()) - - result = resolver.main( - [ - "--phase", - "stage_a", - "--input", - str(source_path), - "--dry-run", - "--mbpp-revision", - REVISIONS["mbpp"], - "--pg19-revision", - REVISIONS["pg19"], - "--ruler-revision", - REVISIONS["ruler"], - "--humaneval-plus-revision", - REVISIONS["humaneval_plus"], - ] - ) + binding_path = tmp_path / "binding.json" + binding_path.write_bytes(FIXTURE_BINDING_ARTIFACT) + + verified = SimpleNamespace(binding=dict(FIXTURE_BINDING)) + with patch.object( + resolver, + "deserialize_stage_a_calibration_binding_artifact", + return_value=verified, + ): + result = resolver.main( + [ + "--phase", + "stage_a", + "--input", + str(source_path), + "--calibration-binding", + str(binding_path), + "--dry-run", + "--mbpp-revision", + REVISIONS["mbpp"], + "--pg19-revision", + REVISIONS["pg19"], + "--ruler-revision", + REVISIONS["ruler"], + "--humaneval-plus-revision", + REVISIONS["humaneval_plus"], + ] + ) assert result == 0 assert len(capsys.readouterr().out.strip()) == 64 - assert sorted(path.name for path in tmp_path.iterdir()) == ["source.json"] + assert sorted(path.name for path in tmp_path.iterdir()) == ["binding.json", "source.json"] def test_candidate_requires_quarantine_then_exact_hash_promotion(tmp_path: Path) -> None: source_path = tmp_path / "source.json" _write_json(source_path, _stage_a_source()) + binding_path = tmp_path / "binding.json" + binding_path.write_bytes(FIXTURE_BINDING_ARTIFACT) candidate_path = tmp_path / ".quarantine" / "stage-a-candidate.json" base_args = [ "--phase", "stage_a", "--input", str(source_path), + "--calibration-binding", + str(binding_path), "--mbpp-revision", REVISIONS["mbpp"], "--pg19-revision", @@ -374,10 +489,177 @@ def test_candidate_requires_quarantine_then_exact_hash_promotion(tmp_path: Path) REVISIONS["humaneval_plus"], ] - assert resolver.main([*base_args, "--output", str(candidate_path)]) == 0 - candidate_hash = resolver.sha256_bytes(candidate_path.read_bytes()) - frozen_path = tmp_path / "frozen" / "stage-a-identity.json" - assert ( + verified = SimpleNamespace(binding=dict(FIXTURE_BINDING)) + with patch.object( + resolver, + "deserialize_stage_a_calibration_binding_artifact", + return_value=verified, + ): + assert resolver.main([*base_args, "--output", str(candidate_path)]) == 0 + candidate_hash = resolver.sha256_bytes(candidate_path.read_bytes()) + frozen_path = tmp_path / "frozen" / "stage-a-identity.json" + assert ( + resolver.main( + [ + "--phase", + "stage_a", + "--input", + str(candidate_path), + "--output", + str(frozen_path), + "--promote", + "--calibration-binding", + str(binding_path), + "--expected-candidate-sha256", + candidate_hash, + ] + ) + == 0 + ) + frozen = json.loads(frozen_path.read_text(encoding="utf-8")) + assert frozen["evidence"]["status"] == "frozen" + assert frozen["evidence"]["promotion"]["candidate_file_sha256"] == candidate_hash + assert frozen["evidence"]["model_contracts"]["weights_loaded"] is False + + +def test_stage_a_promotion_requires_the_verified_binding_artifact() -> None: + candidate = _build_candidate(_stage_a_source()) + candidate_hash = resolver.sha256_bytes(resolver.canonical_json_bytes(candidate)) + + with pytest.raises(ValueError, match="promotion requires a verified calibration binding"): + resolver.promote_candidate(candidate, candidate_file_sha256=candidate_hash) + + verified = SimpleNamespace(binding=dict(FIXTURE_BINDING)) + with patch.object( + resolver, + "deserialize_stage_a_calibration_binding_artifact", + return_value=verified, + ): + frozen = resolver.promote_candidate( + candidate, + candidate_file_sha256=candidate_hash, + calibration_binding_artifact=FIXTURE_BINDING_ARTIFACT, + ) + + assert frozen["evidence"]["status"] == "frozen" + + +def test_frozen_stage_a_decoder_reauthenticates_promotion_records_and_binding() -> None: + candidate = _build_candidate(_stage_a_source()) + candidate_hash = resolver.sha256_bytes(resolver.canonical_json_bytes(candidate)) + verified = SimpleNamespace(binding=dict(FIXTURE_BINDING)) + with patch.object( + resolver, + "deserialize_stage_a_calibration_binding_artifact", + return_value=verified, + ): + frozen = resolver.promote_candidate( + candidate, + candidate_file_sha256=candidate_hash, + calibration_binding_artifact=FIXTURE_BINDING_ARTIFACT, + ) + frozen_bytes = resolver.canonical_json_bytes(frozen) + decoded = resolver.deserialize_frozen_stage_a_identity_artifact( + frozen_bytes, + calibration_binding_artifact=FIXTURE_BINDING_ARTIFACT, + ) + + assert decoded.file_sha256 == resolver.sha256_bytes(frozen_bytes) + assert len(decoded.records) == 12 + assert decoded.calibration_binding == FIXTURE_BINDING + + tampered = copy.deepcopy(frozen) + tampered["evidence"]["records"][0]["source_content_sha256"] = "0" * 64 + tampered["canonical_evidence_sha256"] = resolver.sha256_bytes( + resolver.canonical_json_bytes(tampered["evidence"]) + ) + with ( + patch.object( + resolver, + "deserialize_stage_a_calibration_binding_artifact", + return_value=verified, + ), + pytest.raises(ValueError, match="identity record SHA-256 drifted"), + ): + resolver.deserialize_frozen_stage_a_identity_artifact( + resolver.canonical_json_bytes(tampered), + calibration_binding_artifact=FIXTURE_BINDING_ARTIFACT, + ) + + wrong_binding = dict(FIXTURE_BINDING) + wrong_binding["static_k29334_policy_file_sha256"] = "0" * 64 + with ( + patch.object( + resolver, + "deserialize_stage_a_calibration_binding_artifact", + return_value=SimpleNamespace(binding=wrong_binding), + ), + pytest.raises(ValueError, match="differs from the verified calibration binding"), + ): + resolver.deserialize_frozen_stage_a_identity_artifact( + frozen_bytes, + calibration_binding_artifact=FIXTURE_BINDING_ARTIFACT, + ) + + tampered = copy.deepcopy(frozen) + tampered["evidence"]["promotion"]["candidate_file_sha256"] = "0" * 64 + tampered["canonical_evidence_sha256"] = resolver.sha256_bytes( + resolver.canonical_json_bytes(tampered["evidence"]) + ) + with ( + patch.object( + resolver, + "deserialize_stage_a_calibration_binding_artifact", + return_value=verified, + ), + pytest.raises(ValueError, match="candidate file SHA-256 drifted"), + ): + resolver.deserialize_frozen_stage_a_identity_artifact( + resolver.canonical_json_bytes(tampered), + calibration_binding_artifact=FIXTURE_BINDING_ARTIFACT, + ) + + +def test_handcrafted_incomplete_candidate_cannot_be_promoted() -> None: + evidence = { + "identity_schema": resolver.CANDIDATE_SCHEMA, + "status": "candidate", + "phase": "stage_a", + "identity_only": True, + "claim_boundary": resolver.CLAIM_BOUNDARY, + "promotion_required": True, + "model_contracts": {"weights_loaded": False}, + "records": [], + "record_count": 0, + "content_manifest_sha256": resolver.sha256_bytes(resolver.canonical_json_bytes([])), + "protected_identity": { + "stage_b_read": False, + "stage_c_read": False, + "ordinary_tests_may_read_protected_content": False, + }, + } + candidate = { + "canonical_evidence_sha256": resolver.sha256_bytes(resolver.canonical_json_bytes(evidence)), + "evidence": evidence, + } + + with pytest.raises(ValueError, match="candidate evidence fields drifted"): + resolver.promote_candidate( + candidate, + candidate_file_sha256=resolver.sha256_bytes(resolver.canonical_json_bytes(candidate)), + calibration_binding_artifact=FIXTURE_BINDING_ARTIFACT, + ) + + +def test_noncanonical_candidate_bytes_cannot_be_promoted(tmp_path: Path) -> None: + candidate = _build_candidate(_stage_a_source()) + candidate_path = tmp_path / ".quarantine" / "candidate.json" + candidate_path.parent.mkdir(parents=True) + candidate_path.write_text(json.dumps(candidate, indent=2), encoding="utf-8") + binding_path = tmp_path / "binding.json" + binding_path.write_bytes(FIXTURE_BINDING_ARTIFACT) + + with pytest.raises(ValueError, match="not canonical resolver JSON"): resolver.main( [ "--phase", @@ -385,24 +667,49 @@ def test_candidate_requires_quarantine_then_exact_hash_promotion(tmp_path: Path) "--input", str(candidate_path), "--output", - str(frozen_path), + str(tmp_path / "frozen" / "identity.json"), "--promote", + "--calibration-binding", + str(binding_path), "--expected-candidate-sha256", - candidate_hash, + resolver.sha256_bytes(candidate_path.read_bytes()), ] ) - == 0 - ) - frozen = json.loads(frozen_path.read_text(encoding="utf-8")) - assert frozen["evidence"]["status"] == "frozen" - assert frozen["evidence"]["promotion"]["candidate_file_sha256"] == candidate_hash - assert frozen["evidence"]["model_contracts"]["weights_loaded"] is False + + +def test_atomic_identity_publish_cannot_overwrite_an_existing_or_racing_file( + tmp_path: Path, +) -> None: + existing = tmp_path / "existing.json" + existing.write_bytes(b"existing") + with pytest.raises(FileExistsError, match="refusing to overwrite"): + resolver.atomic_write(existing, b"replacement") + assert existing.read_bytes() == b"existing" + + racing = tmp_path / "racing.json" + + def create_racing_destination(_source: object, destination: object) -> None: + Path(destination).write_bytes(b"racer") + raise FileExistsError + + with ( + patch.object(resolver.os, "link", side_effect=create_racing_destination), + pytest.raises( + FileExistsError, + match="refusing to overwrite", + ), + ): + resolver.atomic_write(racing, b"candidate") + assert racing.read_bytes() == b"racer" + assert list(tmp_path.glob(".*.tmp")) == [] def test_candidate_wrong_hash_cannot_be_promoted(tmp_path: Path) -> None: - candidate = resolver.build_candidate(_stage_a_source(), expected_revisions=REVISIONS) + candidate = _build_candidate(_stage_a_source()) candidate_path = tmp_path / ".quarantine" / "candidate.json" _write_json(candidate_path, candidate) + binding_path = tmp_path / "binding.json" + binding_path.write_bytes(FIXTURE_BINDING_ARTIFACT) with pytest.raises(ValueError, match="does not match explicit promotion hash"): resolver.main( @@ -414,6 +721,8 @@ def test_candidate_wrong_hash_cannot_be_promoted(tmp_path: Path) -> None: "--output", str(tmp_path / "frozen" / "identity.json"), "--promote", + "--calibration-binding", + str(binding_path), "--expected-candidate-sha256", "0" * 64, ] @@ -429,7 +738,7 @@ def test_protected_phase_is_rejected_before_input_read(tmp_path: Path, phase: st def test_candidate_tampering_is_detected() -> None: - candidate = resolver.build_candidate(_stage_a_source(), expected_revisions=REVISIONS) + candidate = _build_candidate(_stage_a_source()) candidate["evidence"]["records"][0]["source_content_sha256"] = "0" * 64 with pytest.raises(ValueError, match="canonical evidence SHA-256"): From 8f2fec940a72a1950943fbf7bcbe2692744b0409 Mon Sep 17 00:00:00 2001 From: HiddenTrojan <93521146+Labeeb2339@users.noreply.github.com> Date: Sun, 2 Aug 2026 11:41:37 +0800 Subject: [PATCH 09/24] Authenticate RULER receipt generation --- requirements/experiment013-ruler.txt | 40 + .../generate_static_q468_ruler_receipts.py | 1287 +++++++++++++++++ ...est_generate_static_q468_ruler_receipts.py | 268 ++++ 3 files changed, 1595 insertions(+) create mode 100644 requirements/experiment013-ruler.txt create mode 100644 scripts/generate_static_q468_ruler_receipts.py create mode 100644 tests/test_generate_static_q468_ruler_receipts.py diff --git a/requirements/experiment013-ruler.txt b/requirements/experiment013-ruler.txt new file mode 100644 index 0000000..680c107 --- /dev/null +++ b/requirements/experiment013-ruler.txt @@ -0,0 +1,40 @@ +# Complete tokenizer-only environment used to generate the frozen Experiment +# 013 RULER receipts. PyTorch is intentionally absent: this environment must +# not load model weights. +annotated-doc==0.0.5 +anyio==4.14.2 +beautifulsoup4==4.15.0 +certifi==2026.7.22 +click==8.4.2 +colorama==0.4.6 +defusedxml==0.7.1 +filelock==3.32.2 +fsspec==2026.7.0 +h11==0.16.0 +hf-xet==1.5.2 +html2text==2025.4.15 +httpcore==1.0.9 +httpx==0.28.1 +huggingface-hub==1.26.0 +idna==3.18 +joblib==1.5.3 +markdown-it-py==4.2.0 +mdurl==0.1.2 +nltk==3.8.1 +numpy==2.4.6 +packaging==26.2 +pygments==2.20.0 +PyYAML==6.0.3 +regex==2026.7.19 +rich==15.0.0 +safetensors==0.8.0 +scipy==1.17.1 +shellingham==1.5.4 +soupsieve==2.9.1 +tenacity==9.1.4 +tokenizers==0.22.2 +tqdm==4.70.0 +transformers==5.14.1 +typer==0.27.0 +typing-extensions==4.16.0 +wonderwords==3.0.1 diff --git a/scripts/generate_static_q468_ruler_receipts.py b/scripts/generate_static_q468_ruler_receipts.py new file mode 100644 index 0000000..1aca275 --- /dev/null +++ b/scripts/generate_static_q468_ruler_receipts.py @@ -0,0 +1,1287 @@ +#!/usr/bin/env python3 +"""Generate authenticated Experiment 013 RULER receipts without model weights. + +NVIDIA RULER's pinned ``prepare.py`` builds a multiline shell string. On +Windows that string truncates the multiline task template while still exiting +zero and writing a malformed row. This launcher verifies the pinned upstream +files, then invokes the same pinned task scripts with an argument vector. It +independently checks the generated row, tokenizer count, runtime, corpora, and +package resources before atomically publishing a receipt. +""" + +from __future__ import annotations + +import argparse +import ast +import hashlib +import importlib.util +import json +import os +import re +import subprocess +import sys +import tempfile +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Final + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +CAPTURE_PATH = REPOSITORY_ROOT / "scripts" / "capture_static_q468_identity_input.py" + +LAUNCHER_REVISION: Final = "experiment-013-ruler-argv-launcher-v2" +RUNTIME_PYTHON_VERSION: Final = "3.11.15" +RUNTIME_PACKAGES: Final = { + "PyYAML": "6.0.3", + "annotated-doc": "0.0.5", + "anyio": "4.14.2", + "beautifulsoup4": "4.15.0", + "certifi": "2026.7.22", + "click": "8.4.2", + "colorama": "0.4.6", + "defusedxml": "0.7.1", + "filelock": "3.32.2", + "fsspec": "2026.7.0", + "h11": "0.16.0", + "hf-xet": "1.5.2", + "html2text": "2025.4.15", + "httpcore": "1.0.9", + "httpx": "0.28.1", + "huggingface-hub": "1.26.0", + "idna": "3.18", + "joblib": "1.5.3", + "markdown-it-py": "4.2.0", + "mdurl": "0.1.2", + "nltk": "3.8.1", + "numpy": "2.4.6", + "packaging": "26.2", + "pygments": "2.20.0", + "regex": "2026.7.19", + "rich": "15.0.0", + "safetensors": "0.8.0", + "scipy": "1.17.1", + "shellingham": "1.5.4", + "soupsieve": "2.9.1", + "tenacity": "9.1.4", + "tokenizers": "0.22.2", + "tqdm": "4.70.0", + "transformers": "5.14.1", + "typer": "0.27.0", + "typing-extensions": "4.16.0", + "wonderwords": "3.0.1", +} + +EXPECTED_CORPORA: Final = { + "PaulGrahamEssays.json": ( + 3_108_621, + "8d31e1b660e0f2180bcca6d238e18f77921df9d158611582b860da1762b6d3dd", + ), + "english_words.json": ( + 8_564_991, + "affcd6d45fdf3cc843d585c99c97ad615094e760e6c4756b654bab6c73bc2eca", + ), + "hotpotqa.json": ( + 61_065_698, + "e3da074df24e8369009918aa5cdbdd254dadcde4c63f7569d36afd6f2268caa8", + ), + "squad.json": ( + 4_370_528, + "80a5225e94905956a6446d296ca1093975c4d3b3260f1d6c8f68bc2ab77182d8", + ), +} +EXPECTED_PACKAGE_RESOURCES: Final = { + "nltk/punkt/PY3/english.pickle": ( + 406_697, + "5cad3758596392364e3be9803dbd7ebeda384b68937b488a01365f5551bb942c", + ), + "wonderwords/adjectivelist.txt": ( + 7_403, + "66814d46b7e292c83e839d12fe2fa083c8f66779b14b294e1be4e1342c5d4131", + ), + "wonderwords/nounlist.txt": ( + 54_752, + "8d88b3ebc2e2969ed92ebe49cdb0c8a87c2ad3b27493d7afdff608f2e51c5bc9", + ), + "wonderwords/verblist.txt": ( + 6_967, + "9fbf5e4e69b8869aebd88e6d545312ff9f878a747327ede5f6dbbba5c27b08ad", + ), +} +EXPECTED_TOKENIZER_ASSETS: Final = { + "merges.txt": ( + 3_353_259, + "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d", + ), + "tokenizer.json": ( + 12_807_196, + "fe000e3ed39ed12b8d2481d527d44f93c65d37e87645d2dcc80d1bf9d50d2927", + ), + "tokenizer_config.json": ( + 16_712, + "e611fbccc7c29ef3b1cafb1cb7ea548d189968632901d678fd62be68c47885de", + ), + "vocab.json": ( + 6_722_759, + "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003", + ), +} + +NIAH_TEMPLATE: Final = ( + "Some special magic {type_needle_v} are hidden within the following text. " + "Make sure to memorize it. I will quiz you about the {type_needle_v} afterwards.\n" + "{context}\nWhat are all the special magic {type_needle_v} for {query} mentioned " + "in the provided text? The special magic {type_needle_v} for {query} mentioned " + "in the provided text are" +) +VT_TEMPLATE: Final = ( + "Memorize and track the chain(s) of variable assignment hidden in the following " + "text.\n\n{context}\nQuestion: Find all variables that are assigned the value {query} " + "in the text above. Answer: According to the chain(s) of variable assignment in " + "the text above, {num_v} variables are assigned the value {query}, they are: " +) +CWE_TEMPLATE: Final = ( + "Below is a numbered list of words. In these words, some appear more often than " + "others. Memorize the ones that appear most often.\n{context}\nQuestion: What are " + "the 10 most common words in the above list? Answer: The top 10 words that appear " + "most often in the list are:" +) +FWE_TEMPLATE: Final = ( + "Read the following coded text and track the frequency of each coded word. Find the " + "three most frequently appeared coded words. {context}\nQuestion: Do not provide any " + "explanation. Please ignore the dots '....'. What are the three most frequently " + "appeared words in the above coded text? Answer: According to the coded text above, " + "the three most frequently appeared words are:" +) +QA_TEMPLATE: Final = ( + "Answer the question based on the given documents. Only give me the answer and do " + "not output any other words.\n\nThe following are given documents.\n\n{context}\n\n" + "Answer the question based on the given documents. Only give me the answer and do " + "not output any other words.\n\nQuestion: {query} Answer:" +) + + +@dataclass(frozen=True, slots=True) +class TaskSpec: + script: str + tokens_to_generate: int + template: str + arguments: tuple[tuple[str, str], ...] + input_marker: str + answer_prefix_marker: str + expected_output_count: int | None + unique_outputs: bool + output_pattern: str | None = None + outputs_must_appear: bool = False + niah: bool = False + + +TASK_SPECS: Final = { + "niah_multiquery": TaskSpec( + "niah.py", + 128, + NIAH_TEMPLATE, + ( + ("type_haystack", "essay"), + ("type_needle_k", "words"), + ("type_needle_v", "numbers"), + ("num_needle_k", "1"), + ("num_needle_v", "1"), + ("num_needle_q", "4"), + ), + "What are all the special magic numbers", + " The special magic numbers", + 4, + True, + r"[0-9]{7}", + True, + True, + ), + "niah_multikey_2": TaskSpec( + "niah.py", + 128, + NIAH_TEMPLATE, + ( + ("type_haystack", "needle"), + ("type_needle_k", "words"), + ("type_needle_v", "numbers"), + ("num_needle_k", "1"), + ("num_needle_v", "1"), + ("num_needle_q", "1"), + ), + "What is the special magic number", + " The special magic number", + 1, + True, + r"[0-9]{7}", + True, + True, + ), + "niah_single_1": TaskSpec( + "niah.py", + 128, + NIAH_TEMPLATE, + ( + ("type_haystack", "noise"), + ("type_needle_k", "words"), + ("type_needle_v", "numbers"), + ("num_needle_k", "1"), + ("num_needle_v", "1"), + ("num_needle_q", "1"), + ), + "What is the special magic number", + " The special magic number", + 1, + True, + r"[0-9]{7}", + True, + True, + ), + "niah_multivalue": TaskSpec( + "niah.py", + 128, + NIAH_TEMPLATE, + ( + ("type_haystack", "essay"), + ("type_needle_k", "words"), + ("type_needle_v", "numbers"), + ("num_needle_k", "1"), + ("num_needle_v", "4"), + ("num_needle_q", "1"), + ), + "What are all the special magic numbers", + " The special magic numbers", + 4, + True, + r"[0-9]{7}", + True, + True, + ), + "vt": TaskSpec( + "variable_tracking.py", + 30, + VT_TEMPLATE, + (("type_haystack", "noise"), ("num_chains", "1"), ("num_hops", "4")), + "Question: Find all variables", + " Answer:", + 5, + True, + r"[A-Z]{5}", + True, + ), + "cwe": TaskSpec( + "common_words_extraction.py", + 120, + CWE_TEMPLATE, + (("freq_cw", "30"), ("freq_ucw", "3"), ("num_cw", "10")), + "Question: What are the 10 most common words", + " Answer:", + 10, + True, + None, + True, + ), + "fwe": TaskSpec( + "freq_words_extraction.py", + 50, + FWE_TEMPLATE, + (("alpha", "2.0"),), + "What are the three most frequently appeared words", + " Answer:", + 3, + True, + r"[a-z]{6}", + True, + ), + "qa_1": TaskSpec( + "qa.py", + 32, + QA_TEMPLATE, + (("dataset", "squad"), ("pre_samples", "0")), + "The following are given documents.", + " Answer:", + None, + False, + ), + "qa_2": TaskSpec( + "qa.py", + 32, + QA_TEMPLATE, + (("dataset", "hotpotqa"), ("pre_samples", "0")), + "The following are given documents.", + " Answer:", + None, + False, + ), +} + +SOURCE_SCRIPT_BY_TASK: Final = { + "niah": "niah.py", + "variable_tracking": "variable_tracking.py", + "common_words_extraction": "common_words_extraction.py", + "freq_words_extraction": "freq_words_extraction.py", + "qa": "qa.py", +} +LAUNCHER_ONLY_ARGUMENTS: Final = {"qa": {"pre_samples": "0"}} +FORBIDDEN_RUNTIME_MODULES: Final = ( + "accelerate", + "bitsandbytes", + "flax", + "jax", + "onnx", + "onnxruntime", + "tensorflow", + "torch", +) + + +def _load_capture_module() -> Any: + name = "_recurquant_experiment013_ruler_capture" + spec = importlib.util.spec_from_file_location(name, CAPTURE_PATH) + if spec is None or spec.loader is None: # pragma: no cover + raise RuntimeError("cannot load the Experiment 013 capture module") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + try: + spec.loader.exec_module(module) + except BaseException: + sys.modules.pop(name, None) + raise + return module + + +def _canonical_json_bytes(value: object) -> bytes: + return ( + json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + b"\n" + ) + + +def _sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _canonical_distribution_name(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +def _subprocess_env(**updates: str) -> dict[str, str]: + """Return a child environment without caller-controlled Python injection.""" + + env = {key: value for key, value in os.environ.items() if not key.upper().startswith("PYTHON")} + env.update(updates) + env["PYTHONNOUSERSITE"] = "1" + return env + + +def _file_entry(name: str, data: bytes) -> dict[str, object]: + if not data: + raise ValueError(f"bound file {name!r} is empty") + return {"name": name, "sha256": _sha256_bytes(data), "size_bytes": len(data)} + + +def _verified_file(path: Path, *, size: int, sha256: str, name: str) -> bytes: + if not path.is_file(): + raise FileNotFoundError(f"missing required {name}: {path}") + data = path.read_bytes() + if len(data) != size or _sha256_bytes(data) != sha256: + raise ValueError(f"{name} differs from the frozen Experiment 013 bytes") + return data + + +def _strict_json(data: bytes, *, context: str) -> Any: + def pairs(values: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in values: + if key in result: + raise ValueError(f"{context} contains duplicate key {key!r}") + result[key] = value + return result + + try: + return json.loads(data.decode("utf-8"), object_pairs_hook=pairs) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError(f"{context} is not strict UTF-8 JSON") from error + + +def _git_blob_sha1(data: bytes) -> str: + prefix = f"blob {len(data)}\0".encode() + return hashlib.sha1(prefix + data, usedforsecurity=False).hexdigest() + + +def _source_tasks(constants_data: bytes) -> Mapping[str, object]: + try: + tree = ast.parse(constants_data.decode("utf-8")) + except (UnicodeDecodeError, SyntaxError) as error: + raise ValueError("pinned RULER constants.py is not valid UTF-8 Python") from error + assignments = [ + node + for node in tree.body + if isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == "TASKS" for target in node.targets) + ] + if len(assignments) != 1: + raise ValueError("pinned RULER constants.py must assign TASKS exactly once") + try: + value = ast.literal_eval(assignments[0].value) + except (TypeError, ValueError) as error: + raise ValueError("pinned RULER TASKS must be a literal mapping") from error + if not isinstance(value, Mapping): + raise ValueError("pinned RULER TASKS must be a mapping") + return value + + +def _verify_task_specs_against_source(*, synthetic_yaml: bytes, constants_py: bytes) -> None: + import yaml + + try: + configs = yaml.safe_load(synthetic_yaml.decode("utf-8")) + except (UnicodeDecodeError, yaml.YAMLError) as error: + raise ValueError("pinned RULER synthetic.yaml is invalid") from error + if not isinstance(configs, Mapping): + raise ValueError("pinned RULER synthetic.yaml must be a mapping") + tasks = _source_tasks(constants_py) + for config, spec in TASK_SPECS.items(): + source_config = configs.get(config) + if not isinstance(source_config, Mapping) or set(source_config) != {"task", "args"}: + raise ValueError(f"pinned RULER config {config!r} has an unexpected shape") + task = source_config["task"] + arguments = source_config["args"] + if not isinstance(task, str) or not isinstance(arguments, Mapping): + raise ValueError(f"pinned RULER config {config!r} is malformed") + if SOURCE_SCRIPT_BY_TASK.get(task) != spec.script: + raise ValueError(f"launcher script differs from pinned config {config!r}") + expected_arguments = {str(key): str(value) for key, value in arguments.items()} + launcher_arguments = dict(spec.arguments) + if len(launcher_arguments) != len(spec.arguments): + raise ValueError(f"launcher arguments repeat a key for config {config!r}") + extras = LAUNCHER_ONLY_ARGUMENTS.get(task, {}) + if launcher_arguments != expected_arguments | extras: + raise ValueError(f"launcher arguments differ from pinned config {config!r}") + source_task = tasks.get(task) + if not isinstance(source_task, Mapping): + raise ValueError(f"pinned RULER task {task!r} is missing") + if source_task.get("tokens_to_generate") != spec.tokens_to_generate: + raise ValueError(f"launcher token budget differs from pinned task {task!r}") + template = source_task.get("template") + answer_prefix = source_task.get("answer_prefix", "") + if not isinstance(template, str) or not isinstance(answer_prefix, str): + raise ValueError(f"pinned RULER task {task!r} has malformed templates") + if template + answer_prefix != spec.template: + raise ValueError(f"launcher template differs from pinned task {task!r}") + + +def _requirements_bytes() -> bytes: + path = REPOSITORY_ROOT / "requirements" / "experiment013-ruler.txt" + data = path.read_bytes() + pinned: dict[str, str] = {} + for line in data.decode("utf-8").splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + if stripped.count("==") != 1: + raise ValueError("RULER runtime requirements must use exact == pins") + name, version = stripped.split("==", 1) + if not name or not version or name in pinned: + raise ValueError("RULER runtime requirements contain an invalid pin") + pinned[name] = version + if pinned != RUNTIME_PACKAGES: + raise ValueError("RULER runtime requirements differ from the probe contract") + return data + + +def _launcher_source_entry() -> dict[str, object]: + return _file_entry( + "launcher/generate_static_q468_ruler_receipts.py", Path(__file__).read_bytes() + ) + + +def verify_ruler_checkout(ruler_root: Path, capture: Any) -> list[dict[str, object]]: + ruler_root = ruler_root.resolve() + result = subprocess.run( + ["git", "-C", str(ruler_root), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ) + head = result.stdout.strip() + if head != capture.resolver.RULER_REVISION: + raise ValueError("RULER checkout HEAD differs from the frozen revision") + files: dict[str, bytes] = {} + for relative, expected_blob in capture.RULER_GENERATOR_GIT_BLOBS.items(): + path = ruler_root / relative + if not path.is_file(): + raise FileNotFoundError(f"missing pinned RULER source file: {relative}") + # Git may legitimately smudge LF blobs to CRLF in a Windows checkout. + # Hash the worktree path through its clean filters, then bind the exact + # immutable object bytes used by the capture contract. + worktree_hash = subprocess.run( + [ + "git", + "-C", + str(ruler_root), + "hash-object", + f"--path={relative}", + str(path), + ], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + if worktree_hash != expected_blob: + raise ValueError(f"pinned RULER source file drifted: {relative}") + data = subprocess.run( + ["git", "-C", str(ruler_root), "cat-file", "blob", expected_blob], + check=True, + capture_output=True, + ).stdout + if _git_blob_sha1(data) != expected_blob: + raise RuntimeError(f"Git returned corrupt blob bytes for {relative}") + files[relative] = data + _verify_task_specs_against_source( + synthetic_yaml=files["scripts/synthetic.yaml"], + constants_py=files["scripts/data/synthetic/constants.py"], + ) + return capture._ruler_generator_manifest(files) + + +def verify_runtime(python: Path, nltk_data: Path) -> tuple[dict[str, object], dict[str, Path]]: + code = """ +import importlib.metadata as metadata +import importlib.util +import json +import pathlib +import platform +import re +import sys +import nltk +import wonderwords +names = __PACKAGE_NAMES__ +forbidden = __FORBIDDEN_MODULES__ +root = pathlib.Path(wonderwords.__file__).resolve().parent +canonical = lambda name: re.sub(r'[-_.]+', '-', name).lower() +payload = { + 'python': sys.version.split()[0], + 'implementation': sys.implementation.name, + 'cache_tag': sys.implementation.cache_tag, + 'executable': str(pathlib.Path(sys.executable).resolve()), + 'platform': platform.platform(), + 'flags': { + 'ignore_environment': sys.flags.ignore_environment, + 'isolated': sys.flags.isolated, + 'no_user_site': sys.flags.no_user_site, + }, + 'packages': {name: metadata.version(name) for name in names}, + 'installed_distributions': { + canonical(dist.metadata['Name']): dist.version for dist in metadata.distributions() + }, + 'forbidden_modules': { + name: importlib.util.find_spec(name) is not None for name in forbidden + }, + 'resources': { + 'nltk/punkt/PY3/english.pickle': str(nltk.data.find('tokenizers/punkt/english.pickle')), + 'wonderwords/adjectivelist.txt': str(root / 'assets' / 'adjectivelist.txt'), + 'wonderwords/nounlist.txt': str(root / 'assets' / 'nounlist.txt'), + 'wonderwords/verblist.txt': str(root / 'assets' / 'verblist.txt'), + }, +} +print(json.dumps(payload, sort_keys=True, separators=(',', ':'))) +""".replace("__PACKAGE_NAMES__", repr(sorted(RUNTIME_PACKAGES))).replace( + "__FORBIDDEN_MODULES__", repr(FORBIDDEN_RUNTIME_MODULES) + ) + python = python.resolve() + if not python.is_file(): + raise FileNotFoundError(f"missing required RULER Python executable: {python}") + env = _subprocess_env( + NLTK_DATA=str(nltk_data.resolve()), + TOKENIZERS_PARALLELISM="false", + ) + result = subprocess.run( + [str(python), "-I", "-c", code], + check=True, + capture_output=True, + text=True, + env=env, + ) + payload = _strict_json(result.stdout.encode(), context="RULER runtime probe") + if not isinstance(payload, Mapping): + raise ValueError("RULER runtime probe did not return an object") + if payload.get("python") != RUNTIME_PYTHON_VERSION: + raise ValueError("RULER Python patch version drifted") + if payload.get("packages") != RUNTIME_PACKAGES: + raise ValueError("RULER package versions drifted") + expected_distributions = { + _canonical_distribution_name(name): version for name, version in RUNTIME_PACKAGES.items() + } + if payload.get("installed_distributions") != expected_distributions: + raise ValueError("RULER installed-distribution inventory drifted") + expected_absence = {name: False for name in FORBIDDEN_RUNTIME_MODULES} + if payload.get("forbidden_modules") != expected_absence: + raise ValueError("RULER runtime contains a forbidden model framework") + if payload.get("implementation") != "cpython": + raise ValueError("RULER Python implementation drifted") + flags = payload.get("flags") + if flags != {"ignore_environment": 1, "isolated": 1, "no_user_site": 1}: + raise ValueError("RULER runtime probe was not isolated") + executable = payload.get("executable") + if not isinstance(executable, str) or not Path(executable).samefile(python): + raise ValueError("RULER runtime probe used a different Python executable") + for field in ("cache_tag", "platform"): + if not isinstance(payload.get(field), str) or not payload[field]: + raise ValueError(f"RULER runtime probe omitted {field}") + resources = payload.get("resources") + if not isinstance(resources, Mapping) or set(resources) != set(EXPECTED_PACKAGE_RESOURCES): + raise ValueError("RULER package-resource inventory drifted") + paths: dict[str, Path] = {} + for name, raw_path in resources.items(): + if not isinstance(raw_path, str): + raise ValueError("RULER runtime returned a non-string resource path") + paths[str(name)] = Path(raw_path) + executable_data = python.read_bytes() + runtime_manifest = { + "python": payload["python"], + "implementation": payload["implementation"], + "cache_tag": payload["cache_tag"], + "platform": payload["platform"], + "flags": payload["flags"], + "executable": _file_entry("python.exe", executable_data), + "packages": payload["packages"], + "installed_distributions": payload["installed_distributions"], + "forbidden_modules": payload["forbidden_modules"], + } + return runtime_manifest, paths + + +def verify_static_inputs( + *, + ruler_root: Path, + tokenizer_dir: Path, + resource_paths: Mapping[str, Path], + runtime_manifest: Mapping[str, object], + capture: Any, +) -> list[dict[str, object]]: + entries: list[dict[str, object]] = [] + corpus_root = ruler_root / "scripts" / "data" / "synthetic" / "json" + for name, (size, sha256) in sorted(EXPECTED_CORPORA.items()): + data = _verified_file( + corpus_root / name, + size=size, + sha256=sha256, + name=f"RULER corpus {name}", + ) + entries.append(_file_entry(f"corpora/{name}", data)) + for name, (size, sha256) in sorted(EXPECTED_PACKAGE_RESOURCES.items()): + data = _verified_file( + resource_paths[name], + size=size, + sha256=sha256, + name=f"RULER package resource {name}", + ) + entries.append(_file_entry(f"packages/{name}", data)) + + tokenizer_dir = tokenizer_dir.resolve() + if tokenizer_dir.name != capture.resolver.PRIMARY_MODEL_REVISION: + raise ValueError("tokenizer snapshot directory does not name the frozen revision") + files = {path.name: path for path in tokenizer_dir.iterdir() if path.is_file()} + if set(files) != set(EXPECTED_TOKENIZER_ASSETS): + raise ValueError("tokenizer-only asset inventory drifted") + for path in tokenizer_dir.rglob("*"): + if path.is_file() and capture.FORBIDDEN_MODEL_FILE_RE.search( + path.relative_to(tokenizer_dir).as_posix() + ): + raise ValueError(f"model-weight-like file is forbidden in tokenizer snapshot: {path}") + for name, (size, sha256) in sorted(EXPECTED_TOKENIZER_ASSETS.items()): + data = _verified_file( + files[name], + size=size, + sha256=sha256, + name=f"tokenizer asset {name}", + ) + entries.append(_file_entry(f"tokenizer/{name}", data)) + + runtime_bytes = _canonical_json_bytes(runtime_manifest) + entries.append(_file_entry("runtime/package-manifest.json", runtime_bytes)) + entries.append(_file_entry("runtime/requirements.txt", _requirements_bytes())) + entries.append(_launcher_source_entry()) + + return sorted(entries, key=lambda item: str(item["name"])) + + +class IndependentTokenizer: + """Recompute token counts inside the verified tokenizer-only interpreter.""" + + def __init__(self, *, python: Path, tokenizer_dir: Path) -> None: + self._python = python.resolve() + self._tokenizer_dir = tokenizer_dir.resolve() + + def count_tokens(self, text: str) -> int: + code = """ +import json +import sys +from transformers import AutoTokenizer +request = json.load(sys.stdin) +tokenizer = AutoTokenizer.from_pretrained( + request['tokenizer_dir'], local_files_only=True, trust_remote_code=False +) +print(json.dumps({'count': len(tokenizer.tokenize(request['text']))})) +""" + request = {"text": text, "tokenizer_dir": str(self._tokenizer_dir)} + result = subprocess.run( + [str(self._python), "-I", "-c", code], + check=True, + capture_output=True, + text=True, + input=json.dumps(request, ensure_ascii=False, allow_nan=False), + env=_subprocess_env( + HF_HUB_OFFLINE="1", + TOKENIZERS_PARALLELISM="false", + TRANSFORMERS_OFFLINE="1", + ), + ) + payload = _strict_json(result.stdout.encode(), context="independent tokenizer count") + if ( + not isinstance(payload, Mapping) + or set(payload) != {"count"} + or isinstance(payload["count"], bool) + or not isinstance(payload["count"], int) + or payload["count"] < 0 + ): + raise ValueError("independent tokenizer returned an invalid token count") + return int(payload["count"]) + + +def _token_count(tokenizer: Any, text: str) -> int: + count = getattr(tokenizer, "count_tokens", None) + value = count(text) if callable(count) else len(tokenizer.tokenize(text)) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError("tokenizer returned an invalid token count") + return value + + +def generator_argv( + *, + python: Path, + ruler_root: Path, + raw_root: Path, + receipt: Mapping[str, object], +) -> tuple[list[str], list[str]]: + config = str(receipt["config"]) + try: + spec = TASK_SPECS[config] + except KeyError as error: + raise ValueError(f"no frozen launcher specification for RULER config {config}") from error + script = ruler_root / "scripts" / "data" / "synthetic" / spec.script + actual = [ + str(python.resolve()), + "-s", + str(script.resolve()), + "--save_dir", + str(raw_root.resolve()), + "--save_name", + config, + "--subset", + "validation", + "--tokenizer_path", + "", + "--tokenizer_type", + "hf", + "--max_seq_length", + str(receipt["configured_length"]), + "--tokens_to_generate", + str(spec.tokens_to_generate), + "--num_samples", + "1", + "--random_seed", + str(receipt["seed"]), + ] + for name, value in spec.arguments: + actual.extend((f"--{name}", value)) + actual.extend(("--template", spec.template)) + portable = list(actual) + portable[0] = "" + portable[2] = f"scripts/data/synthetic/{spec.script}" + portable[4] = "" + return actual, portable + + +def _entry_named(entries: Sequence[Mapping[str, object]], name: str) -> Mapping[str, object]: + matches = [entry for entry in entries if entry.get("name") == name] + if len(matches) != 1: + raise ValueError(f"bound file inventory must contain exactly one {name!r}") + return matches[0] + + +def _command_manifest( + *, + receipt: Mapping[str, object], + portable_argv: Sequence[str], + capture: Any, + static_entries: Sequence[Mapping[str, object]], +) -> dict[str, object]: + launcher_entry = _entry_named(static_entries, "launcher/generate_static_q468_ruler_receipts.py") + return { + "launcher_revision": LAUNCHER_REVISION, + "launcher_source_sha256": launcher_entry["sha256"], + "ruler_revision": capture.resolver.RULER_REVISION, + "config": receipt["config"], + "configured_length": receipt["configured_length"], + "seed": receipt["seed"], + "argv": list(portable_argv), + "shell": False, + } + + +def _normalize_output_row( + row: object, + *, + config: str, + configured_length: int, + tokenizer: Any, +) -> dict[str, object]: + if not isinstance(row, Mapping): + raise ValueError("RULER generator row must be an object") + spec = TASK_SPECS[config] + required = { + "index", + "input", + "outputs", + "length", + "length_w_model_temp", + "answer_prefix", + } + expected = required | ({"token_position_answer"} if spec.niah else set()) + if set(row) != expected: + raise ValueError("RULER generator row fields drifted") + input_text = row["input"] + prefix = row["answer_prefix"] + outputs = row["outputs"] + if not isinstance(input_text, str) or not input_text: + raise ValueError("RULER generator input must be a non-empty string") + if spec.input_marker not in input_text: + raise ValueError("RULER generator input is missing the frozen task marker") + if not isinstance(prefix, str) or not prefix.startswith(spec.answer_prefix_marker): + raise ValueError("RULER generator answer prefix is truncated or drifted") + if ( + isinstance(outputs, (str, bytes)) + or not isinstance(outputs, Sequence) + or not outputs + or any(not isinstance(value, str) or not value for value in outputs) + ): + raise ValueError("RULER generator outputs must be non-empty strings") + normalized_outputs = list(outputs) + if ( + spec.expected_output_count is not None + and len(normalized_outputs) != spec.expected_output_count + ): + raise ValueError(f"RULER {config} output count must equal {spec.expected_output_count}") + if spec.unique_outputs and len(set(normalized_outputs)) != len(normalized_outputs): + raise ValueError(f"RULER {config} outputs must be unique") + if spec.output_pattern is not None and any( + re.fullmatch(spec.output_pattern, output) is None for output in normalized_outputs + ): + raise ValueError(f"RULER {config} output format drifted") + if spec.outputs_must_appear and any(output not in input_text for output in normalized_outputs): + raise ValueError(f"RULER {config} required answer is absent from its input") + if isinstance(row["length"], bool) or not isinstance(row["length"], int): + raise ValueError("RULER generator length must be an integer") + if row["length_w_model_temp"] != row["length"]: + raise ValueError("base-template RULER lengths disagree") + if not 1 <= row["length"] <= configured_length: + raise ValueError("RULER generator length exceeds its configured bound") + independent_length = _token_count(tokenizer, input_text + prefix) + spec.tokens_to_generate + if independent_length != row["length"]: + raise ValueError("RULER generator length disagrees with independent tokenization") + index = row["index"] + if isinstance(index, bool) or not isinstance(index, int): + raise ValueError("RULER generator index must be an integer") + if spec.niah: + if index < 0 or input_text.find(normalized_outputs[0]) != index: + raise ValueError("RULER NIAH answer is not present at the reported position") + token_position = row["token_position_answer"] + if isinstance(token_position, bool) or not isinstance(token_position, int): + raise ValueError("RULER NIAH token position must be an integer") + if token_position != _token_count(tokenizer, input_text[:index]): + raise ValueError("RULER NIAH answer token position drifted") + elif index != 0: + raise ValueError("single-sample RULER generator index must equal zero") + return { + "input": input_text, + "answer_prefix": prefix, + "outputs": normalized_outputs, + "generator_reported_length": int(row["length"]), + } + + +def _atomic_publish_new(path: Path, data: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", suffix=".tmp", dir=path.parent + ) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(data) + handle.flush() + os.fsync(handle.fileno()) + try: + os.link(temporary, path) + except FileExistsError as error: + raise FileExistsError(f"refusing to overwrite RULER artifact: {path}") from error + finally: + temporary.unlink(missing_ok=True) + + +def _atomic_publish_same(path: Path, data: bytes) -> None: + """Publish new bytes, or accept an already-published byte-identical file.""" + + try: + _atomic_publish_new(path, data) + except FileExistsError: + if path.read_bytes() != data: + raise FileExistsError(f"existing RULER artifact differs: {path}") from None + + +def _load_existing_receipt_result( + *, + path: Path, + receipt: Mapping[str, object], + capture: Any, + python: Path, + ruler_root: Path, + raw_root: Path, + tokenizer: Any, + static_entries: Sequence[Mapping[str, object]], +) -> dict[str, object]: + payload = path.read_bytes() + value = _strict_json(payload, context=f"existing RULER receipt {path.name}") + if not isinstance(value, Mapping) or _canonical_json_bytes(value) != payload: + raise ValueError(f"existing RULER receipt is not canonical: {path}") + config = str(receipt["config"]) + capture._normalize_ruler_receipt( + value, + category=str(receipt["category"]), + config=config, + configured_length=int(receipt["configured_length"]), + seed=int(receipt["seed"]), + ) + normalized = _normalize_output_row( + { + "index": ( + str(value["input"]).find(value["outputs"][0]) if TASK_SPECS[config].niah else 0 + ), + "input": value["input"], + "outputs": value["outputs"], + "length": value["generator_reported_length"], + "length_w_model_temp": value["generator_reported_length"], + "answer_prefix": value["answer_prefix"], + **( + { + "token_position_answer": _token_count( + tokenizer, + str(value["input"])[: str(value["input"]).find(value["outputs"][0])], + ) + } + if TASK_SPECS[config].niah + else {} + ), + }, + config=config, + configured_length=int(receipt["configured_length"]), + tokenizer=tokenizer, + ) + auxiliary = value["auxiliary_files"] + if not isinstance(auxiliary, Sequence): # capture normalization already rejects this + raise ValueError("existing RULER receipt auxiliary inventory is invalid") + auxiliary_by_name = {str(entry["name"]): entry for entry in auxiliary} + if len(auxiliary_by_name) != len(auxiliary): + raise ValueError("existing RULER receipt repeats an auxiliary file") + for expected in static_entries: + if auxiliary_by_name.get(str(expected["name"])) != expected: + raise ValueError("existing RULER receipt has a stale static-input binding") + _, portable = generator_argv( + python=python, + ruler_root=ruler_root, + raw_root=raw_root, + receipt=receipt, + ) + command_bytes = _canonical_json_bytes( + _command_manifest( + receipt=receipt, + portable_argv=portable, + capture=capture, + static_entries=static_entries, + ) + ) + expected_command = _file_entry("generator/command-manifest.json", command_bytes) + if auxiliary_by_name.get("generator/command-manifest.json") != expected_command: + raise ValueError("existing RULER receipt has a stale launcher binding") + expected_names = {str(entry["name"]) for entry in static_entries} | { + "generator/command-manifest.json", + "generator/raw-validation.jsonl", + } + if set(auxiliary_by_name) != expected_names: + raise ValueError("existing RULER receipt auxiliary inventory drifted") + return { + "filename": path.name, + "phase": receipt["phase"], + "sha256": _sha256_bytes(payload), + "size_bytes": len(payload), + "generator_reported_length": normalized["generator_reported_length"], + } + + +def generate_receipt( + *, + receipt: Mapping[str, object], + capture: Any, + python: Path, + ruler_root: Path, + tokenizer_dir: Path, + nltk_data: Path, + raw_root: Path, + output_dir: Path, + tokenizer: Any, + static_entries: Sequence[Mapping[str, object]], + runtime_manifest: Mapping[str, object], + timeout_seconds: int, +) -> dict[str, object]: + filename = str(receipt["filename"]) + output_path = output_dir / filename + if output_path.exists(): + raise FileExistsError(f"refusing to overwrite RULER receipt: {output_path}") + raw_root.mkdir(parents=True, exist_ok=True) + receipt_key = _sha256_bytes(filename.encode("utf-8"))[:12] + receipt_raw_root = Path( + tempfile.mkdtemp(prefix=f".rq-{receipt_key}.", suffix=".staging", dir=raw_root) + ) + + actual, portable = generator_argv( + python=python, + ruler_root=ruler_root, + raw_root=receipt_raw_root, + receipt=receipt, + ) + tokenizer_index = actual.index("") + actual[tokenizer_index] = str(tokenizer_dir.resolve()) + command_manifest = _command_manifest( + receipt=receipt, + portable_argv=portable, + capture=capture, + static_entries=static_entries, + ) + command_bytes = _canonical_json_bytes(command_manifest) + runtime_bytes = _canonical_json_bytes(runtime_manifest) + _atomic_publish_new(receipt_raw_root / "command-manifest.json", command_bytes) + _atomic_publish_new(receipt_raw_root / "runtime-manifest.json", runtime_bytes) + + env = _subprocess_env( + HF_HUB_OFFLINE="1", + NLTK_DATA=str(nltk_data.resolve()), + PYTHONHASHSEED="0", + TOKENIZERS_PARALLELISM="false", + TRANSFORMERS_OFFLINE="1", + ) + result = subprocess.run( + actual, + cwd=ruler_root / "scripts" / "data", + env=env, + capture_output=True, + text=True, + timeout=timeout_seconds, + check=False, + ) + (receipt_raw_root / "stdout.log").write_text(result.stdout, encoding="utf-8") + (receipt_raw_root / "stderr.log").write_text(result.stderr, encoding="utf-8") + if result.returncode != 0: + raise RuntimeError( + f"RULER generator failed for {filename} with exit code {result.returncode}" + ) + config = str(receipt["config"]) + row_path = receipt_raw_root / config / "validation.jsonl" + if not row_path.is_file(): + raise RuntimeError(f"RULER generator returned zero without producing {row_path}") + raw_data = row_path.read_bytes() + lines = raw_data.splitlines() + if len(lines) != 1 or not lines[0]: + raise ValueError("RULER generator must produce exactly one non-empty JSONL row") + row = _strict_json(lines[0], context=f"RULER output {filename}") + normalized = _normalize_output_row( + row, + config=config, + configured_length=int(receipt["configured_length"]), + tokenizer=tokenizer, + ) + dynamic_entries = [ + _file_entry("generator/command-manifest.json", command_bytes), + _file_entry("generator/raw-validation.jsonl", raw_data), + ] + auxiliary = sorted( + [dict(item) for item in static_entries] + dynamic_entries, + key=lambda item: str(item["name"]), + ) + receipt_data = { + "schema": capture.RULER_RECEIPT_SCHEMA, + "source_id": capture.resolver.RULER_SOURCE_ID, + "revision": capture.resolver.RULER_REVISION, + "category": receipt["category"], + "config": config, + "configured_length": receipt["configured_length"], + "seed": receipt["seed"], + "sample_index": 0, + **normalized, + "auxiliary_files": auxiliary, + } + capture._normalize_ruler_receipt( + receipt_data, + category=str(receipt["category"]), + config=config, + configured_length=int(receipt["configured_length"]), + seed=int(receipt["seed"]), + ) + payload = _canonical_json_bytes(receipt_data) + _atomic_publish_new(output_path, payload) + published_raw_root = raw_root / filename.removesuffix(".json") + try: + receipt_raw_root.rename(published_raw_root) + except FileExistsError: + # Raw logs are diagnostics, not publication inputs. Keep this staged + # run rather than replacing an older diagnostic directory. + print( + f"warning: kept raw diagnostics at {receipt_raw_root} because " + f"{published_raw_root} already exists", + file=sys.stderr, + flush=True, + ) + return { + "filename": filename, + "phase": receipt["phase"], + "sha256": _sha256_bytes(payload), + "size_bytes": len(payload), + "generator_reported_length": normalized["generator_reported_length"], + } + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--ruler-root", type=Path, required=True) + parser.add_argument("--python", type=Path, required=True) + parser.add_argument("--tokenizer-dir", type=Path, required=True) + parser.add_argument("--nltk-data", type=Path, required=True) + parser.add_argument("--raw-dir", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--timeout-seconds", type=int, default=600) + parser.add_argument( + "--receipt", + action="append", + default=[], + help="Generate only an exact required receipt filename; may be repeated.", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.timeout_seconds < 1: + raise ValueError("--timeout-seconds must be positive") + capture = _load_capture_module() + source_manifest = verify_ruler_checkout(args.ruler_root, capture) + runtime_manifest, resource_paths = verify_runtime(args.python, args.nltk_data) + static_entries = verify_static_inputs( + ruler_root=args.ruler_root, + tokenizer_dir=args.tokenizer_dir, + resource_paths=resource_paths, + runtime_manifest=runtime_manifest, + capture=capture, + ) + static_entries = sorted( + [ + *static_entries, + _file_entry("ruler/source-manifest.json", _canonical_json_bytes(source_manifest)), + ], + key=lambda item: str(item["name"]), + ) + tokenizer = IndependentTokenizer(python=args.python, tokenizer_dir=args.tokenizer_dir) + required = list(capture.required_ruler_receipts()) + by_filename = {str(item["filename"]): item for item in required} + if len(by_filename) != len(required): + raise ValueError("required RULER receipt filenames must be unique") + identities = { + ( + item["phase"], + item["category"], + item["config"], + item["configured_length"], + item["seed"], + ) + for item in required + } + if len(identities) != len(required): + raise ValueError("required RULER receipt identities must be unique") + if args.receipt: + if len(set(args.receipt)) != len(args.receipt): + raise ValueError("--receipt values must be unique") + unknown = sorted(set(args.receipt) - set(by_filename)) + if unknown: + raise ValueError(f"unknown required RULER receipts: {unknown}") + selected = [by_filename[name] for name in args.receipt] + else: + selected = required + results = [] + for index, receipt in enumerate(selected, start=1): + output_path = args.output_dir / str(receipt["filename"]) + if output_path.exists(): + print( + f"[{index}/{len(selected)}] verifying existing {receipt['filename']}", + flush=True, + ) + result = _load_existing_receipt_result( + path=output_path, + receipt=receipt, + capture=capture, + python=args.python, + ruler_root=args.ruler_root, + raw_root=args.raw_dir, + tokenizer=tokenizer, + static_entries=static_entries, + ) + else: + print(f"[{index}/{len(selected)}] generating {receipt['filename']}", flush=True) + result = generate_receipt( + receipt=receipt, + capture=capture, + python=args.python, + ruler_root=args.ruler_root, + tokenizer_dir=args.tokenizer_dir, + nltk_data=args.nltk_data, + raw_root=args.raw_dir, + output_dir=args.output_dir, + tokenizer=tokenizer, + static_entries=static_entries, + runtime_manifest=runtime_manifest, + timeout_seconds=args.timeout_seconds, + ) + results.append(result) + launcher_entry = _entry_named(static_entries, "launcher/generate_static_q468_ruler_receipts.py") + manifest = { + "schema": "recurquant.experiment013.ruler-generation-manifest.v1", + "launcher_revision": LAUNCHER_REVISION, + "launcher_source_sha256": launcher_entry["sha256"], + "ruler_revision": capture.resolver.RULER_REVISION, + "source_manifest_sha256": _sha256_bytes(_canonical_json_bytes(source_manifest)), + "runtime_manifest_sha256": _sha256_bytes(_canonical_json_bytes(runtime_manifest)), + "selected_receipts": results, + } + _atomic_publish_same( + args.output_dir / "generation-manifest.json", _canonical_json_bytes(manifest) + ) + print(json.dumps(manifest, indent=2, sort_keys=True), flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_generate_static_q468_ruler_receipts.py b/tests/test_generate_static_q468_ruler_receipts.py new file mode 100644 index 0000000..52d869a --- /dev/null +++ b/tests/test_generate_static_q468_ruler_receipts.py @@ -0,0 +1,268 @@ +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "generate_static_q468_ruler_receipts.py" +SPEC = importlib.util.spec_from_file_location("generate_static_q468_ruler_receipts", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +ruler = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = ruler +SPEC.loader.exec_module(ruler) + + +class WhitespaceTokenizer: + @staticmethod + def tokenize(text: str) -> list[str]: + return text.split() + + +def _valid_niah_row() -> dict[str, object]: + answers = ["8028695", "4670027", "8310696", "7938875"] + input_text = ( + "Some special magic numbers are hidden here. " + + " ".join(f"One special value is {answer}." for answer in answers) + + " " + "What are all the special magic numbers in the provided text?" + ) + prefix = " The special magic numbers are" + return { + "index": input_text.find(answers[0]), + "input": input_text, + "outputs": answers, + "length": len(WhitespaceTokenizer.tokenize(input_text + prefix)) + 128, + "length_w_model_temp": len(WhitespaceTokenizer.tokenize(input_text + prefix)) + 128, + "answer_prefix": prefix, + "token_position_answer": len( + WhitespaceTokenizer.tokenize(input_text[: input_text.find(answers[0])]) + ), + } + + +def test_task_specs_cover_the_exact_required_receipt_configs() -> None: + capture = ruler._load_capture_module() + required_configs = {str(item["config"]) for item in capture.required_ruler_receipts()} + + assert set(ruler.TASK_SPECS) == required_configs + assert len(capture.required_ruler_receipts()) == 20 + assert {config: spec.expected_output_count for config, spec in ruler.TASK_SPECS.items()} == { + config: capture.RULER_REQUIRED_OUTPUT_COUNTS.get(config) for config in required_configs + } + + +def test_generator_argv_preserves_multiline_template_as_one_argument(tmp_path) -> None: + receipt = { + "config": "niah_multiquery", + "configured_length": 2048, + "seed": 12339, + } + + actual, portable = ruler.generator_argv( + python=tmp_path / "python.exe", + ruler_root=tmp_path / "ruler", + raw_root=tmp_path / "raw", + receipt=receipt, + ) + + template_index = actual.index("--template") + 1 + assert actual[template_index] == ruler.NIAH_TEMPLATE + assert "\n{context}\n" in actual[template_index] + assert portable[0] == "" + assert portable[1] == "-s" + assert portable[2] == "scripts/data/synthetic/niah.py" + assert portable[4] == "" + + +def test_valid_niah_row_is_independently_recounted() -> None: + normalized = ruler._normalize_output_row( + _valid_niah_row(), + config="niah_multiquery", + configured_length=2048, + tokenizer=WhitespaceTokenizer(), + ) + + assert normalized["outputs"] == ["8028695", "4670027", "8310696", "7938875"] + assert normalized["generator_reported_length"] == _valid_niah_row()["length"] + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + (lambda row: row.update(input="prompt without the required question"), "task marker"), + (lambda row: row.update(answer_prefix="."), "answer prefix"), + (lambda row: row.update(index=-1), "not present"), + (lambda row: row.update(length=int(row["length"]) + 1), "tokenization"), + (lambda row: row.update(outputs=[]), "outputs"), + ( + lambda row: row.update(outputs=["8028695", "4670027", "8310696"]), + "output count", + ), + ( + lambda row: row.update(outputs=["8028695", "4670027", "8310696", "8310696"]), + "unique", + ), + ( + lambda row: row.update(outputs=["8028695", "4670027", "8310696", "1234567"]), + "absent", + ), + ], +) +def test_malformed_or_truncated_ruler_rows_fail_closed(mutation, message: str) -> None: + row = _valid_niah_row() + mutation(row) + if row["length"] != row["length_w_model_temp"]: + row["length_w_model_temp"] = row["length"] + + with pytest.raises(ValueError, match=message): + ruler._normalize_output_row( + row, + config="niah_multiquery", + configured_length=2048, + tokenizer=WhitespaceTokenizer(), + ) + + +def test_original_windows_false_success_is_rejected() -> None: + malformed = { + "index": -1, + "input": ( + "Some special magic numbers are hidden within the following text. " + "Make sure to memorize it. I will quiz you about the numbers afterwards" + ), + "outputs": ["8028695", "4670027", "8310696", "7938875"], + "length": 155, + "length_w_model_temp": 155, + "answer_prefix": ".", + "token_position_answer": 26, + } + + with pytest.raises(ValueError, match="task marker"): + ruler._normalize_output_row( + malformed, + config="niah_multiquery", + configured_length=2048, + tokenizer=WhitespaceTokenizer(), + ) + + +def test_strict_json_rejects_duplicate_keys() -> None: + with pytest.raises(ValueError, match="duplicate key"): + ruler._strict_json(b'{"a":1,"a":2}', context="fixture") + + +def test_atomic_publish_refuses_to_replace_existing_artifact(tmp_path) -> None: + path = tmp_path / "receipt.json" + ruler._atomic_publish_new(path, b"first\n") + + with pytest.raises(FileExistsError, match="refusing to overwrite"): + ruler._atomic_publish_new(path, b"second\n") + + assert path.read_bytes() == b"first\n" + assert list(tmp_path.glob(".receipt.json.*.tmp")) == [] + + +def test_atomic_publish_same_is_idempotent_but_rejects_drift(tmp_path) -> None: + path = tmp_path / "generation-manifest.json" + ruler._atomic_publish_same(path, b"same\n") + ruler._atomic_publish_same(path, b"same\n") + + with pytest.raises(FileExistsError, match="differs"): + ruler._atomic_publish_same(path, b"different\n") + + +def test_subprocess_environment_removes_python_injection(monkeypatch) -> None: + monkeypatch.setenv("PYTHONHOME", "untrusted-home") + monkeypatch.setenv("PYTHONPATH", "untrusted-path") + + env = ruler._subprocess_env(TEST_MARKER="bound") + + assert "PYTHONHOME" not in env + assert "PYTHONPATH" not in env + assert env["PYTHONNOUSERSITE"] == "1" + assert env["TEST_MARKER"] == "bound" + + +def test_independent_tokenizer_uses_verified_isolated_python(monkeypatch, tmp_path) -> None: + python = tmp_path / "python.exe" + tokenizer_dir = tmp_path / "tokenizer" + python.write_bytes(b"fixture") + tokenizer_dir.mkdir() + observed: dict[str, object] = {} + + def fake_run(argv, **kwargs): + observed["argv"] = argv + observed["env"] = kwargs["env"] + observed["request"] = json.loads(kwargs["input"]) + return subprocess.CompletedProcess(argv, 0, stdout='{"count":3}\n', stderr="") + + monkeypatch.setattr(ruler.subprocess, "run", fake_run) + monkeypatch.setenv("PYTHONHOME", "untrusted-home") + tokenizer = ruler.IndependentTokenizer(python=python, tokenizer_dir=tokenizer_dir) + + assert tokenizer.count_tokens("one two three") == 3 + assert observed["argv"][:2] == [str(python.resolve()), "-I"] + assert observed["request"] == { + "text": "one two three", + "tokenizer_dir": str(tokenizer_dir.resolve()), + } + assert "PYTHONHOME" not in observed["env"] + assert observed["env"]["TRANSFORMERS_OFFLINE"] == "1" + + +def test_source_config_correspondence_fails_on_argument_drift() -> None: + source_task_for_script = {script: task for task, script in ruler.SOURCE_SCRIPT_BY_TASK.items()} + configs = {} + tasks = {} + for config, spec in ruler.TASK_SPECS.items(): + task = source_task_for_script[spec.script] + extras = ruler.LAUNCHER_ONLY_ARGUMENTS.get(task, {}) + configs[config] = { + "task": task, + "args": {name: value for name, value in spec.arguments if name not in extras}, + } + tasks[task] = { + "tokens_to_generate": spec.tokens_to_generate, + "template": spec.template, + "answer_prefix": "", + } + yaml_lines: list[str] = [] + for config, value in configs.items(): + yaml_lines.extend( + [ + f"{config}:", + f" task: {value['task']}", + " args:", + *[f" {name}: {argument}" for name, argument in value["args"].items()], + ] + ) + synthetic_yaml = "\n".join(yaml_lines) + constants_py = f"TASKS = {tasks!r}\n".encode() + + ruler._verify_task_specs_against_source( + synthetic_yaml=synthetic_yaml.encode(), constants_py=constants_py + ) + drifted = synthetic_yaml.replace("num_needle_q: 4", "num_needle_q: 3", 1) + with pytest.raises(ValueError, match="launcher arguments differ"): + ruler._verify_task_specs_against_source( + synthetic_yaml=drifted.encode(), constants_py=constants_py + ) + + +def test_frozen_runtime_requirements_match_the_probe_contract() -> None: + requirements = ( + Path(__file__).resolve().parents[1] / "requirements" / "experiment013-ruler.txt" + ).read_text(encoding="utf-8") + pinned = { + line.split("==", 1)[0]: line.split("==", 1)[1] + for line in requirements.splitlines() + if line and not line.startswith("#") + } + + assert pinned == ruler.RUNTIME_PACKAGES + assert "torch" not in {ruler._canonical_distribution_name(name) for name in pinned} + assert "torch" in ruler.FORBIDDEN_RUNTIME_MODULES From 9754a9dc4eb403b93ebfa5bd8bf8c2208622dfd1 Mon Sep 17 00:00:00 2001 From: HiddenTrojan <93521146+Labeeb2339@users.noreply.github.com> Date: Sun, 2 Aug 2026 11:43:24 +0800 Subject: [PATCH 10/24] Freeze corrected Experiment 013 protocol --- ...EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md | 73 ++++++++++++++++--- scripts/capture_statelease_stage0.py | 2 + scripts/screen_statelease_stage_a.py | 2 + scripts/verify_statelease_stage0.py | 2 + 4 files changed, 70 insertions(+), 9 deletions(-) diff --git a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md index 36755ee..9ac16fd 100644 --- a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md +++ b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md @@ -1,6 +1,6 @@ # Experiment 013: static RHT-Q468 packed-native adoption protocol -> **Status: frozen before Experiment 013 identity resolution, policy fitting, +> **Status: amended and frozen before Experiment 013 identity resolution, policy fitting, > protected-set access, model-weight loading, or quality measurement.** > > Upstream revisions are frozen below. Canonical row identities, token spans, @@ -10,6 +10,12 @@ Protocol frozen: 2026-08-02 +Pre-resolution audit amendment: 2026-08-02. The amendment corrects a +cache-exposed-span off-by-one, binds the Stage-A calibration chain by exact +artifact-file hashes, and replaces an invalid first-output-only RULER target +with the complete task-aware serialization below. No model weights or quality +results had been opened. + ## Question Can a calibration-frozen, static Q4/Q6/Q8 recurrent-state layout retain the @@ -163,11 +169,11 @@ segment_stop = segment_start + 2304 No tokenizer special tokens are added. The same URL identity and eligibility rule applies to validation, except eligibility requires at least 4,224 tokens -for the frozen 4,096-token prefill and 128 scored tokens. For an accepted +for the frozen 4,096-token prefill and 128 continuation tokens. For an accepted validation book, replace `2304` by `4224` in the equation above and use the independent namespace `recurquant.experiment013.pg19.validation-segment.v1\0`; the first 4,096 -tokens of that slice are prefill and the last 128 are scored. HumanEval+ uses +tokens of that slice are prefill and the last 128 are the continuation. HumanEval+ uses the exact `task_id` field. RULER uses the complete domain-separated configuration identity that the later generator amendment must freeze. @@ -208,6 +214,19 @@ sequence. Identity records therefore bind `configured_length`, actual `sequence_length`, prompt/scored half-open spans, and the generator's own length receipt separately. Anchors use the actual processed token count only. +The pinned upstream `prepare.py` constructs its child command as a multiline +shell string. On Windows, an initial compatibility smoke test returned exit +zero but emitted only a truncated 155-token row with no generated context or +question. That row is rejected and is not evidence. Experiment 013 launcher v2 +invokes the pinned task generator directly with a no-shell argument vector. +Every receipt must authenticate the RULER commit and source blobs, the exact +launcher source, +the isolated Python/package manifest, all tokenizer assets, all auxiliary +corpora, NLTK Punkt, and all Wonderwords noun, adjective, and verb lists. It +must also contain exactly one generated row, the frozen task markers, the +configuration's required output cardinality, and every required NIAH answer in +the prompt. A partial generation batch is not promotable. + Within each broad calibration family, and separately within each of RULER's four official categories, SHA-rank canonical sequence IDs and alternate even and odd ranks into split halves A and B. Recompute the complete equation and @@ -286,18 +305,53 @@ creates the identity that must be committed before model weights are loaded. Stage-B and Stage-C content is protected and requires separate authorization; ordinary resolver tests and dry runs must not read it. +Stage-A resolution additionally consumes one strictly decoded +`experiment-013-stage-a-calibration-binding-v2` artifact. The resolved Stage-A +identity binds these five dependency files directly, not merely semantic IDs +copied from a caller: + +```text +calibration_identity_file_sha256 +calibration_score_artifact_file_sha256 +split_half_stability_artifact_file_sha256 +static_k27030_policy_file_sha256 +static_k29334_policy_file_sha256 +``` + +Changing any byte in any dependency requires a new binding artifact and a new +Stage-A identity candidate. + ### Stage A: multi-workload falsification Stage A contains exactly 12 examples: - the first four SHA-ranked eligible PG19 validation books, each using 4,096 - prefill tokens followed by 128 scored tokens; + prefill tokens followed by 128 continuation tokens, of which 127 predictions + are exposed to the committed quantized cache; - four RULER category representatives at configured length 4,096 and seed - 2,339: `niah_multiquery`, `vt`, `fwe`, and `qa_1`, scoring only each - identity-bound official answer span; and -- the first four SHA-ranked HumanEval+ canonical IDs, scoring at most the first + 2,339: `niah_multiquery`, `vt`, `fwe`, and `qa_1`, using each + identity-bound teacher-forced target derived from the official references; + and +- the first four SHA-ranked HumanEval+ canonical IDs, using at most the first 128 canonical-solution tokens after the identity-bound prompt. +Every continuation is evaluated through one-token forwards. The prefill's last +logit predicts continuation token zero before the candidate checkpoint is +committed, so that unaffected prompt-to-first-token prediction is excluded. +For a continuation of `m` tokens, the metric covers exactly the `m - 1` +cache-exposed transitions from continuation token `i` to token `i + 1`. +Identities bind the complete continuation span and this narrower half-open +cache-exposed metric span separately; `m < 2` fails closed. + +RULER's `outputs` field has two different meanings. Retrieval, variable +tracing, and aggregation list multiple required answer atoms; their Stage-A +target is every atom in source order joined by the exact separator `", "`. +Question-answering rows list alternative acceptable references; their target +is the first pinned alternative in source order. The receipt binds the complete +official output array in both cases. This is a deterministic teacher-forced NLL +target for this adoption study, not a claim that comma-space joining is +RULER's official generation metric. + The exact canonical IDs, configurations, source revisions, formatter hashes, content hashes, prompt-token hashes, target-token hashes, and half-open token spans must be committed before weights are opened. No example may be replaced @@ -348,8 +402,9 @@ through a pre-result identity amendment with its exact implementation and byte accounting; an incompatible or unavailable comparator is documented rather than imitated under its name. -Primary quality is task-macro aligned excess next-token NLL relative to the -matched FP32 trajectory. Report task-macro and token-micro excess NLL, mean and +Primary quality is task-macro aligned excess next-token NLL over only the +identity-bound cache-exposed transitions relative to the matched FP32 +trajectory. Report task-macro and token-micro excess NLL, mean and tail KL, top-1 agreement, local codec SSE, trajectory error, result by workload family, resident bytes, transient bytes, peak HBM, and latency. Statistical intervals are paired task bootstraps with 10,000 resamples and seed 2,339. diff --git a/scripts/capture_statelease_stage0.py b/scripts/capture_statelease_stage0.py index 158fdad..ff88df4 100644 --- a/scripts/capture_statelease_stage0.py +++ b/scripts/capture_statelease_stage0.py @@ -133,6 +133,8 @@ "src/recurquant/row_policy.py", "src/recurquant/signals.py", "src/recurquant/static_q468.py", + "src/recurquant/static_q468_cache.py", + "src/recurquant/static_q468_calibration.py", "src/recurquant/statelease.py", "src/recurquant/statelease_artifact.py", "src/recurquant/statelease_baselines.py", diff --git a/scripts/screen_statelease_stage_a.py b/scripts/screen_statelease_stage_a.py index 262182b..626e423 100644 --- a/scripts/screen_statelease_stage_a.py +++ b/scripts/screen_statelease_stage_a.py @@ -402,6 +402,8 @@ "src/recurquant/row_policy.py", "src/recurquant/signals.py", "src/recurquant/static_q468.py", + "src/recurquant/static_q468_cache.py", + "src/recurquant/static_q468_calibration.py", "src/recurquant/statelease.py", "src/recurquant/statelease_artifact.py", "src/recurquant/statelease_baselines.py", diff --git a/scripts/verify_statelease_stage0.py b/scripts/verify_statelease_stage0.py index d8452c4..c41be40 100644 --- a/scripts/verify_statelease_stage0.py +++ b/scripts/verify_statelease_stage0.py @@ -178,6 +178,8 @@ "src/recurquant/row_policy.py", "src/recurquant/signals.py", "src/recurquant/static_q468.py", + "src/recurquant/static_q468_cache.py", + "src/recurquant/static_q468_calibration.py", "src/recurquant/statelease.py", "src/recurquant/statelease_artifact.py", "src/recurquant/statelease_baselines.py", From 71cd7da6d0b942d7fd18a04d5f6818a79e9210d9 Mon Sep 17 00:00:00 2001 From: HiddenTrojan Date: Sun, 2 Aug 2026 16:23:22 +0800 Subject: [PATCH 11/24] Seal Experiment 013 calibration evidence chain --- requirements/experiment013-calibration.txt | 42 + ...EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md | 109 +- ...xperiment013-parquet-materializations.json | 104 + scripts/capture_static_q468_identity_input.py | 2113 ++++++-- .../generate_static_q468_ruler_receipts.py | 1512 +++++- scripts/launch_static_q468_calibration.py | 1732 +++++++ scripts/resolve_static_q468_identity.py | 146 +- scripts/run_static_q468_calibration.py | 4292 +++++++++++++++++ .../experiment013_calibration_api.py | 233 + src/recurquant/experiment013_parquet.py | 1197 +++++ .../experiment013_qwen35_adapter.py | 1094 +++++ src/recurquant/experiment013_source.py | 848 ++++ ...test_capture_static_q468_identity_input.py | 1219 ++++- tests/test_experiment013_calibration_api.py | 161 + tests/test_experiment013_parquet.py | 675 +++ tests/test_experiment013_qwen35_adapter.py | 1124 +++++ tests/test_experiment013_source.py | 315 ++ ...est_generate_static_q468_ruler_receipts.py | 480 +- tests/test_launch_static_q468_calibration.py | 427 ++ tests/test_resolve_static_q468_identity.py | 52 + tests/test_run_static_q468_calibration.py | 1782 +++++++ tests/test_static_q468.py | 63 + 22 files changed, 19129 insertions(+), 591 deletions(-) create mode 100644 requirements/experiment013-calibration.txt create mode 100644 research/experiment013-parquet-materializations.json create mode 100644 scripts/launch_static_q468_calibration.py create mode 100644 scripts/run_static_q468_calibration.py create mode 100644 src/recurquant/experiment013_calibration_api.py create mode 100644 src/recurquant/experiment013_parquet.py create mode 100644 src/recurquant/experiment013_qwen35_adapter.py create mode 100644 src/recurquant/experiment013_source.py create mode 100644 tests/test_experiment013_calibration_api.py create mode 100644 tests/test_experiment013_parquet.py create mode 100644 tests/test_experiment013_qwen35_adapter.py create mode 100644 tests/test_experiment013_source.py create mode 100644 tests/test_launch_static_q468_calibration.py create mode 100644 tests/test_run_static_q468_calibration.py diff --git a/requirements/experiment013-calibration.txt b/requirements/experiment013-calibration.txt new file mode 100644 index 0000000..779cd46 --- /dev/null +++ b/requirements/experiment013-calibration.txt @@ -0,0 +1,42 @@ +# Exact Windows/Python 3.11 calibration environment accepted on 2026-08-02. +# The official CUDA wheel is URL- and SHA-256-pinned because PyPI's Windows +# wheel is not the required CUDA 13.0 build. The sealed runtime manifest binds +# every installed file at point of use; this file freezes the resolved versions. +annotated-doc==0.0.5 +anyio==4.14.2 +certifi==2026.7.22 +charset-normalizer==3.4.9 +click==8.4.2 +colorama==0.4.6 +filelock==3.32.2 +fsspec==2026.7.0 +h11==0.16.0 +hf-xet==1.5.2 +httpcore==1.0.9 +httpx==0.28.1 +huggingface-hub==1.26.0 +idna==3.18 +Jinja2==3.1.6 +markdown-it-py==4.2.0 +MarkupSafe==3.0.3 +mdurl==0.1.2 +mpmath==1.3.0 +networkx==3.6.1 +numpy==2.4.6 +packaging==26.2 +pyarrow==25.0.0 +Pygments==2.20.0 +PyYAML==6.0.3 +regex==2026.7.19 +requests==2.34.2 +rich==15.0.0 +safetensors==0.8.0 +shellingham==1.5.4 +sympy==1.14.0 +tokenizers==0.22.2 +torch @ https://download-r2.pytorch.org/whl/cu130/torch-2.13.0%2Bcu130-cp311-cp311-win_amd64.whl#sha256=45e97bd9bc0416f4f4190b5098c55119a389fa5a7c8bbf2639f08f1d04e0a0dc +tqdm==4.70.0 +transformers==5.14.1 +typer==0.27.0 +typing-extensions==4.16.0 +urllib3==2.7.0 diff --git a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md index 9ac16fd..2fc1087 100644 --- a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md +++ b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md @@ -16,6 +16,71 @@ artifact-file hashes, and replaces an invalid first-output-only RULER target with the complete task-aware serialization below. No model weights or quality results had been opened. +Second pre-resolution audit amendment: 2026-08-02. Launcher v2 generated all +16 calibration receipts but failed before the first Stage-A receipt when +Windows attempted to encode non-ASCII prompt text as `cp1252` on the isolated +tokenizer subprocess stdin. That partial batch is rejected. A first sender-only +UTF-8 correction, launcher v3, was also rejected on its first receipt because +the isolated child still decoded stdin under the Windows locale and therefore +failed the independent-length check. Launcher v4 fixes both ends: strict UTF-8 +on the parent pipe plus Python `-X utf8` in the isolated child. It is +authenticated by a new source hash. No identity had been promoted and no model +weights or quality results had been opened. + +Third pre-resolution audit amendment: 2026-08-02. The first hardened launcher +v4 run stopped before publishing a receipt because Python `-I` ignored the +`PYTHONDONTWRITEBYTECODE` environment variable and imported bytecode changed the +otherwise authenticated staged-source inventory. The first v5 bootstrap smoke +then stopped nonzero because the isolated import path omitted the task script's +authenticated sibling directory. Both failed attempts are rejected. Launcher +v5 adds explicit `-B` and prepends only the authenticated task directory and +data root. A live single-receipt regression then passed every generator-side +source, runtime, command, raw-row, and tokenizer check. No identity was promoted +and no model weights or quality results were opened. + +Fourth pre-resolution audit amendment: 2026-08-02. Launcher v5 subsequently +generated all 20 required receipts, but an independent producer-to-consumer +audit found that its runtime manifest could never pass identity capture: the +producer emitted isolation `flags` but omitted `machine`, while the strict +consumer required `machine` and rejected `flags` as an extra field. The entire +v5 batch is therefore rejected and is not evidence. Launcher v6 and RULER +runtime-manifest schema v3 bind both non-empty `machine` identity and the exact +isolated flags (`ignore_environment=1`, `isolated=1`, `no_user_site=1`). A +producer-to-consumer regression and a new complete 20-receipt batch are required +before identity resolution. No identity was promoted and no model weights or +quality results were opened. + +Fifth pre-resolution audit amendment: 2026-08-02. A startup audit showed that +`-I -B` alone still permits `site`, virtual-environment `.pth` hooks, and reads +of unbound bytecode. Launcher v6 therefore uses `-I -S -B`, UTF-8 mode, and a +fresh verified-empty bytecode prefix; stages a complete authenticated Python +standard-library and adjacent-DLL tree; stages only RECORD-declared package +files; excludes the two exact verified virtual-environment startup files; and +binds the tokenizer and both Punkt resource layouts before each child. The +producer and consumer now agree on runtime-manifest v3. A real isolated probe +on this machine reported AMD64, 37 exact distributions, 829 Python-runtime +files, and five runtime resources and normalized identically on both sides. +This validates the launcher boundary, not any generated sequence. The v5 +receipts remain rejected; a fresh complete v6 batch is still required. No +identity was promoted and no model weights or quality results were opened. + +Sixth pre-resolution audit amendment: 2026-08-02. The exact 20-entry v6 +command-manifest hash table is now regenerated from the authenticated launcher +and frozen by a producer-to-consumer regression. Identity input, candidate, and +frozen schemas advance together to v4 and bind a fourth execution artifact: +the exact immutable-Parquet materialization manifest. Bulk population reads +are restricted before network access to `url` for PG19 and `task_id` for +HumanEval+; prompts, solutions, and book text cannot be requested through that +surface. A live immutable-commit verification returned 13,684 PG19 training +IDs (`31050fa90be75b8c49a33c0fef9e2b1891ded6791e81a33451a9f05b8980c355`), +50 PG19 validation IDs +(`68da047a7274c57c1ee938beb2a604d082314b831ed21ed5590c1584aa4ec354`), +and 164 HumanEval+ IDs +(`913967d673127c28dc6858d4ded52e063b87496c1b4424bee696535243ebff9d`). +These are canonical projection hashes, not quality results. A clean source +commit, sealed calibration runtime, fresh v6 receipt batch, and promoted v4 +identity are still required before model access. + ## Question Can a calibration-frozen, static Q4/Q6/Q8 recurrent-state layout retain the @@ -150,14 +215,15 @@ recurquant.experiment013.humaneval-plus.stage-c.v1\0 The canonical PG19 ID is the exact UTF-8 `url` field; the pinned PG19 schema does not contain a `book_id` field. The population is the ordered `url` -projection from the active Hugging Face Dataset Viewer `/parquet` manifest for -the `default` configuration and pinned split. The capture must authenticate the -manifest's `x-revision`, hash that complete ordered projection, and exclude -repository-tree parquet files that the active manifest does not reference. A -training book is eligible when the pinned tokenizer produces at least 2,304 -tokens. Rank all 13,684 training URLs before opening text, then inspect them in -that fixed order only until 16 eligible books have been accepted. For an -accepted book with `N` tokens, define +projection from the exact Parquet objects at conversion commit +`b3624dc44b60cb01e74876e8869234d2660812cf`. Capture authenticates the pinned +source revision, conversion revision, ordered file paths, Git blob IDs, LFS +SHA-256 identities and sizes, and row-group footer counts before and after the +projection. It does not use a mutable Dataset Viewer endpoint. A training book +is eligible when the pinned tokenizer produces at least 2,304 tokens. Rank all +13,684 training URLs before opening text, then inspect them in that fixed order +only until 16 eligible books have been accepted. For an accepted book with `N` +tokens, define ```text M = N - 2304 @@ -217,7 +283,7 @@ length receipt separately. Anchors use the actual processed token count only. The pinned upstream `prepare.py` constructs its child command as a multiline shell string. On Windows, an initial compatibility smoke test returned exit zero but emitted only a truncated 155-token row with no generated context or -question. That row is rejected and is not evidence. Experiment 013 launcher v2 +question. That row is rejected and is not evidence. Experiment 013 launcher v6 invokes the pinned task generator directly with a no-shell argument vector. Every receipt must authenticate the RULER commit and source blobs, the exact launcher source, @@ -305,6 +371,26 @@ creates the identity that must be committed before model weights are loaded. Stage-B and Stage-C content is protected and requires separate authorization; ordinary resolver tests and dry runs must not read it. +Identity schema v4 also binds four exact pre-model evidence files under +`execution_bindings`: + +```text +repository_source_manifest_file_sha256 +calibration_runtime_manifest_file_sha256 +model_file_manifest_file_sha256 +parquet_materialization_manifest_file_sha256 +``` + +The source manifest authenticates the implementation, tests, protocol, and +runner at point of use. The runtime manifest authenticates the Python and +installed package-code inventory used by calibration. The model manifest is +derived from immutable Hub repository/LFS metadata without downloading or +opening weight payloads. The Parquet manifest authenticates the source and +conversion commits plus the selected Parquet Git/LFS objects. Only after +promotion may the runner hash local model files and compare them with the +frozen model manifest. A missing, malformed, or byte-different dependency +stops before adapter data access or model loading. + Stage-A resolution additionally consumes one strictly decoded `experiment-013-stage-a-calibration-binding-v2` artifact. The resolved Stage-A identity binds these five dependency files directly, not merely semantic IDs @@ -450,8 +536,9 @@ already fits calibration-based mixed-precision rate-distortion policies for KV caches. [Q-Mamba](https://aclanthology.org/2025.findings-acl.551/) quantizes Mamba state caches, while [Quamba2](https://arxiv.org/abs/2503.22879v4) provides quantized SSM deployment and kernels. [Gated DeltaNet-2](https://arxiv.org/abs/2605.22791v1) -motivates the architecture family, but its public reproducibility remains -blocked. Therefore only a confirmed exact-byte static Q4/Q6/Q8 packed-native +motivates the architecture family. Its official repository currently provides +training code but no tagged release or pretrained checkpoint for the reported +1.3B run. Therefore only a confirmed exact-byte static Q4/Q6/Q8 packed-native Gated DeltaNet path plus end-to-end adoption benefit could be differentiated; this protocol makes no novelty claim. diff --git a/research/experiment013-parquet-materializations.json b/research/experiment013-parquet-materializations.json new file mode 100644 index 0000000..baef150 --- /dev/null +++ b/research/experiment013-parquet-materializations.json @@ -0,0 +1,104 @@ +{ + "datasets": { + "humaneval_plus": { + "conversion_revision": "1cf4467306a94e0828b355ff1f32e9222d2d588a", + "dataset_id": "evalplus/humanevalplus", + "failed": [], + "files": [ + { + "config": "default", + "git_blob_oid": "9877db06683d4245bc39aed18ee7cbad013ba5fa", + "immutable_path": "default/test/0000.parquet", + "lfs_sha256": "4436f5c03d77c17e0cbc57543b90665b5c1266f55a43992a5ed7922cd34a7558", + "lfs_size_bytes": 2902210, + "logical_split": "test", + "size_bytes": 2902210 + } + ], + "partial": false, + "pending": [], + "selected_splits": [ + "test" + ], + "source_revision": "d32357cf319e50e9c8d8dab5ea876c72b0fd321b" + }, + "pg19": { + "conversion_revision": "b3624dc44b60cb01e74876e8869234d2660812cf", + "dataset_id": "emozilla/pg19", + "failed": [], + "files": [ + { + "config": "default", + "git_blob_oid": "00245b214ff9806a04f32debff0fd2e7b0737997", + "immutable_path": "default/partial-train/0000.parquet", + "lfs_sha256": "ea701af2e8a11bb8601150a47affff658452d687494dbed52a82d3b1fcf48811", + "lfs_size_bytes": 603127902, + "logical_split": "train", + "size_bytes": 603127902 + }, + { + "config": "default", + "git_blob_oid": "1169b6deb8c1cd46a46f0ae752b68806c9b5cca9", + "immutable_path": "default/partial-train/0001.parquet", + "lfs_sha256": "5c1c025f46b4a6b52b56167efeb89a2b9378f9ea8a50cdf5ddcbca8c4e17db1f", + "lfs_size_bytes": 526793959, + "logical_split": "train", + "size_bytes": 526793959 + }, + { + "config": "default", + "git_blob_oid": "b9413777553240574488fa36b74f4bd286c06719", + "immutable_path": "default/partial-train/0002.parquet", + "lfs_sha256": "80cc198a2ef5239bf22a496eb10e6afd6fba075c4f6dd3d26dae7ed82c3bb1ad", + "lfs_size_bytes": 576668259, + "logical_split": "train", + "size_bytes": 576668259 + }, + { + "config": "default", + "git_blob_oid": "08f92b1ad15eb9f90268fa7cf9823523f1fb056a", + "immutable_path": "default/partial-train/0003.parquet", + "lfs_sha256": "326718129b7d13a9f45ae8e6e68ae90d95c15bf40fa457a053716832e4d07c1c", + "lfs_size_bytes": 583939098, + "logical_split": "train", + "size_bytes": 583939098 + }, + { + "config": "default", + "git_blob_oid": "9cb48d05cf6568879582eb5bd894a7d1b34aee7b", + "immutable_path": "default/partial-train/0004.parquet", + "lfs_sha256": "c4dff8b2cd993d1bb6bded41eb0eef56dff5449753ea13c79e730b7a9e1f6907", + "lfs_size_bytes": 588756614, + "logical_split": "train", + "size_bytes": 588756614 + }, + { + "config": "default", + "git_blob_oid": "6a1b9b38d31ca025cbf06192a3dfd66067eb7571", + "immutable_path": "default/partial-train/0005.parquet", + "lfs_sha256": "9ab4d07d379720a9b18e7e3a060a948e2338b7aa338e9534a12d52fbc4fd8e2e", + "lfs_size_bytes": 321273724, + "logical_split": "train", + "size_bytes": 321273724 + }, + { + "config": "default", + "git_blob_oid": "3e86263f595fae38387a938ec882417649c2bbd4", + "immutable_path": "default/partial-validation/0000.parquet", + "lfs_sha256": "81680529564d4ead1c0e3859509a62d86c7126c32afc95dce6bd98e729e491ef", + "lfs_size_bytes": 10803864, + "logical_split": "validation", + "size_bytes": 10803864 + } + ], + "partial": true, + "pending": [], + "selected_splits": [ + "train", + "validation" + ], + "source_revision": "c021754c8e01c5b1cc83a1f549c1f97fbbb756b8" + } + }, + "schema": "recurquant.experiment013.parquet-materializations.v1" +} diff --git a/scripts/capture_static_q468_identity_input.py b/scripts/capture_static_q468_identity_input.py index f4ebc6f..dcee8cf 100644 --- a/scripts/capture_static_q468_identity_input.py +++ b/scripts/capture_static_q468_identity_input.py @@ -12,7 +12,8 @@ class or requests a weight file. from __future__ import annotations import argparse -import concurrent.futures +import base64 +import binascii import hashlib import importlib.metadata import importlib.util @@ -28,12 +29,19 @@ class or requests a weight file. import urllib.request from collections.abc import Mapping, Sequence from dataclasses import dataclass -from pathlib import Path +from dataclasses import field as dataclass_field +from pathlib import Path, PurePosixPath +from types import MappingProxyType from typing import Any, Final, Protocol REPOSITORY_ROOT = Path(__file__).resolve().parents[1] RESOLVER_PATH = REPOSITORY_ROOT / "scripts" / "resolve_static_q468_identity.py" +CALIBRATION_RUNNER_PATH = REPOSITORY_ROOT / "scripts" / "run_static_q468_calibration.py" +PARQUET_MATERIALIZATION_MANIFEST_PATH = ( + REPOSITORY_ROOT / "research" / "experiment013-parquet-materializations.json" +) _RESOLVER_MODULE_NAME = "recurquant_experiment013_identity_resolver" +_CALIBRATION_RUNNER_MODULE_NAME = "_recurquant_experiment013_calibration_runner_for_capture" _RESOLVER_SPEC = importlib.util.spec_from_file_location(_RESOLVER_MODULE_NAME, RESOLVER_PATH) if _RESOLVER_SPEC is None or _RESOLVER_SPEC.loader is None: # pragma: no cover raise RuntimeError("cannot load the Experiment 013 identity resolver") @@ -52,8 +60,44 @@ class or requests a weight file. sys.modules.pop(_RESOLVER_MODULE_NAME, None) raise -CAPTURE_VERSION: Final = 2 +CAPTURE_VERSION: Final = 4 +RUNTIME_AUTHENTICATION_CONTEXT_FIELDS: Final = frozenset( + { + "base_runtime_root", + "staged_interpreter", + "package_runtime_roots", + "package_import_paths", + } +) +_RUNTIME_ROOT_NAME_RE: Final = re.compile(r"[a-z][a-z0-9-]{0,63}") RULER_RECEIPT_SCHEMA: Final = "recurquant.experiment013.ruler-receipt.v1" +RULER_GENERATION_MANIFEST_SCHEMA: Final = "recurquant.experiment013.ruler-generation-manifest.v2" +RULER_RUNTIME_MANIFEST_SCHEMA: Final = "recurquant.experiment013.ruler-runtime-manifest.v3" +RULER_LAUNCHER_REVISION: Final = "experiment-013-ruler-argv-launcher-v6" +RULER_RUNTIME_PYTHON_VERSION: Final = "3.11.15" +RULER_SEALED_STARTUP_POLICY: Final = { + "dont_write_bytecode": 1, + "no_site": 1, + "package_path_mode": "staged-record-only-site-packages-v1", + "pycache_mode": "verified-empty-prefix-no-write-v1", + "site_loaded": False, + "utf8_mode": 1, + "virtualenv_hook_loaded": False, +} +RULER_EXCLUDED_VIRTUALENV_STARTUP_FILES: Final = { + "_virtualenv.pth": ( + 18, + "69ac3d8f27e679c81b94ab30b3b56e9cd138219b1ba94a1fa3606d5a76a1433d", + ), + "_virtualenv.py": ( + 5_246, + "cfb3db86aaa53bb62b5ff764970bec2d71c9228590a0ebec57f6ec926cc0bf1a", + ), +} +RULER_LAUNCHER_PATH: Final = ( + REPOSITORY_ROOT / "scripts" / ("generate_static_q468_ruler_receipts.py") +) +RULER_REQUIREMENTS_PATH: Final = REPOSITORY_ROOT / "requirements" / ("experiment013-ruler.txt") RULER_SEQUENCE_NAMESPACE: Final = "recurquant.experiment013.ruler.sequence.v1" PG19_TRAIN_SEGMENT_NAMESPACE: Final = "recurquant.experiment013.pg19.segment.v1\0" PG19_VALIDATION_SEGMENT_NAMESPACE: Final = "recurquant.experiment013.pg19.validation-segment.v1\0" @@ -109,6 +153,274 @@ class or requests a weight file. "fwe": 3, } RULER_REQUIRED_OUTPUT_SEPARATOR: Final = ", " +RULER_GENERATOR_TOKENS: Final = { + "niah_multiquery": 128, + "niah_multikey_2": 128, + "niah_single_1": 128, + "niah_multivalue": 128, + "vt": 30, + "cwe": 120, + "fwe": 50, + "qa_1": 32, + "qa_2": 32, +} +RULER_NIAH_CONFIGS: Final = frozenset( + {"niah_multiquery", "niah_multikey_2", "niah_single_1", "niah_multivalue"} +) + + +@dataclass(frozen=True, slots=True) +class RulerTaskInvariant: + """Frozen semantic checks replayed independently of the receipt launcher.""" + + input_marker: str + input_prefix: str + input_suffix: str + answer_prefix_marker: str + answer_prefix_suffix: str + expected_output_count: int | None + unique_outputs: bool + output_pattern: str | None + outputs_must_appear: bool + + +RULER_TASK_INVARIANTS: Final = { + "niah_multiquery": RulerTaskInvariant( + input_marker="What are all the special magic numbers", + input_prefix="Some special magic numbers are hidden within the following text.", + input_suffix="mentioned in the provided text?", + answer_prefix_marker=" The special magic numbers", + answer_prefix_suffix="mentioned in the provided text are", + expected_output_count=4, + unique_outputs=True, + output_pattern=r"[0-9]{7}", + outputs_must_appear=True, + ), + "niah_multikey_2": RulerTaskInvariant( + input_marker="What is the special magic number", + input_prefix="A special magic number is hidden within the following text.", + input_suffix="mentioned in the provided text?", + answer_prefix_marker=" The special magic number", + answer_prefix_suffix="mentioned in the provided text is", + expected_output_count=1, + unique_outputs=True, + output_pattern=r"[0-9]{7}", + outputs_must_appear=True, + ), + "niah_single_1": RulerTaskInvariant( + input_marker="What is the special magic number", + input_prefix="A special magic number is hidden within the following text.", + input_suffix="mentioned in the provided text?", + answer_prefix_marker=" The special magic number", + answer_prefix_suffix="mentioned in the provided text is", + expected_output_count=1, + unique_outputs=True, + output_pattern=r"[0-9]{7}", + outputs_must_appear=True, + ), + "niah_multivalue": RulerTaskInvariant( + input_marker="What are all the special magic numbers", + input_prefix="Some special magic numbers are hidden within the following text.", + input_suffix="mentioned in the provided text?", + answer_prefix_marker=" The special magic numbers", + answer_prefix_suffix="mentioned in the provided text are", + expected_output_count=4, + unique_outputs=True, + output_pattern=r"[0-9]{7}", + outputs_must_appear=True, + ), + "vt": RulerTaskInvariant( + input_marker="Question: Find all variables", + input_prefix="Memorize and track the chain(s) of variable assignment", + input_suffix="in the text above.", + answer_prefix_marker=( + " Answer: According to the chain(s) of variable assignment in the text above," + ), + answer_prefix_suffix=", they are: ", + expected_output_count=5, + unique_outputs=True, + output_pattern=r"[A-Z]{5}", + outputs_must_appear=True, + ), + "cwe": RulerTaskInvariant( + input_marker="Question: What are the 10 most common words", + input_prefix="Below is a numbered list of words.", + input_suffix="in the above list?", + answer_prefix_marker=" Answer: The top 10 words that appear most often in the list are:", + answer_prefix_suffix="in the list are:", + expected_output_count=10, + unique_outputs=True, + output_pattern=None, + outputs_must_appear=True, + ), + "fwe": RulerTaskInvariant( + input_marker="What are the three most frequently appeared words", + input_prefix="Read the following coded text and track the frequency of each coded word.", + input_suffix="in the above coded text?", + answer_prefix_marker=" Answer: According to the coded text above,", + answer_prefix_suffix="the three most frequently appeared words are:", + expected_output_count=3, + unique_outputs=True, + output_pattern=r"[a-z]{6}", + outputs_must_appear=True, + ), + "qa_1": RulerTaskInvariant( + input_marker="The following are given documents.", + input_prefix="Answer the question based on the given documents.", + input_suffix="?", + answer_prefix_marker=" Answer:", + answer_prefix_suffix=" Answer:", + expected_output_count=None, + unique_outputs=False, + output_pattern=None, + outputs_must_appear=False, + ), + "qa_2": RulerTaskInvariant( + input_marker="The following are given documents.", + input_prefix="Answer the question based on the given documents.", + input_suffix="?", + answer_prefix_marker=" Answer:", + answer_prefix_suffix=" Answer:", + expected_output_count=None, + unique_outputs=False, + output_pattern=None, + outputs_must_appear=False, + ), +} +RULER_FORBIDDEN_RUNTIME_MODULES: Final = ( + "accelerate", + "bitsandbytes", + "flax", + "jax", + "onnx", + "onnxruntime", + "tensorflow", + "torch", +) +RULER_EXPECTED_CORPORA: Final = { + "PaulGrahamEssays.json": ( + 3_108_621, + "8d31e1b660e0f2180bcca6d238e18f77921df9d158611582b860da1762b6d3dd", + ), + "english_words.json": ( + 8_564_991, + "affcd6d45fdf3cc843d585c99c97ad615094e760e6c4756b654bab6c73bc2eca", + ), + "hotpotqa.json": ( + 61_065_698, + "e3da074df24e8369009918aa5cdbdd254dadcde4c63f7569d36afd6f2268caa8", + ), + "squad.json": ( + 4_370_528, + "80a5225e94905956a6446d296ca1093975c4d3b3260f1d6c8f68bc2ab77182d8", + ), +} +RULER_EXPECTED_PACKAGE_RESOURCES: Final = { + "nltk/punkt/english.pickle": ( + 433_305, + "dda37972ae88998a6fd3e3ec002697a6bd362b32d050fda7d7ca5276873092aa", + ), + "nltk/punkt/PY3/english.pickle": ( + 406_697, + "5cad3758596392364e3be9803dbd7ebeda384b68937b488a01365f5551bb942c", + ), + "wonderwords/adjectivelist.txt": ( + 7_403, + "66814d46b7e292c83e839d12fe2fa083c8f66779b14b294e1be4e1342c5d4131", + ), + "wonderwords/nounlist.txt": ( + 54_752, + "8d88b3ebc2e2969ed92ebe49cdb0c8a87c2ad3b27493d7afdff608f2e51c5bc9", + ), + "wonderwords/verblist.txt": ( + 6_967, + "9fbf5e4e69b8869aebd88e6d545312ff9f878a747327ede5f6dbbba5c27b08ad", + ), +} +RULER_EXPECTED_TOKENIZER_ASSETS: Final = { + "merges.txt": ( + 3_353_259, + "a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d", + ), + "tokenizer.json": ( + 12_807_196, + "fe000e3ed39ed12b8d2481d527d44f93c65d37e87645d2dcc80d1bf9d50d2927", + ), + "tokenizer_config.json": ( + 16_712, + "e611fbccc7c29ef3b1cafb1cb7ea548d189968632901d678fd62be68c47885de", + ), + "vocab.json": ( + 6_722_759, + "ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003", + ), +} + +# These are hashes of the complete canonical command-manifest bytes for every +# required receipt. They freeze the entire portable argv, including templates +# and task-specific flags, without trusting claims embedded in the receipt set. +RULER_COMMAND_MANIFEST_SHA256_BY_FILENAME: Final = { + "aggregation__cwe__l2048__s12340.json": ( + "de342cdbdfd2876e9a63d1201f00ebf1ab539ba030311dd4d6b8c9e7539cc38b" + ), + "aggregation__cwe__l4096__s12340.json": ( + "dba911e2ccd64251688a34c23974c14b38f9b6f56cee1dabff92b772e4da9684" + ), + "aggregation__fwe__l2048__s12339.json": ( + "60396cfcf9a5528fd6c2f9be4bf8727adde8f3794fae478962c6535abb950eb3" + ), + "aggregation__fwe__l4096__s12339.json": ( + "b33f0f12630b8b94db46fd2c7ba9cfcef3097d6161e28d6fec6d332548e3d9af" + ), + "aggregation__fwe__l4096__s2339.json": ( + "747d0a4b3af13a91a0aed560023bd5da11546e06eb6471ee9d964612f3cf64cc" + ), + "multi_hop_tracing__vt__l2048__s12339.json": ( + "edd4e2e387bd48bd7fe26211d9bd6fa56e23d187b33dba2ceb522d211b853888" + ), + "multi_hop_tracing__vt__l2048__s12340.json": ( + "ab2cc8b7b60fa459ea9b592ad890e37b39991bbe61163426daa39152970f1c92" + ), + "multi_hop_tracing__vt__l4096__s12339.json": ( + "15f9b2c1e1d79ebb466e771febddc0640ee4e3b21894591df70f0f2b7b750bec" + ), + "multi_hop_tracing__vt__l4096__s12340.json": ( + "612759085fdf165a665281e43bcec060d86e6e0b900e8f3ab25bc9d4d3aa5eac" + ), + "multi_hop_tracing__vt__l4096__s2339.json": ( + "5a5ebdf3ab054a825bc9e18176a8513f28964a555536e5a2138dd71a4a53a042" + ), + "question_answering__qa_1__l2048__s12339.json": ( + "574119d0a8fe4d8ec4518c2665a5ecbc9dc0f0d97c040c7e07cfc7c7a3b6e59e" + ), + "question_answering__qa_1__l4096__s12339.json": ( + "c9da9b5c0a2b4a8e691a4df84ca1f23bcf8d4427bc6b29e940c065715e31b3a2" + ), + "question_answering__qa_1__l4096__s2339.json": ( + "fd5935bc5ba0da67015995fb08df1288d33b81cc0b4f29b0fbb5b40f8e43a690" + ), + "question_answering__qa_2__l2048__s12340.json": ( + "16f9804bdd9705a068dafa3746936501f8d01120930f180a81511e02eba50e3b" + ), + "question_answering__qa_2__l4096__s12340.json": ( + "abe15545a61b50fc06ad77d9262cffdbfe8480f84ff468e30cf038678736a591" + ), + "retrieval__niah_multikey_2__l2048__s12340.json": ( + "f12039ecaf89bbf7fe7180255ad36aa304148d84f2fdae551648e04c5d25c428" + ), + "retrieval__niah_multiquery__l2048__s12339.json": ( + "099c511108e1b8d4431d03b530b772298e510b6dbefb6d9ff2f51b72b059641d" + ), + "retrieval__niah_multiquery__l4096__s2339.json": ( + "f36ead6ac85202ae799598dc2d9086442686aed2cc918951cc46166ad3780918" + ), + "retrieval__niah_multivalue__l4096__s12340.json": ( + "03cd0aa9b7e2d16d23ab157a6138cfbf344fd884863cb32723f31131de8ecde2" + ), + "retrieval__niah_single_1__l4096__s12339.json": ( + "4edd45298317225a7f6cd3ae22c93f05b33d47902b62d2dea7425bd7d86d19c2" + ), +} # Git object IDs from the recursive tree at RULER_REVISION. Verifying Git's # blob hash catches an unexpected raw response before it enters the formatter @@ -254,6 +566,197 @@ class TokenizerMaterial: model_weights_loaded: bool = False +@dataclass(frozen=True, slots=True) +class _RuntimeAuthenticationContext: + base_runtime_root: Path + staged_interpreter: Path + package_runtime_roots: Mapping[str, Path] + package_import_paths: Mapping[str, str] + + +@dataclass(frozen=True, slots=True) +class _AuthenticatedExecutionBindings: + bindings: Mapping[str, str] + source_manifest: Mapping[str, object] + runtime_manifest: Any + model_manifest: Any + runner: Any + runtime_context: _RuntimeAuthenticationContext + + +@dataclass(frozen=True, slots=True) +class _DecodedExecutionBindingArtifacts: + bindings: Mapping[str, str] + source_manifest: Mapping[str, object] + runtime_manifest: Any + model_manifest: Any + source_module: Any + parquet_module: Any + + +@dataclass(frozen=True, slots=True) +class VerifiedRulerBundle: + receipts: Mapping[str, dict[str, Any]] + generator_manifest: tuple[dict[str, Any], ...] + generation_manifest_sha256: str + + +@dataclass(frozen=True, slots=True) +class MaterializedCalibrationSequence: + """One exact calibration identity record with its formatter-produced tokens. + + Canonical record bytes are kept privately and decoded to a fresh mapping on + every access. The object therefore cannot leak source payloads or let an + adapter mutate the captured identity record in place. + """ + + _identity_record_bytes: bytes = dataclass_field(repr=False) + prompt_token_ids: tuple[int, ...] + target_token_ids: tuple[int, ...] + + def __post_init__(self) -> None: + record = _strict_json( + self._identity_record_bytes, + context="materialized calibration identity record", + ) + if canonical_json_bytes(record) != self._identity_record_bytes: + raise ValueError("materialized calibration identity record is not canonical") + expected_fields = set(resolver.IDENTITY_RECORD_PAYLOAD_FIELDS) | {"identity_record_sha256"} + if set(record) != expected_fields: + raise ValueError("materialized calibration identity record fields drifted") + if not isinstance(self.prompt_token_ids, tuple) or not isinstance( + self.target_token_ids, tuple + ): + raise TypeError("materialized token IDs must be tuples") + for side, token_ids in ( + ("prompt", self.prompt_token_ids), + ("target", self.target_token_ids), + ): + if any( + isinstance(token_id, bool) or not isinstance(token_id, int) or token_id < 0 + for token_id in token_ids + ): + raise ValueError(f"materialized {side} token IDs must be non-negative integers") + sequence_ids = self.sequence_token_ids + span = record.get("token_span") + if not isinstance(span, Mapping): + raise ValueError("materialized identity record token span is malformed") + if ( + record.get("identity_record_sha256") != resolver.identity_record_sha256(record) + or record.get("prompt_token_ids_sha256") != _token_hash(self.prompt_token_ids) + or record.get("target_token_ids_sha256") != _token_hash(self.target_token_ids) + or record.get("sequence_token_ids_sha256") != _token_hash(sequence_ids) + or record.get("sequence_length") != len(sequence_ids) + or span.get("prefill_start") != 0 + or span.get("prefill_stop") != len(self.prompt_token_ids) + or span.get("scored_start") != len(self.prompt_token_ids) + or span.get("scored_stop") != len(sequence_ids) + or span.get("cache_exposed_start") != len(sequence_ids) + or span.get("cache_exposed_stop") != len(sequence_ids) + ): + raise ValueError("materialized calibration tokens differ from their identity record") + forbidden = { + "answer_prefix", + "code", + "formatted_payload", + "input", + "outputs", + "source_payload", + "text", + } + if forbidden & set(record): + raise ValueError("materialized identity record contains forbidden raw content") + + @property + def identity_record(self) -> dict[str, Any]: + """Return a fresh exact copy of the canonical identity record.""" + + return _strict_json( + self._identity_record_bytes, + context="materialized calibration identity record", + ) + + @property + def identity_record_sha256(self) -> str: + return str(self.identity_record["identity_record_sha256"]) + + @property + def sequence_token_ids(self) -> tuple[int, ...]: + return self.prompt_token_ids + self.target_token_ids + + +@dataclass(frozen=True, slots=True) +class CalibrationIdentityMaterialization: + """Content-redacted token materialization consumed by a future live adapter.""" + + sequences: tuple[MaterializedCalibrationSequence, ...] + tokenizer_manifest_sha256: str + capture_input_sha256: str + + def __post_init__(self) -> None: + if not isinstance(self.sequences, tuple) or len(self.sequences) != 160: + raise ValueError("calibration materialization must contain exactly 160 sequences") + tokenizer_hash = _require_sha256( + self.tokenizer_manifest_sha256, + context="calibration materialization tokenizer manifest", + ) + _require_sha256( + self.capture_input_sha256, + context="calibration materialization capture input", + ) + records = [sequence.identity_record for sequence in self.sequences] + identities = [sequence.identity_record_sha256 for sequence in self.sequences] + if len(set(identities)) != len(identities): + raise ValueError("calibration materialization contains duplicate identities") + if any(record["tokenizer_manifest_sha256"] != tokenizer_hash for record in records): + raise ValueError("calibration materialization tokenizer commitments differ") + if records != sorted(records, key=resolver._record_sort_key): + raise ValueError("calibration materialization records are not in canonical order") + + @property + def identity_records(self) -> tuple[dict[str, Any], ...]: + """Return fresh record copies in canonical capture order.""" + + return tuple(sequence.identity_record for sequence in self.sequences) + + @property + def by_identity_record_sha256( + self, + ) -> Mapping[str, MaterializedCalibrationSequence]: + """Return an immutable digest lookup independent of traversal order.""" + + return MappingProxyType( + {sequence.identity_record_sha256: sequence for sequence in self.sequences} + ) + + def lookup(self, identity_record_sha256: str) -> MaterializedCalibrationSequence: + digest = _require_sha256( + identity_record_sha256, + context="calibration materialization identity lookup", + ) + try: + return self.by_identity_record_sha256[digest] + except KeyError as error: + raise KeyError(f"unknown calibration identity record: {digest}") from error + + @property + def token_sequence_manifest_sha256(self) -> str: + commitments = [ + { + "identity_record_sha256": sequence.identity_record_sha256, + "prompt_token_ids_sha256": _token_hash(sequence.prompt_token_ids), + "target_token_ids_sha256": _token_hash(sequence.target_token_ids), + "sequence_token_ids_sha256": _token_hash(sequence.sequence_token_ids), + "sequence_length": len(sequence.sequence_token_ids), + } + for sequence in self.sequences + ] + return sha256_bytes(canonical_json_bytes(commitments)) + + +TokenCaptureSink = dict[tuple[str, str], tuple[tuple[int, ...], tuple[int, ...]]] + + class CaptureSource(Protocol): """Narrow, mockable source surface used by the deterministic capturer.""" @@ -269,9 +772,11 @@ def pg19_row(self, split: str, *, offset: int, expected_url: str) -> Mapping[str def ruler_generator_files(self) -> Mapping[str, bytes]: ... - def ruler_receipt( + def ruler_generation_manifest_bytes(self) -> bytes: ... + + def ruler_receipt_bytes( self, *, category: str, config: str, configured_length: int, seed: int - ) -> Mapping[str, Any]: ... + ) -> bytes: ... def humaneval_projection(self) -> Sequence[ProjectionRow]: ... @@ -403,8 +908,11 @@ def _base_record( prompt_ids: Sequence[int], target_ids: Sequence[int], tokenizer_manifest_sha256: str, + token_sink: TokenCaptureSink | None = None, ) -> dict[str, Any]: - sequence_ids = tuple(prompt_ids) + tuple(target_ids) + prompt_token_ids = tuple(prompt_ids) + target_token_ids = tuple(target_ids) + sequence_ids = prompt_token_ids + target_token_ids if not sequence_ids: raise ValueError(f"{family} record {canonical_id!r} produced no tokens") if phase == "stage_a" and len(target_ids) < 2: @@ -436,7 +944,7 @@ def _base_record( selection_hash = resolver.selection_sha256(namespace, canonical_id) else: raise ValueError(f"no selection namespace for {phase}/{family}") - return { + record = { "family": family, "canonical_id": canonical_id, "config": config, @@ -460,6 +968,12 @@ def _base_record( token_span=token_span, ), } + if token_sink is not None: + key = (family, canonical_id) + if key in token_sink: + raise ValueError(f"duplicate materialized token identity: {family}/{canonical_id}") + token_sink[key] = (prompt_token_ids, target_token_ids) + return record def _assign_sha_ranks(records: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: @@ -559,6 +1073,7 @@ def _capture_mbpp( phase: str, tokenizer: Any, tokenizer_manifest_sha256: str, + token_sink: TokenCaptureSink | None = None, ) -> tuple[list[dict[str, Any]], str]: _selected_ids, population_hash = resolver.mbpp_calibration_identity() if phase == "stage_a": @@ -603,6 +1118,7 @@ def _capture_mbpp( prompt_ids=prompt_ids, target_ids=target_ids, tokenizer_manifest_sha256=tokenizer_manifest_sha256, + token_sink=token_sink, ) ) return _assign_sha_ranks(records), population_hash @@ -648,6 +1164,7 @@ def _capture_pg19( phase: str, tokenizer: Any, tokenizer_manifest_sha256: str, + token_sink: TokenCaptureSink | None = None, ) -> tuple[list[dict[str, Any]], str, str]: split = "train" if phase == "calibration" else "validation" required = 16 if phase == "calibration" else 4 @@ -725,6 +1242,7 @@ def _capture_pg19( prompt_ids=prompt_ids, target_ids=target_ids, tokenizer_manifest_sha256=tokenizer_manifest_sha256, + token_sink=token_sink, ) ) if len(accepted) == required: @@ -774,6 +1292,679 @@ def _ruler_generator_manifest(files: Mapping[str, bytes]) -> list[dict[str, Any] return manifest +def _ruler_runtime_packages() -> dict[str, str]: + raw = RULER_REQUIREMENTS_PATH.read_bytes() + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as error: + raise ValueError("RULER runtime requirements must be UTF-8") from error + result: dict[str, str] = {} + for line in text.splitlines(): + if not line or line.startswith("#"): + continue + if line.count("==") != 1: + raise ValueError("RULER runtime requirements must use exact == pins") + name, version = line.split("==") + if not name or not re.fullmatch(r"[0-9]+(?:\.[0-9A-Za-z]+)+", version): + raise ValueError("RULER runtime requirement is malformed") + if name in result: + raise ValueError("RULER runtime requirements contain a duplicate package") + result[name] = version + if not result: + raise ValueError("RULER runtime requirements cannot be empty") + return result + + +def _canonical_distribution_name(name: str) -> str: + return re.sub(r"[-_.]+", "-", name).lower() + + +def _normalize_bound_file( + value: object, + *, + context: str, + expected_name: str | None = None, + allow_empty: bool = False, +) -> dict[str, Any]: + if not isinstance(value, Mapping): + raise ValueError(f"{context} must be an object") + _require_exact_fields(value, AUXILIARY_FILE_FIELDS, context=context) + name = _require_string(value["name"], context=f"{context}.name") + if expected_name is not None and name != expected_name: + raise ValueError(f"{context} name drifted") + if ( + "\\" in name + or "\0" in name + or "\n" in name + or "\r" in name + or name.startswith("/") + or re.match(r"^[A-Za-z]:", name) + or any(part in {"", ".", ".."} for part in name.split("/")) + ): + raise ValueError(f"{context} name must be a canonical relative POSIX path") + return { + "name": name, + "sha256": _require_sha256(value["sha256"], context=f"{context}.sha256"), + "size_bytes": _require_int( + value["size_bytes"], + context=f"{context}.size_bytes", + minimum=0 if allow_empty else 1, + ), + } + + +def _normalize_ruler_runtime_manifest(value: object) -> dict[str, Any]: + if not isinstance(value, Mapping): + raise ValueError("RULER runtime manifest must be an object") + expected_fields = frozenset( + { + "schema", + "python", + "implementation", + "cache_tag", + "platform", + "machine", + "flags", + "startup_policy", + "excluded_startup_files", + "source_python", + "source_pyvenv_config", + "python_runtime_files", + "executable", + "packages", + "installed_distributions", + "distribution_file_inventory", + "forbidden_modules", + } + ) + _require_exact_fields(value, expected_fields, context="RULER runtime manifest") + if value["schema"] != RULER_RUNTIME_MANIFEST_SCHEMA: + raise ValueError("RULER runtime manifest schema drifted") + if value["python"] != RULER_RUNTIME_PYTHON_VERSION or value["implementation"] != "cpython": + raise ValueError("RULER runtime Python identity drifted") + for field in ("cache_tag", "platform", "machine"): + _require_string(value[field], context=f"RULER runtime {field}") + expected_flags = {"ignore_environment": 1, "isolated": 1, "no_user_site": 1} + flags = value["flags"] + if ( + not isinstance(flags, Mapping) + or set(flags) != set(expected_flags) + or any(type(flags[field]) is not int or flags[field] != 1 for field in expected_flags) + ): + raise ValueError("RULER runtime isolation flags drifted") + startup_policy = value["startup_policy"] + if ( + startup_policy != RULER_SEALED_STARTUP_POLICY + or not isinstance(startup_policy, Mapping) + or any( + type(startup_policy[field]) is not int or startup_policy[field] != 1 + for field in ("dont_write_bytecode", "no_site", "utf8_mode") + ) + or any( + type(startup_policy[field]) is not bool or startup_policy[field] is not False + for field in ("site_loaded", "virtualenv_hook_loaded") + ) + ): + raise ValueError("RULER runtime sealed-startup policy drifted") + excluded_startup_files = [ + { + "name": name, + "sha256": digest, + "size_bytes": size, + } + for name, (size, digest) in sorted(RULER_EXCLUDED_VIRTUALENV_STARTUP_FILES.items()) + ] + if value["excluded_startup_files"] != excluded_startup_files: + raise ValueError("RULER excluded startup-file identity drifted") + source_python = _normalize_bound_file( + value["source_python"], + context="RULER source Python launcher", + expected_name="source/python.exe", + ) + source_pyvenv_config = _normalize_bound_file( + value["source_pyvenv_config"], + context="RULER source pyvenv config", + expected_name="source/pyvenv.cfg", + ) + raw_python_runtime_files = value["python_runtime_files"] + if isinstance(raw_python_runtime_files, (str, bytes)) or not isinstance( + raw_python_runtime_files, Sequence + ): + raise ValueError("RULER Python runtime file inventory must be an array") + python_runtime_files = [ + _normalize_bound_file( + item, + context="RULER Python runtime file", + allow_empty=True, + ) + for item in raw_python_runtime_files + ] + runtime_names = [str(item["name"]) for item in python_runtime_files] + if ( + runtime_names != sorted(runtime_names) + or len(runtime_names) != len(set(runtime_names)) + or not {"python.exe", "python3.dll", "python311.dll"} <= set(runtime_names) + ): + raise ValueError("RULER Python runtime file inventory drifted") + executable = _normalize_bound_file( + value["executable"], context="RULER runtime executable", expected_name="python.exe" + ) + packages = _ruler_runtime_packages() + if value["packages"] != packages: + raise ValueError("RULER runtime package versions drifted") + installed = {_canonical_distribution_name(name): version for name, version in packages.items()} + if value["installed_distributions"] != installed: + raise ValueError("RULER runtime installed-distribution set drifted") + if value["forbidden_modules"] != {name: False for name in RULER_FORBIDDEN_RUNTIME_MODULES}: + raise ValueError("RULER runtime contains a forbidden model framework") + inventory = value["distribution_file_inventory"] + if not isinstance(inventory, Mapping) or set(inventory) != set(packages): + raise ValueError("RULER runtime distribution-file inventory drifted") + normalized_inventory: dict[str, Any] = {} + for package_name in packages: + raw_distribution = inventory[package_name] + if not isinstance(raw_distribution, Mapping): + raise ValueError("RULER package-code inventory must be an object") + _require_exact_fields( + raw_distribution, + frozenset({"canonical_name", "version", "record_sha256", "record_size_bytes", "files"}), + context=f"RULER package-code inventory {package_name}", + ) + if ( + raw_distribution["canonical_name"] != _canonical_distribution_name(package_name) + or raw_distribution["version"] != packages[package_name] + ): + raise ValueError(f"RULER package-code identity drifted for {package_name}") + record_sha256 = _require_sha256( + raw_distribution["record_sha256"], + context=f"RULER {package_name} RECORD SHA-256", + ) + record_size = _require_int( + raw_distribution["record_size_bytes"], + context=f"RULER {package_name} RECORD size", + minimum=1, + ) + raw_files = raw_distribution["files"] + if isinstance(raw_files, (str, bytes)) or not isinstance(raw_files, Sequence): + raise ValueError(f"RULER {package_name} file inventory must be an array") + files = [] + for raw_file in raw_files: + if not isinstance(raw_file, Mapping): + raise ValueError(f"RULER {package_name} installed file must be an object") + _require_exact_fields( + raw_file, + frozenset({"path", "sha256", "size_bytes"}), + context=f"RULER {package_name} installed file", + ) + path = _require_string(raw_file["path"], context=f"RULER {package_name} installed path") + if ( + "\\" in path + or "\0" in path + or "\n" in path + or "\r" in path + or path.startswith("/") + or re.match(r"^[A-Za-z]:", path) + ): + raise ValueError(f"RULER {package_name} installed path is not portable") + files.append( + { + "name": path, + "sha256": _require_sha256( + raw_file["sha256"], + context=f"RULER {package_name} installed SHA-256", + ), + "size_bytes": _require_int( + raw_file["size_bytes"], + context=f"RULER {package_name} installed size", + minimum=0, + ), + } + ) + paths = [item["name"] for item in files] + if paths != sorted(paths) or len(paths) != len(set(paths)) or not paths: + raise ValueError(f"RULER {package_name} installed paths drifted") + records = [item for item in files if item["name"].endswith(".dist-info/RECORD")] + if len(records) != 1 or ( + records[0]["sha256"] != record_sha256 or records[0]["size_bytes"] != record_size + ): + raise ValueError(f"RULER {package_name} RECORD binding drifted") + normalized_inventory[package_name] = { + "canonical_name": raw_distribution["canonical_name"], + "version": raw_distribution["version"], + "record_sha256": record_sha256, + "record_size_bytes": record_size, + "files": [ + {"path": item["name"], "sha256": item["sha256"], "size_bytes": item["size_bytes"]} + for item in files + ], + } + return { + "schema": RULER_RUNTIME_MANIFEST_SCHEMA, + "python": value["python"], + "implementation": value["implementation"], + "cache_tag": value["cache_tag"], + "platform": value["platform"], + "machine": value["machine"], + "flags": expected_flags, + "startup_policy": dict(RULER_SEALED_STARTUP_POLICY), + "excluded_startup_files": excluded_startup_files, + "source_python": source_python, + "source_pyvenv_config": source_pyvenv_config, + "python_runtime_files": python_runtime_files, + "executable": executable, + "packages": packages, + "installed_distributions": installed, + "distribution_file_inventory": normalized_inventory, + "forbidden_modules": dict(value["forbidden_modules"]), + } + + +def _expected_ruler_static_inputs( + *, + source_manifest: Sequence[Mapping[str, Any]], + runtime_manifest: Mapping[str, Any], +) -> list[dict[str, Any]]: + entries: list[dict[str, Any]] = [] + for name, (size, digest) in RULER_EXPECTED_CORPORA.items(): + entries.append({"name": f"corpora/{name}", "sha256": digest, "size_bytes": size}) + for name, (size, digest) in RULER_EXPECTED_PACKAGE_RESOURCES.items(): + entries.append({"name": f"packages/{name}", "sha256": digest, "size_bytes": size}) + for name, (size, digest) in RULER_EXPECTED_TOKENIZER_ASSETS.items(): + entries.append({"name": f"tokenizer/{name}", "sha256": digest, "size_bytes": size}) + requirements = RULER_REQUIREMENTS_PATH.read_bytes() + launcher = RULER_LAUNCHER_PATH.read_bytes() + entries.extend( + ( + { + "name": "runtime/package-manifest.json", + "sha256": sha256_bytes(canonical_json_bytes(runtime_manifest)), + "size_bytes": len(canonical_json_bytes(runtime_manifest)), + }, + { + "name": "runtime/requirements.txt", + "sha256": sha256_bytes(requirements), + "size_bytes": len(requirements), + }, + { + "name": "launcher/generate_static_q468_ruler_receipts.py", + "sha256": sha256_bytes(launcher), + "size_bytes": len(launcher), + }, + { + "name": "ruler/source-manifest.json", + "sha256": sha256_bytes(canonical_json_bytes(list(source_manifest))), + "size_bytes": len(canonical_json_bytes(list(source_manifest))), + }, + ) + ) + return sorted(entries, key=lambda item: item["name"]) + + +def _replay_ruler_task_invariants( + *, + config: str, + input_text: str, + answer_prefix: str, + outputs: Sequence[str], +) -> None: + """Replay the frozen task semantics without trusting launcher-derived claims.""" + + try: + invariant = RULER_TASK_INVARIANTS[config] + except KeyError as error: + raise ValueError("RULER task invariant is unavailable") from error + if ( + not input_text.startswith(invariant.input_prefix) + or invariant.input_marker not in input_text + or not input_text.endswith(invariant.input_suffix) + ): + raise ValueError(f"RULER {config} input task markers drifted") + if ( + not answer_prefix.startswith(invariant.answer_prefix_marker) + or not answer_prefix.endswith(invariant.answer_prefix_suffix) + ): + raise ValueError(f"RULER {config} answer-prefix boundaries drifted") + if ( + invariant.expected_output_count is not None + and len(outputs) != invariant.expected_output_count + ): + raise ValueError( + f"RULER receipt {config} must contain exactly " + f"{invariant.expected_output_count} required outputs" + ) + if invariant.unique_outputs and len(set(outputs)) != len(outputs): + raise ValueError(f"RULER {config} outputs must be unique") + if invariant.output_pattern is not None and any( + re.fullmatch(invariant.output_pattern, output) is None for output in outputs + ): + raise ValueError(f"RULER {config} output format drifted") + if invariant.outputs_must_appear and any(output not in input_text for output in outputs): + raise ValueError(f"RULER {config} required answer is absent from its input") + + +def _verify_ruler_raw_row( + raw_data: bytes, + *, + receipt: Mapping[str, Any], + tokenizer: Any, +) -> None: + lines = raw_data.splitlines() + if len(lines) != 1 or not lines[0]: + raise ValueError("RULER raw validation must contain exactly one JSON row") + row = _strict_json(lines[0], context="RULER raw validation row") + config = str(receipt["config"]) + niah = config in RULER_NIAH_CONFIGS + expected_fields = { + "index", + "input", + "outputs", + "length", + "length_w_model_temp", + "answer_prefix", + } | ({"token_position_answer"} if niah else set()) + if set(row) != expected_fields: + raise ValueError("RULER raw validation fields drifted") + row_length = _require_int( + row["length"], context="RULER raw validation length", minimum=1 + ) + row_length_with_template = _require_int( + row["length_w_model_temp"], + context="RULER raw validation length_w_model_temp", + minimum=1, + ) + row_index = _require_int(row["index"], context="RULER raw validation index") + if ( + row["input"] != receipt["input"] + or row["outputs"] != receipt["outputs"] + or row["answer_prefix"] != receipt["answer_prefix"] + or row_length != receipt["generator_reported_length"] + or row_length_with_template != row_length + ): + raise ValueError("RULER receipt does not reproduce its raw generator row") + input_text = _require_string(row["input"], context="RULER raw validation input") + answer_prefix = _require_string( + row["answer_prefix"], + context="RULER raw validation answer_prefix", + allow_empty=True, + ) + raw_outputs = row["outputs"] + if isinstance(raw_outputs, (str, bytes)) or not isinstance(raw_outputs, Sequence): + raise ValueError("RULER raw validation outputs must be an array") + outputs = tuple( + _require_string(value, context="RULER raw validation output") for value in raw_outputs + ) + if not outputs: + raise ValueError("RULER raw validation outputs cannot be empty") + _replay_ruler_task_invariants( + config=config, + input_text=input_text, + answer_prefix=answer_prefix, + outputs=outputs, + ) + expected_length = ( + len( + _encode( + tokenizer, + input_text + answer_prefix, + add_special_tokens=False, + ) + ) + + RULER_GENERATOR_TOKENS[config] + ) + if row_length != expected_length: + raise ValueError("RULER raw length disagrees with independent tokenization") + if niah: + first_output = outputs[0] + index = input_text.find(first_output) + if index < 0 or row_index != index: + raise ValueError("RULER NIAH raw answer position drifted") + expected_position = len( + _encode(tokenizer, input_text[:index], add_special_tokens=False) + ) + token_position = _require_int( + row["token_position_answer"], + context="RULER raw validation token_position_answer", + ) + if token_position != expected_position: + raise ValueError("RULER NIAH raw token position drifted") + elif row_index != 0: + raise ValueError("RULER non-NIAH raw index must equal zero") + + +def _verify_complete_ruler_bundle( + source: CaptureSource, + *, + tokenizer_material: TokenizerMaterial, +) -> VerifiedRulerBundle: + raw_manifest = source.ruler_generation_manifest_bytes() + manifest = _strict_json(raw_manifest, context="RULER generation manifest") + if canonical_json_bytes(manifest) != raw_manifest: + raise ValueError("RULER generation manifest is not canonical JSON") + _require_exact_fields( + manifest, + frozenset( + { + "schema", + "launcher_revision", + "launcher_source", + "ruler_revision", + "source_manifest", + "source_manifest_sha256", + "runtime_manifest", + "runtime_manifest_sha256", + "static_inputs", + "receipt_count", + "receipts", + } + ), + context="RULER generation manifest", + ) + receipt_count = _require_int( + manifest["receipt_count"], + context="RULER generation manifest receipt_count", + minimum=1, + ) + if ( + manifest["schema"] != RULER_GENERATION_MANIFEST_SCHEMA + or manifest["launcher_revision"] != RULER_LAUNCHER_REVISION + or manifest["ruler_revision"] != resolver.RULER_REVISION + or receipt_count != 20 + ): + raise ValueError("RULER generation manifest identity drifted") + generator_manifest = _ruler_generator_manifest(source.ruler_generator_files()) + if manifest["source_manifest"] != generator_manifest or manifest[ + "source_manifest_sha256" + ] != sha256_bytes(canonical_json_bytes(generator_manifest)): + raise ValueError("RULER generation source manifest drifted") + runtime_manifest = _normalize_ruler_runtime_manifest(manifest["runtime_manifest"]) + if manifest["runtime_manifest"] != runtime_manifest or manifest[ + "runtime_manifest_sha256" + ] != sha256_bytes(canonical_json_bytes(runtime_manifest)): + raise ValueError("RULER runtime manifest binding drifted") + expected_static = _expected_ruler_static_inputs( + source_manifest=generator_manifest, + runtime_manifest=runtime_manifest, + ) + if manifest["static_inputs"] != expected_static: + raise ValueError("RULER static-input inventory drifted") + launcher_entry = _normalize_bound_file( + manifest["launcher_source"], + context="RULER launcher source", + expected_name="launcher/generate_static_q468_ruler_receipts.py", + ) + launcher_expected = next( + item for item in expected_static if item["name"] == launcher_entry["name"] + ) + if launcher_entry != launcher_expected: + raise ValueError("RULER launcher source bytes drifted") + material_files = { + name: (len(data), sha256_bytes(data)) for name, data in tokenizer_material.files.items() + } + if material_files != RULER_EXPECTED_TOKENIZER_ASSETS: + raise ValueError("capture tokenizer assets differ from the RULER tokenizer contract") + required = required_ruler_receipts() + raw_results = manifest["receipts"] + if not isinstance(raw_results, list) or len(raw_results) != len(required): + raise ValueError("RULER generation manifest must contain all 20 receipt results") + if set(RULER_COMMAND_MANIFEST_SHA256_BY_FILENAME) != { + str(item["filename"]) for item in required + }: + raise RuntimeError("frozen RULER command-manifest hash inventory is incomplete") + verified: dict[str, dict[str, Any]] = {} + static_by_name = {item["name"]: item for item in expected_static} + result_fields = frozenset( + { + "category", + "command_manifest", + "command_manifest_file", + "config", + "configured_length", + "filename", + "generator_reported_length", + "phase", + "raw_validation_base64", + "raw_validation_file", + "seed", + "sha256", + "size_bytes", + } + ) + for expected, result in zip(required, raw_results, strict=True): + if not isinstance(result, Mapping): + raise ValueError("RULER receipt result must be an object") + _require_exact_fields(result, result_fields, context="RULER receipt result") + for field, minimum in ( + ("configured_length", 1), + ("seed", 0), + ("generator_reported_length", 1), + ("size_bytes", 1), + ): + _require_int( + result[field], + context=f"RULER receipt result {field}", + minimum=minimum, + ) + for field in ( + "category", + "config", + "configured_length", + "filename", + "phase", + "seed", + ): + if result[field] != expected[field]: + raise ValueError(f"RULER receipt result {field} drifted") + filename = str(expected["filename"]) + receipt_bytes = source.ruler_receipt_bytes( + category=str(expected["category"]), + config=str(expected["config"]), + configured_length=int(expected["configured_length"]), + seed=int(expected["seed"]), + ) + receipt_value = _strict_json(receipt_bytes, context=f"RULER receipt {filename}") + if canonical_json_bytes(receipt_value) != receipt_bytes: + raise ValueError(f"RULER receipt is not canonical: {filename}") + receipt = _normalize_ruler_receipt( + receipt_value, + category=str(expected["category"]), + config=str(expected["config"]), + configured_length=int(expected["configured_length"]), + seed=int(expected["seed"]), + ) + if receipt != receipt_value: + raise ValueError(f"RULER receipt normalization drifted: {filename}") + if ( + result["sha256"] != sha256_bytes(receipt_bytes) + or result["size_bytes"] != len(receipt_bytes) + or result["generator_reported_length"] != receipt["generator_reported_length"] + ): + raise ValueError(f"RULER receipt file identity drifted: {filename}") + command = result["command_manifest"] + if not isinstance(command, Mapping): + raise ValueError("RULER command manifest must be an object") + _require_exact_fields( + command, + frozenset( + { + "launcher_revision", + "launcher_source_sha256", + "ruler_revision", + "config", + "configured_length", + "seed", + "argv", + "shell", + } + ), + context="RULER command manifest", + ) + _require_int( + command["configured_length"], + context="RULER command configured_length", + minimum=1, + ) + _require_int(command["seed"], context="RULER command seed") + if ( + command["launcher_revision"] != RULER_LAUNCHER_REVISION + or command["launcher_source_sha256"] != launcher_entry["sha256"] + or command["ruler_revision"] != resolver.RULER_REVISION + or command["config"] != expected["config"] + or command["configured_length"] != expected["configured_length"] + or command["seed"] != expected["seed"] + or command["shell"] is not False + ): + raise ValueError(f"RULER command identity drifted: {filename}") + command_bytes = canonical_json_bytes(command) + if sha256_bytes(command_bytes) != RULER_COMMAND_MANIFEST_SHA256_BY_FILENAME[filename]: + raise ValueError(f"RULER command argv drifted: {filename}") + command_file = _normalize_bound_file( + result["command_manifest_file"], + context="RULER command-manifest file", + expected_name="generator/command-manifest.json", + ) + if command_file != { + "name": "generator/command-manifest.json", + "sha256": sha256_bytes(command_bytes), + "size_bytes": len(command_bytes), + }: + raise ValueError(f"RULER command-manifest bytes drifted: {filename}") + encoded_raw = _require_string( + result["raw_validation_base64"], context="RULER raw validation base64" + ) + try: + raw_data = base64.b64decode(encoded_raw, validate=True) + except (ValueError, binascii.Error) as error: + raise ValueError("RULER raw validation is not canonical base64") from error + if base64.b64encode(raw_data).decode("ascii") != encoded_raw: + raise ValueError("RULER raw validation base64 encoding drifted") + raw_file = _normalize_bound_file( + result["raw_validation_file"], + context="RULER raw-validation file", + expected_name="generator/raw-validation.jsonl", + ) + if raw_file != { + "name": "generator/raw-validation.jsonl", + "sha256": sha256_bytes(raw_data), + "size_bytes": len(raw_data), + }: + raise ValueError(f"RULER raw-validation bytes drifted: {filename}") + _verify_ruler_raw_row(raw_data, receipt=receipt, tokenizer=tokenizer_material.tokenizer) + auxiliary_by_name = {item["name"]: item for item in receipt["auxiliary_files"]} + expected_auxiliary = { + **static_by_name, + command_file["name"]: command_file, + raw_file["name"]: raw_file, + } + if auxiliary_by_name != expected_auxiliary: + raise ValueError(f"RULER receipt auxiliary inventory drifted: {filename}") + verified[filename] = receipt + return VerifiedRulerBundle( + receipts=verified, + generator_manifest=tuple(generator_manifest), + generation_manifest_sha256=sha256_bytes(raw_manifest), + ) + + def _normalize_auxiliary_files(value: object, *, context: str) -> list[dict[str, Any]]: if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): raise ValueError(f"{context} must be an array") @@ -830,6 +2021,12 @@ def _normalize_ruler_receipt( "seed": seed, "sample_index": 0, } + for field, minimum in ( + ("configured_length", 1), + ("seed", 0), + ("sample_index", 0), + ): + _require_int(value[field], context=f"RULER receipt {field}", minimum=minimum) for field, expected_value in expected.items(): if value[field] != expected_value: raise ValueError(f"RULER receipt {field} drifted") @@ -850,14 +2047,12 @@ def _normalize_ruler_receipt( if isinstance(outputs, (str, bytes)) or not isinstance(outputs, Sequence) or not outputs: raise ValueError("RULER receipt outputs must be a non-empty string array") normalized_outputs = [_require_string(item, context="RULER receipt output") for item in outputs] - required_count = RULER_REQUIRED_OUTPUT_COUNTS.get(config) - if required_count is not None: - if len(normalized_outputs) != required_count: - raise ValueError( - f"RULER receipt {config} must contain exactly {required_count} required outputs" - ) - if len(set(normalized_outputs)) != len(normalized_outputs): - raise ValueError(f"RULER receipt {config} required outputs must be unique") + _replay_ruler_task_invariants( + config=config, + input_text=input_text, + answer_prefix=answer_prefix, + outputs=normalized_outputs, + ) auxiliary_files = _normalize_auxiliary_files( value["auxiliary_files"], context="RULER receipt auxiliary_files" ) @@ -887,13 +2082,14 @@ def _ruler_stage_a_target(*, category: str, config: str, outputs: Sequence[str]) def _capture_ruler( - source: CaptureSource, *, phase: str, tokenizer: Any, tokenizer_manifest_sha256: str, + bundle: VerifiedRulerBundle, + token_sink: TokenCaptureSink | None = None, ) -> tuple[list[dict[str, Any]], str, str]: - generator_manifest = _ruler_generator_manifest(source.ruler_generator_files()) + generator_manifest = list(bundle.generator_manifest) schedule = ( resolver.RULER_CALIBRATION_SCHEDULE if phase == "calibration" @@ -904,12 +2100,14 @@ def _capture_ruler( selected_identities: list[dict[str, Any]] = [] for category, config, configured_length, seed in schedule: receipt = _normalize_ruler_receipt( - source.ruler_receipt( - category=category, - config=config, - configured_length=configured_length, - seed=seed, - ), + bundle.receipts[ + ruler_receipt_filename( + category=category, + config=config, + configured_length=configured_length, + seed=seed, + ) + ], category=category, config=config, configured_length=configured_length, @@ -971,6 +2169,7 @@ def _capture_ruler( prompt_ids=prompt_ids, target_ids=target_ids, tokenizer_manifest_sha256=tokenizer_manifest_sha256, + token_sink=token_sink, ) ) auxiliary_manifests.append( @@ -1018,6 +2217,7 @@ def _capture_ruler( "minimum_stage_a_target_tokens": 2, "cache_exposed_predictions": "target_length - 1", "generator_files": generator_manifest, + "complete_generation_manifest_sha256": bundle.generation_manifest_sha256, "auxiliary_receipts": auxiliary_manifests, } formatter_hash = sha256_bytes(canonical_json_bytes(formatter)) @@ -1030,6 +2230,7 @@ def _capture_humaneval( phase: str, tokenizer: Any, tokenizer_manifest_sha256: str, + token_sink: TokenCaptureSink | None = None, ) -> tuple[list[dict[str, Any]], str]: projection = _validate_projection( source.humaneval_projection(), @@ -1077,6 +2278,7 @@ def _capture_humaneval( prompt_ids=prompt_ids, target_ids=target_ids, tokenizer_manifest_sha256=tokenizer_manifest_sha256, + token_sink=token_sink, ) ) return _assign_sha_ranks(records), identity_manifest_hash @@ -1146,16 +2348,288 @@ def _normalize_calibration_binding(value: object) -> dict[str, str]: return dict(verified.binding) -def capture_identity_input( +def _normalize_runtime_authentication_context( + value: object, +) -> _RuntimeAuthenticationContext: + if not isinstance(value, Mapping): + raise ValueError("runtime_authentication_context must be a mapping") + _require_exact_fields( + value, + RUNTIME_AUTHENTICATION_CONTEXT_FIELDS, + context="runtime_authentication_context", + ) + base_runtime_root = value["base_runtime_root"] + staged_interpreter = value["staged_interpreter"] + if not isinstance(base_runtime_root, Path) or not isinstance(staged_interpreter, Path): + raise ValueError("runtime authentication roots and interpreter must be Path values") + raw_roots = value["package_runtime_roots"] + raw_import_paths = value["package_import_paths"] + if not isinstance(raw_roots, Mapping) or not raw_roots: + raise ValueError("package_runtime_roots must be a non-empty mapping") + if not isinstance(raw_import_paths, Mapping) or not raw_import_paths: + raise ValueError("package_import_paths must be a non-empty mapping") + roots: dict[str, Path] = {} + for raw_name, raw_path in raw_roots.items(): + if not isinstance(raw_name, str) or _RUNTIME_ROOT_NAME_RE.fullmatch(raw_name) is None: + raise ValueError("package runtime root name is not canonical") + if not isinstance(raw_path, Path): + raise ValueError("package runtime root must be a Path") + roots[raw_name] = raw_path + import_paths: dict[str, str] = {} + for raw_name, raw_path in raw_import_paths.items(): + if not isinstance(raw_name, str) or _RUNTIME_ROOT_NAME_RE.fullmatch(raw_name) is None: + raise ValueError("package import-path name is not canonical") + if not isinstance(raw_path, str) or not raw_path or "\\" in raw_path: + raise ValueError("package import path must be canonical relative POSIX text") + relative = PurePosixPath(raw_path) + if ( + relative.is_absolute() + or relative.as_posix() != raw_path + or not relative.parts + or any(part in {"", ".", ".."} for part in relative.parts) + ): + raise ValueError("package import path must be canonical relative POSIX text") + import_paths[raw_name] = raw_path + if set(roots) != set(import_paths): + raise ValueError("package runtime-root and import-path names differ") + return _RuntimeAuthenticationContext( + base_runtime_root=base_runtime_root, + staged_interpreter=staged_interpreter, + package_runtime_roots=MappingProxyType(dict(sorted(roots.items()))), + package_import_paths=MappingProxyType(dict(sorted(import_paths.items()))), + ) + + +def _load_calibration_runner_module() -> Any: + if _CALIBRATION_RUNNER_MODULE_NAME in sys.modules: + raise RuntimeError("refusing a preloaded Experiment 013 calibration runner") + spec = importlib.util.spec_from_file_location( + _CALIBRATION_RUNNER_MODULE_NAME, + CALIBRATION_RUNNER_PATH, + ) + if spec is None or spec.loader is None: # pragma: no cover + raise RuntimeError("cannot load the Experiment 013 calibration artifact validators") + module = importlib.util.module_from_spec(spec) + sys.modules[_CALIBRATION_RUNNER_MODULE_NAME] = module + try: + spec.loader.exec_module(module) + except BaseException: + sys.modules.pop(_CALIBRATION_RUNNER_MODULE_NAME, None) + raise + return module + + +def _decode_execution_binding_artifacts( + artifacts: Mapping[str, bytes], + *, + runner: Any, +) -> _DecodedExecutionBindingArtifacts: + """Strictly decode each execution artifact before binding its exact bytes.""" + + if not isinstance(artifacts, Mapping) or set(artifacts) != set( + resolver.EXECUTION_BINDING_FIELDS + ): + raise ValueError("all four verified execution-binding artifacts are required") + for field, data in artifacts.items(): + if not isinstance(data, bytes) or not data: + raise ValueError(f"execution-binding artifact {field} must be non-empty bytes") + source_bytes = artifacts["repository_source_manifest_file_sha256"] + runtime_bytes = artifacts["calibration_runtime_manifest_file_sha256"] + model_bytes = artifacts["model_file_manifest_file_sha256"] + parquet_bytes = artifacts["parquet_materialization_manifest_file_sha256"] + try: + from recurquant import experiment013_parquet, experiment013_source + except ImportError as error: # pragma: no cover - installation guard + raise RuntimeError("Experiment 013 source/Parquet validators are unavailable") from error + source_value = _strict_json(source_bytes, context="repository source manifest") + normalized_source = experiment013_source.validate_experiment013_source_manifest(source_value) + if ( + experiment013_source.canonical_experiment013_source_manifest_bytes(normalized_source) + != source_bytes + ): + raise ValueError("repository source manifest is not canonical JSON") + runtime_manifest = runner.parse_calibration_runtime_manifest(runtime_bytes) + model_manifest = runner.parse_model_file_manifest(model_bytes) + if runtime_manifest.file_sha256 != sha256_bytes(runtime_bytes): + raise ValueError("calibration runtime manifest file identity drifted") + if ( + model_manifest.file_sha256 != sha256_bytes(model_bytes) + or model_manifest.model_id != resolver.PRIMARY_MODEL_ID + or model_manifest.revision != resolver.PRIMARY_MODEL_REVISION + or model_manifest.transformers_version != resolver.TRANSFORMERS_VERSION + ): + raise ValueError("model file manifest does not match the frozen primary model") + parquet_path = Path(experiment013_parquet.EXPERIMENT013_PARQUET_MANIFEST_PATH) + if ( + parquet_path.resolve(strict=True) + != PARQUET_MATERIALIZATION_MANIFEST_PATH.resolve(strict=True) + or experiment013_parquet.EXPERIMENT013_PARQUET_MANIFEST_SHA256 + != resolver.PARQUET_MATERIALIZATION_MANIFEST_FILE_SHA256 + or sha256_bytes(parquet_bytes) + != resolver.PARQUET_MATERIALIZATION_MANIFEST_FILE_SHA256 + or parquet_bytes != parquet_path.read_bytes() + ): + raise ValueError("Parquet materialization manifest file identity drifted") + experiment013_parquet.load_experiment013_parquet_manifest(parquet_path) + bindings = { + field: sha256_bytes(artifacts[field]) for field in sorted(resolver.EXECUTION_BINDING_FIELDS) + } + return _DecodedExecutionBindingArtifacts( + bindings=MappingProxyType(bindings), + source_manifest=MappingProxyType(dict(normalized_source)), + runtime_manifest=runtime_manifest, + model_manifest=model_manifest, + source_module=experiment013_source, + parquet_module=experiment013_parquet, + ) + + +def _validate_execution_binding_artifacts( + artifacts: Mapping[str, bytes], +) -> dict[str, str]: + """Strictly decode all four artifacts without retaining an imported runner.""" + + runner = _load_calibration_runner_module() + try: + decoded = _decode_execution_binding_artifacts(artifacts, runner=runner) + return dict(decoded.bindings) + finally: + if sys.modules.get(_CALIBRATION_RUNNER_MODULE_NAME) is runner: + sys.modules.pop(_CALIBRATION_RUNNER_MODULE_NAME, None) + + +def _verify_loaded_runner_source( + runner: Any, + source_manifest: Mapping[str, object], +) -> None: + entries = { + str(item["path"]): item + for item in source_manifest["paths"] # type: ignore[index] + } + relative = "scripts/run_static_q468_calibration.py" + entry = entries.get(relative) + raw_file = getattr(runner, "__file__", None) + if not isinstance(entry, Mapping) or not isinstance(raw_file, (str, os.PathLike)): + raise RuntimeError("calibration runner is absent from the authenticated source manifest") + declared = Path(raw_file) + try: + resolved = declared.resolve(strict=True) + except OSError as error: + raise RuntimeError("calibration runner source is unavailable") from error + if ( + declared.is_symlink() + or resolved != CALIBRATION_RUNNER_PATH.resolve(strict=True) + or sha256_bytes(resolved.read_bytes()) != entry["raw_sha256"] + ): + raise RuntimeError("loaded calibration runner source bytes drifted") + + +def _validate_runtime_context_for_manifest( + context: _RuntimeAuthenticationContext, + runtime_manifest: Any, +) -> None: + expected_import_paths = { + item.name: item.import_path for item in runtime_manifest.package_roots + } + if dict(context.package_import_paths) != expected_import_paths: + raise ValueError("runtime package import paths differ from the frozen manifest") + if set(context.package_runtime_roots) != set(expected_import_paths): + raise ValueError("runtime package root names differ from the frozen manifest") + for name, raw_root in context.package_runtime_roots.items(): + try: + root = raw_root.resolve(strict=True) + import_root = (root / PurePosixPath(context.package_import_paths[name])).resolve( + strict=True + ) + import_root.relative_to(root) + except (OSError, ValueError) as error: + raise ValueError("runtime package import path escapes or is unavailable") from error + if not root.is_dir() or not import_root.is_dir(): + raise ValueError("runtime package root and import path must be directories") + + +def _authenticate_execution_binding_artifacts( + artifacts: Mapping[str, bytes], + *, + runtime_context: _RuntimeAuthenticationContext, + previous: _AuthenticatedExecutionBindings | None = None, +) -> _AuthenticatedExecutionBindings: + """Reauthenticate source, runtime, model metadata, and Parquet at point of use.""" + + created_runner = previous is None + if created_runner: + runner = _load_calibration_runner_module() + else: + runner = previous.runner + if sys.modules.get(_CALIBRATION_RUNNER_MODULE_NAME) is not runner: + raise RuntimeError("authenticated calibration runner module binding drifted") + if runtime_context != previous.runtime_context: + raise ValueError("runtime authentication context changed during capture") + try: + decoded = _decode_execution_binding_artifacts(artifacts, runner=runner) + if previous is not None and dict(decoded.bindings) != dict(previous.bindings): + raise ValueError("execution-binding artifacts changed during capture") + verified_source = decoded.source_module.verify_experiment013_source_manifest( + decoded.source_manifest, + repo_root=REPOSITORY_ROOT, + ) + if verified_source != dict(decoded.source_manifest): + raise RuntimeError("repository source verifier returned a different manifest") + decoded.source_module.verify_loaded_experiment013_recurquant_modules( + decoded.source_manifest, + REPOSITORY_ROOT, + ( + "recurquant.experiment013_source", + "recurquant.experiment013_parquet", + ), + ) + _verify_loaded_runner_source(runner, decoded.source_manifest) + _validate_runtime_context_for_manifest(runtime_context, decoded.runtime_manifest) + authenticated_runtime = runner.authenticate_calibration_runtime( + decoded.runtime_manifest, + base_runtime_root=runtime_context.base_runtime_root, + package_roots=runtime_context.package_runtime_roots, + interpreter_path=runtime_context.staged_interpreter, + ) + if ( + authenticated_runtime.manifest_file_sha256 + != decoded.bindings["calibration_runtime_manifest_file_sha256"] + ): + raise RuntimeError("runtime authenticator returned a different manifest identity") + live_model_manifest = runner.capture_model_file_manifest_from_hub( + resolver.PRIMARY_MODEL_ID, + resolver.PRIMARY_MODEL_REVISION, + transformers_version=resolver.TRANSFORMERS_VERSION, + ) + if live_model_manifest != artifacts["model_file_manifest_file_sha256"]: + raise ValueError("pinned model Hub metadata differs from the frozen manifest") + return _AuthenticatedExecutionBindings( + bindings=decoded.bindings, + source_manifest=decoded.source_manifest, + runtime_manifest=decoded.runtime_manifest, + model_manifest=decoded.model_manifest, + runner=runner, + runtime_context=runtime_context, + ) + except BaseException: + if created_runner and sys.modules.get(_CALIBRATION_RUNNER_MODULE_NAME) is runner: + sys.modules.pop(_CALIBRATION_RUNNER_MODULE_NAME, None) + raise + + +def _capture_identity_input_with_tokens( *, phase: str, source: CaptureSource, calibration_binding: bytes | None = None, -) -> dict[str, Any]: - """Capture one deterministic calibration or Stage-A resolver input.""" + execution_binding_artifacts: Mapping[str, bytes] | None = None, + runtime_authentication_context: Mapping[str, object] | None = None, + collect_tokens: bool, +) -> tuple[dict[str, Any], TokenCaptureSink]: + """Run the sole capture flow, optionally retaining formatter token IDs.""" if phase in resolver.PROTECTED_STAGES: - raise PermissionError(f"{phase} is protected; capture v2 refuses it before source access") + raise PermissionError(f"{phase} is protected; capture v4 refuses it before source access") if phase not in resolver.ALLOWED_PHASES: raise ValueError(f"unsupported identity phase: {phase!r}") if phase == "stage_a" and calibration_binding is None: @@ -1163,66 +2637,166 @@ def capture_identity_input( if phase == "calibration" and calibration_binding is not None: raise ValueError("calibration capture forbids a Stage-A binding") - before = _validate_heads(source.source_heads(), context="pre-capture") - material = source.tokenizer_material() - tokenizer_contract, tokenizer_manifest_hash = _tokenizer_contract(material) - mbpp_records, mbpp_manifest_hash = _capture_mbpp( - source, - phase=phase, - tokenizer=material.tokenizer, - tokenizer_manifest_sha256=tokenizer_manifest_hash, + if runtime_authentication_context is None: + runtime_provider = getattr(source, "runtime_authentication_context", None) + if runtime_provider is None or not callable(runtime_provider): + raise ValueError("capture requires an explicit sealed runtime authentication context") + runtime_authentication_context = runtime_provider() + runtime_context = _normalize_runtime_authentication_context( + runtime_authentication_context ) - pg19_records, pg19_manifest_hash, pg19_split = _capture_pg19( - source, - phase=phase, - tokenizer=material.tokenizer, - tokenizer_manifest_sha256=tokenizer_manifest_hash, + if execution_binding_artifacts is None: + fixture_provider = getattr(source, "execution_binding_artifacts", None) + if fixture_provider is None or not callable(fixture_provider): + raise ValueError("capture requires all four verified execution-binding artifacts") + execution_binding_artifacts = fixture_provider() + authentication = _authenticate_execution_binding_artifacts( + execution_binding_artifacts, + runtime_context=runtime_context, ) - ruler_records, ruler_manifest_hash, ruler_formatter_hash = _capture_ruler( - source, + try: + before = _validate_heads(source.source_heads(), context="pre-capture") + material = source.tokenizer_material() + tokenizer_contract, tokenizer_manifest_hash = _tokenizer_contract(material) + ruler_bundle = _verify_complete_ruler_bundle(source, tokenizer_material=material) + token_sink: TokenCaptureSink | None = {} if collect_tokens else None + mbpp_records, mbpp_manifest_hash = _capture_mbpp( + source, + phase=phase, + tokenizer=material.tokenizer, + tokenizer_manifest_sha256=tokenizer_manifest_hash, + token_sink=token_sink, + ) + pg19_records, pg19_manifest_hash, pg19_split = _capture_pg19( + source, + phase=phase, + tokenizer=material.tokenizer, + tokenizer_manifest_sha256=tokenizer_manifest_hash, + token_sink=token_sink, + ) + ruler_records, ruler_manifest_hash, ruler_formatter_hash = _capture_ruler( + phase=phase, + tokenizer=material.tokenizer, + tokenizer_manifest_sha256=tokenizer_manifest_hash, + bundle=ruler_bundle, + token_sink=token_sink, + ) + humaneval_records, humaneval_manifest_hash = _capture_humaneval( + source, + phase=phase, + tokenizer=material.tokenizer, + tokenizer_manifest_sha256=tokenizer_manifest_hash, + token_sink=token_sink, + ) + after = _validate_heads(source.source_heads(), context="post-capture") + if after != before: # Defensive even though both were pinned. + raise ValueError("source HEAD changed during capture") + _authenticate_execution_binding_artifacts( + execution_binding_artifacts, + runtime_context=runtime_context, + previous=authentication, + ) + + result: dict[str, Any] = { + "schema": resolver.INPUT_SCHEMA, + "phase": phase, + "datasets": _dataset_contracts( + mbpp_manifest_hash=mbpp_manifest_hash, + pg19_manifest_hash=pg19_manifest_hash, + pg19_split=pg19_split, + ruler_manifest_hash=ruler_manifest_hash, + ruler_formatter_hash=ruler_formatter_hash, + humaneval_manifest_hash=humaneval_manifest_hash, + ), + "tokenizer": tokenizer_contract, + "execution_bindings": dict(authentication.bindings), + "records": [ + *mbpp_records, + *pg19_records, + *ruler_records, + *humaneval_records, + ], + "model_weights_loaded": False, + } + if phase == "stage_a": + result["calibration_binding"] = _normalize_calibration_binding(calibration_binding) + expected_revisions = dict(resolver.FROZEN_DATASET_REVISIONS) + resolver.build_candidate( + result, + expected_revisions=expected_revisions, + calibration_binding_artifact=calibration_binding, + ) + return result, {} if token_sink is None else token_sink + finally: + if sys.modules.get(_CALIBRATION_RUNNER_MODULE_NAME) is authentication.runner: + sys.modules.pop(_CALIBRATION_RUNNER_MODULE_NAME, None) + + +def capture_identity_input( + *, + phase: str, + source: CaptureSource, + calibration_binding: bytes | None = None, + execution_binding_artifacts: Mapping[str, bytes] | None = None, + runtime_authentication_context: Mapping[str, object] | None = None, +) -> dict[str, Any]: + """Capture one deterministic calibration or Stage-A resolver input.""" + + result, _token_sink = _capture_identity_input_with_tokens( phase=phase, - tokenizer=material.tokenizer, - tokenizer_manifest_sha256=tokenizer_manifest_hash, + source=source, + calibration_binding=calibration_binding, + execution_binding_artifacts=execution_binding_artifacts, + runtime_authentication_context=runtime_authentication_context, + collect_tokens=False, ) - humaneval_records, humaneval_manifest_hash = _capture_humaneval( - source, - phase=phase, - tokenizer=material.tokenizer, - tokenizer_manifest_sha256=tokenizer_manifest_hash, + return result + + +def materialize_calibration_identity_sequences( + *, + source: CaptureSource, + execution_binding_artifacts: Mapping[str, bytes] | None = None, + runtime_authentication_context: Mapping[str, object] | None = None, +) -> CalibrationIdentityMaterialization: + """Materialize exact calibration token IDs through the canonical capture path. + + This read-only API is the sole supported bridge for a future live adapter. + It returns no dataset text, formatted prompts, tokenizer object, or RULER + receipt body. Every public identity record is a fresh copy reconstructed + from canonical bytes and is addressable by its frozen record digest. + """ + + result, token_sink = _capture_identity_input_with_tokens( + phase="calibration", + source=source, + execution_binding_artifacts=execution_binding_artifacts, + runtime_authentication_context=runtime_authentication_context, + collect_tokens=True, ) - after = _validate_heads(source.source_heads(), context="post-capture") - if after != before: # Defensive even though both were pinned. - raise ValueError("source HEAD changed during capture") - - result: dict[str, Any] = { - "schema": resolver.INPUT_SCHEMA, - "phase": phase, - "datasets": _dataset_contracts( - mbpp_manifest_hash=mbpp_manifest_hash, - pg19_manifest_hash=pg19_manifest_hash, - pg19_split=pg19_split, - ruler_manifest_hash=ruler_manifest_hash, - ruler_formatter_hash=ruler_formatter_hash, - humaneval_manifest_hash=humaneval_manifest_hash, - ), - "tokenizer": tokenizer_contract, - "records": [ - *mbpp_records, - *pg19_records, - *ruler_records, - *humaneval_records, - ], - "model_weights_loaded": False, - } - if phase == "stage_a": - result["calibration_binding"] = _normalize_calibration_binding(calibration_binding) - expected_revisions = dict(resolver.FROZEN_DATASET_REVISIONS) - resolver.build_candidate( - result, - expected_revisions=expected_revisions, - calibration_binding_artifact=calibration_binding, + sequences: list[MaterializedCalibrationSequence] = [] + remaining = dict(token_sink) + for record in result["records"]: + key = (str(record["family"]), str(record["canonical_id"])) + try: + prompt_ids, target_ids = remaining.pop(key) + except KeyError as error: + raise RuntimeError(f"missing materialized tokens for {key[0]}/{key[1]}") from error + sequences.append( + MaterializedCalibrationSequence( + _identity_record_bytes=canonical_json_bytes(record), + prompt_token_ids=prompt_ids, + target_token_ids=target_ids, + ) + ) + if remaining: + raise RuntimeError("materialized token sink contains identities absent from capture") + tokenizer_manifest_sha256 = sha256_bytes(canonical_json_bytes(result["tokenizer"]["files"])) + return CalibrationIdentityMaterialization( + sequences=tuple(sequences), + tokenizer_manifest_sha256=tokenizer_manifest_sha256, + capture_input_sha256=sha256_bytes(canonical_json_bytes(result)), ) - return result def atomic_write_no_overwrite(path: Path, payload: bytes) -> None: @@ -1257,33 +2831,24 @@ def __init__(self, *, cache_dir: Path, ruler_receipt_dir: Path) -> None: self.ruler_receipt_dir = ruler_receipt_dir.resolve() @staticmethod - def _github_head(repo_id: str) -> str: - repo_request = urllib.request.Request( - f"https://api.github.com/repos/{repo_id}", + def _github_revision(repo_id: str, revision: str) -> str: + commit_request = urllib.request.Request( + f"https://api.github.com/repos/{repo_id}/commits/" + f"{urllib.parse.quote(revision, safe='')}", headers={ "Accept": "application/vnd.github+json", "User-Agent": "RecurQuant-Experiment-013-identity-capture", }, ) try: - with urllib.request.urlopen(repo_request, timeout=30) as response: - metadata = json.load(response) - branch = _require_string( - metadata.get("default_branch"), context=f"{repo_id} default branch" - ) - commit_request = urllib.request.Request( - f"https://api.github.com/repos/{repo_id}/commits/" - f"{urllib.parse.quote(branch, safe='')}", - headers={ - "Accept": "application/vnd.github+json", - "User-Agent": "RecurQuant-Experiment-013-identity-capture", - }, - ) with urllib.request.urlopen(commit_request, timeout=30) as response: commit = json.load(response) except (OSError, urllib.error.HTTPError, json.JSONDecodeError) as error: - raise RuntimeError(f"cannot resolve GitHub HEAD for {repo_id}") from error - return _require_string(commit.get("sha"), context=f"{repo_id} HEAD") + raise RuntimeError(f"cannot resolve pinned GitHub revision for {repo_id}") from error + resolved = _require_string(commit.get("sha"), context=f"{repo_id} pinned revision") + if resolved != revision: + raise ValueError(f"GitHub returned a different object for pinned {repo_id} revision") + return resolved def source_heads(self) -> Mapping[str, str]: try: @@ -1292,12 +2857,27 @@ def source_heads(self) -> Mapping[str, str]: raise RuntimeError("live capture requires huggingface-hub") from error api = HfApi() return { - "primary_model": str(api.model_info(resolver.PRIMARY_MODEL_ID).sha), - "mbpp": str(api.dataset_info(resolver.MBPP_DATASET_ID).sha), - "pg19": str(api.dataset_info(resolver.PG19_DATASET_ID).sha), - "ruler": self._github_head(resolver.RULER_SOURCE_ID), - "humaneval_plus": str(api.dataset_info(resolver.HUMANEVAL_PLUS_DATASET_ID).sha), - "evalplus": self._github_head(resolver.EVALPLUS_SOURCE_ID), + "primary_model": str( + api.model_info( + resolver.PRIMARY_MODEL_ID, revision=resolver.PRIMARY_MODEL_REVISION + ).sha + ), + "mbpp": str( + api.dataset_info(resolver.MBPP_DATASET_ID, revision=resolver.MBPP_REVISION).sha + ), + "pg19": str( + api.dataset_info(resolver.PG19_DATASET_ID, revision=resolver.PG19_REVISION).sha + ), + "ruler": self._github_revision(resolver.RULER_SOURCE_ID, resolver.RULER_REVISION), + "humaneval_plus": str( + api.dataset_info( + resolver.HUMANEVAL_PLUS_DATASET_ID, + revision=resolver.HUMANEVAL_PLUS_REVISION, + ).sha + ), + "evalplus": self._github_revision( + resolver.EVALPLUS_SOURCE_ID, resolver.EVALPLUS_SOURCE_REVISION + ), } def tokenizer_material(self) -> TokenizerMaterial: @@ -1370,173 +2950,36 @@ def mbpp_train_rows(self) -> Sequence[Mapping[str, Any]]: ) return tuple(dict(row) for row in rows) - @staticmethod - def _active_parquet_aliases( - *, dataset_id: str, revision: str, config: str, split: str - ) -> tuple[str, ...]: - """Resolve only Dataset Viewer's active parquet conversion at *revision*. - - Dataset repositories can retain stale or sibling parquet files that are - not members of the active builder split. The `/parquet` manifest is the - authority for the active conversion; its `x-revision` must bind it to the - pinned source commit before any alias is opened. - """ - - query = urllib.parse.urlencode({"dataset": dataset_id}) - request = urllib.request.Request( - f"https://datasets-server.huggingface.co/parquet?{query}", - headers={"User-Agent": "RecurQuant-Experiment-013-identity-capture"}, + def pg19_projection(self, split: str) -> Sequence[ProjectionRow]: + from recurquant import experiment013_parquet + + expected_count = 13_684 if split == "train" else 50 + projection = experiment013_parquet.project_experiment013_parquet_columns( + "pg19", + split, + columns=("url",), + expected_count=expected_count, ) - try: - with urllib.request.urlopen(request, timeout=60) as response: - source_revision = response.headers.get("x-revision") - payload = json.load(response) - except (OSError, urllib.error.HTTPError, json.JSONDecodeError) as error: - raise RuntimeError( - f"cannot resolve active parquet manifest for {dataset_id}" - ) from error - if source_revision != revision: - raise ValueError("Dataset Viewer parquet x-revision is not pinned") - raw_entries = payload.get("parquet_files") - if not isinstance(raw_entries, list): - raise ValueError("Dataset Viewer parquet manifest is malformed") - aliases: list[str] = [] - names: set[str] = set() - for raw in raw_entries: - if not isinstance(raw, Mapping): - raise ValueError("Dataset Viewer parquet entry is malformed") - if ( - raw.get("dataset") != dataset_id - or raw.get("config") != config - or raw.get("split") != split - ): - continue - filename = _require_string(raw.get("filename"), context="active parquet filename") - if Path(filename).name != filename or filename in names: - raise ValueError("active parquet filenames must be unique basenames") - _require_int(raw.get("size"), context="active parquet size", minimum=1) - url = _require_string(raw.get("url"), context="active parquet URL") - expected_prefix = ( - f"https://huggingface.co/datasets/{dataset_id}/resolve/refs%2Fconvert%2Fparquet/" + return tuple( + ProjectionRow( + _require_string(row.values[0], context=f"PG19 {split} url"), + row.global_offset, ) - if not url.startswith(expected_prefix): - raise ValueError("active parquet URL does not match its alias identity") - relative = url.removeprefix(expected_prefix) - parts = relative.split("/") - if ( - len(parts) != 3 - or parts[0] != config - or parts[1] not in {split, f"partial-{split}"} - or parts[2] != filename - ): - raise ValueError("active parquet URL does not match its alias identity") - names.add(filename) - aliases.append(f"datasets/{dataset_id}@~parquet/{relative}") - if not aliases: - raise ValueError(f"no active {dataset_id} {config}/{split} parquet files") - return tuple(sorted(aliases)) - - @staticmethod - def _parquet_projection( - *, dataset_id: str, revision: str, config: str, split: str, column: str - ) -> tuple[ProjectionRow, ...]: - try: - import pyarrow.parquet as pq - from huggingface_hub import HfFileSystem - except ModuleNotFoundError as error: # pragma: no cover - dependency guard - raise RuntimeError("ID-only projection requires pyarrow and huggingface-hub") from error - aliases = LiveCaptureSource._active_parquet_aliases( - dataset_id=dataset_id, - revision=revision, - config=config, - split=split, - ) - - def read_column(alias: str) -> list[Any]: - fs = HfFileSystem() - with fs.open(alias, "rb") as handle: - table = pq.ParquetFile(handle).read(columns=[column]) - return table.column(column).to_pylist() - - worker_count = min(4, len(aliases)) - with concurrent.futures.ThreadPoolExecutor(max_workers=worker_count) as executor: - shards = tuple(executor.map(read_column, aliases)) - result: list[ProjectionRow] = [] - offset = 0 - for values in shards: - for value in values: - result.append( - ProjectionRow( - _require_string(value, context=f"{dataset_id} {column}"), - offset, - ) - ) - offset += 1 - return tuple(result) - - @staticmethod - def _viewer_row( - *, - dataset_id: str, - config: str, - split: str, - offset: int, - expected_revision: str, - ) -> Mapping[str, Any]: - query = urllib.parse.urlencode( - { - "dataset": dataset_id, - "config": config, - "split": split, - "offset": offset, - "length": 1, - } - ) - request = urllib.request.Request( - f"https://datasets-server.huggingface.co/rows?{query}", - headers={"User-Agent": "RecurQuant-Experiment-013-identity-capture"}, - ) - try: - with urllib.request.urlopen(request, timeout=60) as response: - revision = response.headers.get("x-revision") - payload = json.load(response) - except (OSError, urllib.error.HTTPError, json.JSONDecodeError) as error: - raise RuntimeError(f"Dataset Viewer row request failed for {dataset_id}") from error - if revision != expected_revision: - raise ValueError("Dataset Viewer x-revision does not match the pinned commit") - rows = payload.get("rows") - if not isinstance(rows, list) or len(rows) != 1: - raise ValueError("Dataset Viewer did not return exactly one row") - item = rows[0] - if not isinstance(item, Mapping) or item.get("row_idx") != offset: - raise ValueError("Dataset Viewer row offset drifted") - truncated = item.get("truncated_cells") - if truncated not in (None, []): - raise ValueError("Dataset Viewer truncated a selected source row") - row = item.get("row") - if not isinstance(row, Mapping): - raise ValueError("Dataset Viewer selected row is malformed") - return dict(row) - - def pg19_projection(self, split: str) -> Sequence[ProjectionRow]: - return self._parquet_projection( - dataset_id=resolver.PG19_DATASET_ID, - revision=resolver.PG19_REVISION, - config="default", - split=split, - column="url", + for row in projection.rows ) def pg19_row(self, split: str, *, offset: int, expected_url: str) -> Mapping[str, Any]: - row = self._viewer_row( - dataset_id=resolver.PG19_DATASET_ID, - config="default", - split=split, - offset=offset, - expected_revision=resolver.PG19_REVISION, + from recurquant import experiment013_parquet + + selected = experiment013_parquet.read_experiment013_parquet_row( + "pg19", + split, + offset, + columns=("url", "text"), ) + row = dict(selected.values) if row.get("url") != expected_url: - raise ValueError("Dataset Viewer PG19 URL does not match the pinned projection") + raise ValueError("immutable PG19 row URL does not match the pinned projection") return row def ruler_generator_files(self) -> Mapping[str, bytes]: @@ -1566,9 +3009,15 @@ def _receipt_filename(*, category: str, config: str, configured_length: int, see seed=seed, ) - def ruler_receipt( + def ruler_generation_manifest_bytes(self) -> bytes: + path = self.ruler_receipt_dir / "generation-manifest.json" + if not path.is_file(): + raise FileNotFoundError(f"missing complete RULER generation manifest: {path}") + return path.read_bytes() + + def ruler_receipt_bytes( self, *, category: str, config: str, configured_length: int, seed: int - ) -> Mapping[str, Any]: + ) -> bytes: path = self.ruler_receipt_dir / self._receipt_filename( category=category, config=config, @@ -1579,30 +3028,97 @@ def ruler_receipt( raise FileNotFoundError( f"missing audited RULER receipt; generation is intentionally separate: {path}" ) - return _strict_json(path.read_bytes(), context=f"RULER receipt {path.name}") + return path.read_bytes() + + def ruler_receipt( + self, *, category: str, config: str, configured_length: int, seed: int + ) -> Mapping[str, Any]: + """Compatibility helper for callers inspecting one receipt directly.""" + + raw = self.ruler_receipt_bytes( + category=category, + config=config, + configured_length=configured_length, + seed=seed, + ) + return _strict_json(raw, context="RULER receipt") def humaneval_projection(self) -> Sequence[ProjectionRow]: - return self._parquet_projection( - dataset_id=resolver.HUMANEVAL_PLUS_DATASET_ID, - revision=resolver.HUMANEVAL_PLUS_REVISION, - config="default", - split="test", - column="task_id", + from recurquant import experiment013_parquet + + projection = experiment013_parquet.project_experiment013_parquet_columns( + "humaneval_plus", + "test", + columns=("task_id",), + expected_count=164, + ) + return tuple( + ProjectionRow( + _require_string(row.values[0], context="HumanEval+ task_id"), + row.global_offset, + ) + for row in projection.rows ) def humaneval_row(self, *, offset: int, expected_task_id: str) -> Mapping[str, Any]: - row = self._viewer_row( - dataset_id=resolver.HUMANEVAL_PLUS_DATASET_ID, - config="default", - split="test", - offset=offset, - expected_revision=resolver.HUMANEVAL_PLUS_REVISION, + from recurquant import experiment013_parquet + + selected = experiment013_parquet.read_experiment013_parquet_row( + "humaneval_plus", + "test", + offset, + columns=("task_id", "prompt", "canonical_solution"), ) + row = dict(selected.values) if row.get("task_id") != expected_task_id: - raise ValueError("Dataset Viewer HumanEval+ ID does not match projection") + raise ValueError("immutable HumanEval+ task ID does not match projection") return row +def _parse_named_cli_values( + values: Sequence[str], + *, + context: str, + paths: bool, +) -> dict[str, object]: + parsed: dict[str, object] = {} + for raw in values: + if not isinstance(raw, str) or "=" not in raw: + raise ValueError(f"{context} must use NAME=VALUE") + name, rendered = raw.split("=", 1) + if _RUNTIME_ROOT_NAME_RE.fullmatch(name) is None or not rendered: + raise ValueError(f"{context} contains a non-canonical name or empty value") + if name in parsed: + raise ValueError(f"{context} contains a duplicate name") + parsed[name] = Path(rendered) if paths else rendered + if not parsed: + raise ValueError(f"at least one {context} is required") + return parsed + + +def _runtime_context_from_cli(args: argparse.Namespace) -> _RuntimeAuthenticationContext: + if args.base_runtime_root is None or args.staged_interpreter is None: + raise ValueError("capture requires --base-runtime-root and --staged-interpreter") + roots = _parse_named_cli_values( + args.package_root, + context="--package-root", + paths=True, + ) + import_paths = _parse_named_cli_values( + args.package_import_path, + context="--package-import-path", + paths=False, + ) + return _normalize_runtime_authentication_context( + { + "base_runtime_root": args.base_runtime_root, + "staged_interpreter": args.staged_interpreter, + "package_runtime_roots": roots, + "package_import_paths": import_paths, + } + ) + + def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser( description=( @@ -1620,6 +3136,14 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.add_argument("--cache-dir", type=Path, default=Path(".cache/exp013-identity")) parser.add_argument("--ruler-receipt-dir", type=Path) parser.add_argument("--calibration-binding", type=Path) + parser.add_argument("--repository-source-manifest", type=Path) + parser.add_argument("--calibration-runtime-manifest", type=Path) + parser.add_argument("--model-file-manifest", type=Path) + parser.add_argument("--parquet-materialization-manifest", type=Path) + parser.add_argument("--base-runtime-root", type=Path) + parser.add_argument("--staged-interpreter", type=Path) + parser.add_argument("--package-root", action="append", default=[]) + parser.add_argument("--package-import-path", action="append", default=[]) return parser.parse_args(argv) @@ -1627,7 +3151,7 @@ def main(argv: Sequence[str] | None = None) -> int: args = parse_args(argv) if args.phase in resolver.PROTECTED_STAGES: raise PermissionError( - f"{args.phase} is protected; capture v2 refuses it before file or source access" + f"{args.phase} is protected; capture v4 refuses it before file or source access" ) if args.ruler_receipt_dir is None: raise ValueError("--ruler-receipt-dir is required") @@ -1635,6 +3159,7 @@ def main(argv: Sequence[str] | None = None) -> int: raise ValueError("--dry-run forbids --output") if not args.dry_run and args.output is None: raise ValueError("capture requires --output or --dry-run") + runtime_context = _runtime_context_from_cli(args) binding: bytes | None = None if args.phase == "stage_a": if args.calibration_binding is None: @@ -1642,6 +3167,23 @@ def main(argv: Sequence[str] | None = None) -> int: binding = args.calibration_binding.read_bytes() elif args.calibration_binding is not None: raise ValueError("--calibration-binding is valid only for Stage A") + binding_paths = { + "repository_source_manifest_file_sha256": args.repository_source_manifest, + "calibration_runtime_manifest_file_sha256": args.calibration_runtime_manifest, + "model_file_manifest_file_sha256": args.model_file_manifest, + "parquet_materialization_manifest_file_sha256": ( + args.parquet_materialization_manifest + ), + } + if any(path is None for path in binding_paths.values()): + raise ValueError( + "capture requires --repository-source-manifest, " + "--calibration-runtime-manifest, --model-file-manifest, and " + "--parquet-materialization-manifest" + ) + execution_binding_artifacts = { + field: path.read_bytes() for field, path in binding_paths.items() if path is not None + } source = LiveCaptureSource( cache_dir=args.cache_dir, ruler_receipt_dir=args.ruler_receipt_dir, @@ -1650,6 +3192,13 @@ def main(argv: Sequence[str] | None = None) -> int: phase=args.phase, source=source, calibration_binding=binding, + execution_binding_artifacts=execution_binding_artifacts, + runtime_authentication_context={ + "base_runtime_root": runtime_context.base_runtime_root, + "staged_interpreter": runtime_context.staged_interpreter, + "package_runtime_roots": runtime_context.package_runtime_roots, + "package_import_paths": runtime_context.package_import_paths, + }, ) payload = canonical_json_bytes(captured) digest = sha256_bytes(payload) diff --git a/scripts/generate_static_q468_ruler_receipts.py b/scripts/generate_static_q468_ruler_receipts.py index 1aca275..b84b3ba 100644 --- a/scripts/generate_static_q468_ruler_receipts.py +++ b/scripts/generate_static_q468_ruler_receipts.py @@ -13,11 +13,14 @@ import argparse import ast +import base64 import hashlib +import importlib.metadata import importlib.util import json import os import re +import shutil import subprocess import sys import tempfile @@ -29,8 +32,12 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[1] CAPTURE_PATH = REPOSITORY_ROOT / "scripts" / "capture_static_q468_identity_input.py" -LAUNCHER_REVISION: Final = "experiment-013-ruler-argv-launcher-v2" +LAUNCHER_REVISION: Final = "experiment-013-ruler-argv-launcher-v6" +GENERATION_MANIFEST_SCHEMA: Final = "recurquant.experiment013.ruler-generation-manifest.v2" +RUNTIME_MANIFEST_SCHEMA: Final = "recurquant.experiment013.ruler-runtime-manifest.v3" RUNTIME_PYTHON_VERSION: Final = "3.11.15" +RUNTIME_PROBE_TIMEOUT_SECONDS: Final = 300 +TOKENIZER_TIMEOUT_SECONDS: Final = 180 RUNTIME_PACKAGES: Final = { "PyYAML": "6.0.3", "annotated-doc": "0.0.5", @@ -70,6 +77,127 @@ "typing-extensions": "4.16.0", "wonderwords": "3.0.1", } +RUNTIME_IMPORTABLE_SUFFIXES: Final = frozenset({".dll", ".pyd", ".py", ".pyw", ".pth"}) +EXPECTED_EXCLUDED_VIRTUALENV_STARTUP_FILES: Final = { + "_virtualenv.pth": ( + 18, + "69ac3d8f27e679c81b94ab30b3b56e9cd138219b1ba94a1fa3606d5a76a1433d", + ), + "_virtualenv.py": ( + 5_246, + "cfb3db86aaa53bb62b5ff764970bec2d71c9228590a0ebec57f6ec926cc0bf1a", + ), +} +SEALED_STARTUP_POLICY: Final = { + "dont_write_bytecode": 1, + "no_site": 1, + "package_path_mode": "staged-record-only-site-packages-v1", + "pycache_mode": "verified-empty-prefix-no-write-v1", + "site_loaded": False, + "utf8_mode": 1, + "virtualenv_hook_loaded": False, +} +SEALED_STARTUP_BOOTSTRAP: Final = r""" +import importlib.metadata as _rq_metadata +import pathlib as _rq_pathlib +import re as _rq_re +import sys as _rq_sys + +_rq_package_root_raw = _rq_pathlib.Path(_rq_sys.argv[1]) +_rq_pycache_root_raw = _rq_pathlib.Path(_rq_sys.argv[2]) +if not _rq_package_root_raw.is_absolute() or not _rq_pycache_root_raw.is_absolute(): + raise RuntimeError("sealed RULER startup paths must be absolute") +_rq_package_root = _rq_package_root_raw.resolve() +_rq_pycache_root = _rq_pycache_root_raw.resolve() +_rq_reparse = lambda path: path.is_symlink() or bool( + getattr(path.stat(), "st_file_attributes", 0) & 0x400 +) +if ( + not _rq_package_root.is_dir() + or not _rq_pycache_root.is_dir() + or _rq_reparse(_rq_package_root) + or _rq_reparse(_rq_pycache_root) + or any(_rq_pycache_root.iterdir()) +): + raise RuntimeError("sealed RULER startup paths are missing, redirected, or non-empty") +_rq_flags = { + "ignore_environment": _rq_sys.flags.ignore_environment, + "isolated": _rq_sys.flags.isolated, + "no_user_site": _rq_sys.flags.no_user_site, +} +if _rq_flags != {"ignore_environment": 1, "isolated": 1, "no_user_site": 1}: + raise RuntimeError("sealed RULER isolation flags drifted") +if ( + _rq_sys.flags.no_site != 1 + or _rq_sys.flags.dont_write_bytecode != 1 + or _rq_sys.flags.utf8_mode != 1 + or _rq_sys.pycache_prefix is None + or _rq_pathlib.Path(_rq_sys.pycache_prefix).resolve() != _rq_pycache_root + or "site" in _rq_sys.modules + or "_virtualenv" in _rq_sys.modules +): + raise RuntimeError("sealed RULER startup policy drifted") +_rq_canonical = lambda name: _rq_re.sub(r"[-_.]+", "-", name).lower() +_rq_expected_packages = __PACKAGE_VERSIONS__ +_rq_distributions = list(_rq_metadata.distributions(path=[str(_rq_package_root)])) +_rq_by_name = {} +for _rq_dist in _rq_distributions: + _rq_name = _rq_canonical(_rq_dist.metadata["Name"]) + if _rq_name in _rq_by_name: + raise RuntimeError("sealed RULER package root contains a duplicate distribution") + _rq_by_name[_rq_name] = _rq_dist +if { + name: dist.version for name, dist in _rq_by_name.items() +} != {_rq_canonical(name): version for name, version in _rq_expected_packages.items()}: + raise RuntimeError("sealed RULER package root distribution set drifted") +_rq_recorded_paths = set() +for _rq_dist in _rq_distributions: + _rq_files = list(_rq_dist.files or ()) + if not _rq_files: + raise RuntimeError("sealed RULER distribution has no RECORD inventory") + for _rq_item in _rq_files: + _rq_path = _rq_pathlib.Path(_rq_dist.locate_file(_rq_item)) + if not _rq_path.is_file() or _rq_reparse(_rq_path): + raise RuntimeError("sealed RULER RECORD path is missing or redirected") + _rq_recorded_paths.add(_rq_path.resolve()) +for _rq_path in _rq_package_root.rglob("*"): + if _rq_path.is_symlink() or ( + _rq_path.exists() and bool(getattr(_rq_path.stat(), "st_file_attributes", 0) & 0x400) + ): + raise RuntimeError("sealed RULER package root contains a redirected path") + if ( + _rq_path.is_file() + and _rq_path.suffix.lower() in {".dll", ".pyd", ".py", ".pyw", ".pth"} + and _rq_path.resolve() not in _rq_recorded_paths + ): + raise RuntimeError("sealed RULER package root contains unrecorded importable code") +_recurquant_startup_flags = _rq_flags +_recurquant_startup_policy = { + "dont_write_bytecode": _rq_sys.flags.dont_write_bytecode, + "no_site": _rq_sys.flags.no_site, + "package_path_mode": "staged-record-only-site-packages-v1", + "pycache_mode": "verified-empty-prefix-no-write-v1", + "site_loaded": "site" in _rq_sys.modules, + "utf8_mode": _rq_sys.flags.utf8_mode, + "virtualenv_hook_loaded": "_virtualenv" in _rq_sys.modules, +} +_rq_sys.path.insert(0, str(_rq_package_root)) +""".strip().replace("__PACKAGE_VERSIONS__", repr(RUNTIME_PACKAGES)) +ISOLATED_SOURCE_BOOTSTRAP: Final = ( + SEALED_STARTUP_BOOTSTRAP + + "\n" + + r""" +import runpy as _rq_runpy + +_rq_root = _rq_pathlib.Path(_rq_sys.argv[3]).resolve() +_rq_relative = _rq_pathlib.PurePosixPath(_rq_sys.argv[4]) +_rq_script = (_rq_root / _rq_relative).resolve() +_rq_script.relative_to(_rq_root) +_rq_sys.path[:0] = [str(_rq_script.parent), str(_rq_root)] +_rq_sys.argv = [str(_rq_script), *_rq_sys.argv[5:]] +_rq_runpy.run_path(str(_rq_script), run_name="__main__") +""".strip() +) EXPECTED_CORPORA: Final = { "PaulGrahamEssays.json": ( @@ -90,6 +218,10 @@ ), } EXPECTED_PACKAGE_RESOURCES: Final = { + "nltk/punkt/english.pickle": ( + 433_305, + "dda37972ae88998a6fd3e3ec002697a6bd362b32d050fda7d7ca5276873092aa", + ), "nltk/punkt/PY3/english.pickle": ( 406_697, "5cad3758596392364e3be9803dbd7ebeda384b68937b488a01365f5551bb942c", @@ -175,6 +307,34 @@ class TaskSpec: niah: bool = False +@dataclass(frozen=True, slots=True) +class VerifiedRulerCheckout: + source_manifest: tuple[dict[str, object], ...] + source_files: Mapping[str, bytes] + + +@dataclass(frozen=True, slots=True) +class VerifiedStaticInputs: + entries: tuple[dict[str, object], ...] + corpus_files: Mapping[str, bytes] + sealed_runtime_files: Mapping[str, bytes] + + +@dataclass(frozen=True, slots=True) +class VerifiedRuntimePackageTree: + entries: tuple[dict[str, object], ...] + source_files: Mapping[str, Path] + excluded_startup_files: tuple[dict[str, object], ...] + + +@dataclass(frozen=True, slots=True) +class VerifiedPythonRuntime: + entries: tuple[dict[str, object], ...] + source_files: Mapping[str, Path] + source_launcher: dict[str, object] + source_pyvenv_config: dict[str, object] + + TASK_SPECS: Final = { "niah_multiquery": TaskSpec( "niah.py", @@ -370,6 +530,14 @@ def _canonical_distribution_name(name: str) -> str: return re.sub(r"[-_.]+", "-", name).lower() +def _is_exact_one_flag_mapping(value: object, fields: frozenset[str]) -> bool: + return ( + isinstance(value, Mapping) + and set(value) == set(fields) + and all(type(value[field]) is int and value[field] == 1 for field in fields) + ) + + def _subprocess_env(**updates: str) -> dict[str, str]: """Return a child environment without caller-controlled Python injection.""" @@ -379,12 +547,301 @@ def _subprocess_env(**updates: str) -> dict[str, str]: return env +def _git_env() -> dict[str, str]: + """Return a Git environment without caller or machine configuration.""" + + env = {key: value for key, value in os.environ.items() if not key.upper().startswith("GIT_")} + env.update( + { + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_NO_REPLACE_OBJECTS": "1", + "LC_ALL": "C", + "LANG": "C", + } + ) + return env + + def _file_entry(name: str, data: bytes) -> dict[str, object]: if not data: raise ValueError(f"bound file {name!r} is empty") return {"name": name, "sha256": _sha256_bytes(data), "size_bytes": len(data)} +def _tree_file_entry(name: str, data: bytes) -> dict[str, object]: + return {"name": name, "sha256": _sha256_bytes(data), "size_bytes": len(data)} + + +def _is_reparse_point(path: Path) -> bool: + return path.is_symlink() or bool(getattr(path.stat(), "st_file_attributes", 0) & 0x400) + + +def _runtime_layout(python: Path) -> tuple[Path, Path, Path]: + unresolved_python = Path(os.path.abspath(python)) + if not unresolved_python.is_file() or _is_reparse_point(unresolved_python): + raise ValueError("RULER Python executable is missing or redirected") + resolved_python = unresolved_python.resolve() + if ( + resolved_python.name.casefold() != "python.exe" + or resolved_python.parent.name.casefold() != "scripts" + ): + raise ValueError("RULER Python executable is not in a Windows virtual environment") + runtime_root = resolved_python.parent.parent + package_root = runtime_root / "Lib" / "site-packages" + for path in (runtime_root, runtime_root / "Lib", package_root): + if not path.is_dir() or _is_reparse_point(path): + raise ValueError("RULER virtual-environment package path is missing or redirected") + return resolved_python, runtime_root.resolve(), package_root.resolve() + + +def verify_python_runtime_source(python: Path) -> VerifiedPythonRuntime: + source_python, venv_root, _package_root = _runtime_layout(python) + pyvenv_path = venv_root / "pyvenv.cfg" + if not pyvenv_path.is_file() or _is_reparse_point(pyvenv_path): + raise ValueError("RULER Python virtual environment lacks a bound pyvenv.cfg") + pyvenv_data = pyvenv_path.read_bytes() + try: + config_lines = pyvenv_data.decode("utf-8").splitlines() + except UnicodeDecodeError as error: + raise ValueError("RULER pyvenv.cfg must be UTF-8") from error + config: dict[str, str] = {} + for line in config_lines: + if not line.strip(): + continue + if line.count("=") != 1: + raise ValueError("RULER pyvenv.cfg is malformed") + key, value = (part.strip() for part in line.split("=", 1)) + if not key or key in config: + raise ValueError("RULER pyvenv.cfg contains a duplicate or empty key") + config[key] = value + if config.get("version_info") != "3.11" or config.get("include-system-site-packages") != ( + "false" + ): + raise ValueError("RULER pyvenv.cfg version or site-package policy drifted") + raw_home = config.get("home") + if not raw_home: + raise ValueError("RULER pyvenv.cfg omitted its base runtime") + base_root = Path(raw_home) + if not base_root.is_absolute() or not base_root.is_dir() or _is_reparse_point(base_root): + raise ValueError("RULER base Python runtime is missing or redirected") + base_root = base_root.resolve() + + source_files: dict[str, Path] = {} + for root_name in ("Lib", "DLLs"): + root = base_root / root_name + if not root.is_dir() or _is_reparse_point(root): + raise ValueError(f"RULER base Python runtime omitted {root_name}") + for path in root.rglob("*"): + if _is_reparse_point(path): + raise ValueError("RULER base Python runtime contains a redirected path") + if not path.is_file(): + continue + relative = path.relative_to(base_root) + if "site-packages" in {part.casefold() for part in relative.parts}: + continue + if "__pycache__" in relative.parts or path.suffix.casefold() == ".pyc": + continue + source_files[relative.as_posix()] = path.resolve() + root_patterns = ( + re.compile(r"^python.*\.(?:dll|exe|zip)$", flags=re.IGNORECASE), + re.compile(r"^vcruntime.*\.dll$", flags=re.IGNORECASE), + ) + for path in base_root.iterdir(): + if _is_reparse_point(path): + raise ValueError("RULER base Python root contains a redirected path") + if path.is_file() and any(pattern.fullmatch(path.name) for pattern in root_patterns): + source_files[path.name] = path.resolve() + required_names = {"python.exe", "python3.dll", "python311.dll"} + if not required_names <= set(source_files): + raise ValueError("RULER base Python runtime omitted an executable or adjacent DLL") + entries = tuple( + _tree_file_entry(name, source_files[name].read_bytes()) for name in sorted(source_files) + ) + return VerifiedPythonRuntime( + entries=entries, + source_files=dict(source_files), + source_launcher=_tree_file_entry("source/python.exe", source_python.read_bytes()), + source_pyvenv_config=_tree_file_entry("source/pyvenv.cfg", pyvenv_data), + ) + + +def verify_staged_python_runtime(root: Path, *, expected: Sequence[Mapping[str, object]]) -> None: + root = root.resolve() + observed: dict[str, Path] = {} + for path in root.rglob("*"): + if _is_reparse_point(path): + raise ValueError("staged RULER Python runtime contains a redirected path") + if path.is_file(): + observed[path.relative_to(root).as_posix()] = path + expected_by_name = {str(item["name"]): item for item in expected} + if set(observed) != set(expected_by_name): + raise ValueError("staged RULER Python runtime inventory drifted") + for name, path in observed.items(): + if _tree_file_entry(name, path.read_bytes()) != expected_by_name[name]: + raise ValueError(f"staged RULER Python runtime bytes drifted: {name}") + + +def stage_verified_python_runtime(root: Path, *, verified: VerifiedPythonRuntime) -> Path: + if root.exists(): + raise FileExistsError(f"refusing to replace staged RULER Python runtime: {root}") + root.mkdir(parents=True) + for name, source in verified.source_files.items(): + destination = root / Path(name) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + verify_staged_python_runtime(root, expected=verified.entries) + python = root / "python.exe" + if not python.is_file(): + raise ValueError("staged RULER Python runtime omitted python.exe") + return python + + +def verify_runtime_package_source(python: Path) -> VerifiedRuntimePackageTree: + """Freeze a live venv into a RECORD-only, non-executable source inventory.""" + + _python, runtime_root, package_root = _runtime_layout(python) + distributions = list(importlib.metadata.distributions(path=[str(package_root)])) + by_name: dict[str, importlib.metadata.Distribution] = {} + for distribution in distributions: + canonical_name = _canonical_distribution_name(distribution.metadata["Name"]) + if canonical_name in by_name: + raise ValueError("RULER runtime contains a duplicate installed distribution") + by_name[canonical_name] = distribution + expected_versions = { + _canonical_distribution_name(name): version for name, version in RUNTIME_PACKAGES.items() + } + if {name: distribution.version for name, distribution in by_name.items()} != expected_versions: + raise ValueError("RULER runtime installed-distribution inventory drifted") + + entries_by_name: dict[str, dict[str, object]] = {} + source_files: dict[str, Path] = {} + recorded_paths: set[Path] = set() + for canonical_name in sorted(by_name): + distribution = by_name[canonical_name] + files = list(distribution.files or ()) + records = [ + item for item in files if str(item).replace("\\", "/").endswith(".dist-info/RECORD") + ] + if len(records) != 1: + raise ValueError(f"RULER package {canonical_name} must have exactly one RECORD") + for item in files: + unresolved = Path(distribution.locate_file(item)) + if not unresolved.is_file() or _is_reparse_point(unresolved): + raise ValueError(f"RULER package {canonical_name} has a missing or redirected file") + resolved = unresolved.resolve() + try: + relative = resolved.relative_to(runtime_root).as_posix() + except ValueError as error: + raise ValueError( + f"RULER package {canonical_name} RECORD escapes the virtual environment" + ) from error + data = resolved.read_bytes() + entry = _tree_file_entry(relative, data) + previous = entries_by_name.get(relative) + if previous is not None and previous != entry: + raise ValueError("RULER distributions disagree about a shared installed file") + entries_by_name[relative] = entry + source_files[relative] = resolved + recorded_paths.add(resolved) + + excluded: dict[str, dict[str, object]] = {} + for path in package_root.rglob("*"): + if _is_reparse_point(path): + raise ValueError("RULER live package tree contains a redirected path") + if not path.is_file() or path.suffix.lower() not in RUNTIME_IMPORTABLE_SUFFIXES: + continue + resolved = path.resolve() + if resolved in recorded_paths: + continue + relative = path.relative_to(package_root).as_posix() + try: + expected_size, expected_sha256 = EXPECTED_EXCLUDED_VIRTUALENV_STARTUP_FILES[relative] + except KeyError as error: + raise ValueError( + f"RULER live package tree contains unrecorded importable code: {relative}" + ) from error + data = path.read_bytes() + entry = _tree_file_entry(relative, data) + if entry["size_bytes"] != expected_size or entry["sha256"] != expected_sha256: + raise ValueError(f"RULER excluded startup file drifted: {relative}") + excluded[relative] = entry + if set(excluded) != set(EXPECTED_EXCLUDED_VIRTUALENV_STARTUP_FILES): + raise ValueError("RULER excluded virtualenv startup-file inventory drifted") + entries = tuple(entries_by_name[name] for name in sorted(entries_by_name)) + return VerifiedRuntimePackageTree( + entries=entries, + source_files=dict(source_files), + excluded_startup_files=tuple(excluded[name] for name in sorted(excluded)), + ) + + +def verify_staged_runtime_package_tree( + runtime_root: Path, *, expected: Sequence[Mapping[str, object]] +) -> None: + runtime_root = runtime_root.resolve() + observed: dict[str, Path] = {} + for path in runtime_root.rglob("*"): + if _is_reparse_point(path): + raise ValueError("staged RULER runtime package tree contains a redirected path") + if path.is_file(): + observed[path.relative_to(runtime_root).as_posix()] = path + expected_by_name = {str(entry["name"]): entry for entry in expected} + if set(observed) != set(expected_by_name): + raise ValueError("staged RULER runtime package inventory drifted") + for name, path in observed.items(): + if _tree_file_entry(name, path.read_bytes()) != expected_by_name[name]: + raise ValueError(f"staged RULER runtime package bytes drifted: {name}") + + +def stage_verified_runtime_package_tree( + runtime_root: Path, *, verified: VerifiedRuntimePackageTree +) -> Path: + if runtime_root.exists(): + raise FileExistsError(f"refusing to replace staged RULER runtime: {runtime_root}") + runtime_root.mkdir(parents=True) + for name, source in verified.source_files.items(): + destination = runtime_root / Path(name) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + verify_staged_runtime_package_tree(runtime_root, expected=verified.entries) + package_root = runtime_root / "Lib" / "site-packages" + if not package_root.is_dir(): + raise ValueError("staged RULER runtime omitted its site-packages directory") + return package_root + + +def _verify_empty_pycache_prefix(path: Path) -> None: + if not path.is_dir() or _is_reparse_point(path) or any(path.iterdir()): + raise ValueError("sealed RULER pycache prefix is missing, redirected, or non-empty") + + +def _sealed_python_argv( + *, + python: Path, + package_root: Path, + pycache_prefix: Path, + code: str, + arguments: Sequence[str] = (), +) -> list[str]: + return [ + str(python.resolve()), + "-I", + "-S", + "-B", + "-X", + f"pycache_prefix={pycache_prefix.resolve()}", + "-X", + "utf8", + "-c", + code, + str(package_root.resolve()), + str(pycache_prefix.resolve()), + *arguments, + ] + + def _verified_file(path: Path, *, size: int, sha256: str, name: str) -> bytes: if not path.is_file(): raise FileNotFoundError(f"missing required {name}: {path}") @@ -501,44 +958,25 @@ def _launcher_source_entry() -> dict[str, object]: ) -def verify_ruler_checkout(ruler_root: Path, capture: Any) -> list[dict[str, object]]: +def verify_ruler_checkout(ruler_root: Path, capture: Any) -> VerifiedRulerCheckout: ruler_root = ruler_root.resolve() result = subprocess.run( ["git", "-C", str(ruler_root), "rev-parse", "HEAD"], check=True, capture_output=True, text=True, + env=_git_env(), ) head = result.stdout.strip() if head != capture.resolver.RULER_REVISION: raise ValueError("RULER checkout HEAD differs from the frozen revision") files: dict[str, bytes] = {} for relative, expected_blob in capture.RULER_GENERATOR_GIT_BLOBS.items(): - path = ruler_root / relative - if not path.is_file(): - raise FileNotFoundError(f"missing pinned RULER source file: {relative}") - # Git may legitimately smudge LF blobs to CRLF in a Windows checkout. - # Hash the worktree path through its clean filters, then bind the exact - # immutable object bytes used by the capture contract. - worktree_hash = subprocess.run( - [ - "git", - "-C", - str(ruler_root), - "hash-object", - f"--path={relative}", - str(path), - ], - check=True, - capture_output=True, - text=True, - ).stdout.strip() - if worktree_hash != expected_blob: - raise ValueError(f"pinned RULER source file drifted: {relative}") data = subprocess.run( ["git", "-C", str(ruler_root), "cat-file", "blob", expected_blob], check=True, capture_output=True, + env=_git_env(), ).stdout if _git_blob_sha1(data) != expected_blob: raise RuntimeError(f"Git returned corrupt blob bytes for {relative}") @@ -547,43 +985,114 @@ def verify_ruler_checkout(ruler_root: Path, capture: Any) -> list[dict[str, obje synthetic_yaml=files["scripts/synthetic.yaml"], constants_py=files["scripts/data/synthetic/constants.py"], ) - return capture._ruler_generator_manifest(files) + return VerifiedRulerCheckout( + source_manifest=tuple(capture._ruler_generator_manifest(files)), + source_files=dict(files), + ) -def verify_runtime(python: Path, nltk_data: Path) -> tuple[dict[str, object], dict[str, Path]]: +def verify_runtime( + python: Path, + nltk_data: Path, + *, + package_root: Path, + package_tree_manifest: Sequence[Mapping[str, object]], + excluded_startup_files: Sequence[Mapping[str, object]], + python_runtime_root: Path, + python_runtime_manifest: Sequence[Mapping[str, object]], + source_python: Mapping[str, object], + source_pyvenv_config: Mapping[str, object], +) -> tuple[dict[str, object], dict[str, Path]]: code = """ import importlib.metadata as metadata import importlib.util +import hashlib import json import pathlib import platform import re import sys -import nltk -import wonderwords names = __PACKAGE_NAMES__ forbidden = __FORBIDDEN_MODULES__ -root = pathlib.Path(wonderwords.__file__).resolve().parent canonical = lambda name: re.sub(r'[-_.]+', '-', name).lower() +def distribution_inventory(name): + dist = metadata.distribution(name) + items = list(dist.files or ()) + records = [ + item + for item in items + if str(item).replace('\\\\', '/').endswith('.dist-info/RECORD') + ] + if len(records) != 1: + raise RuntimeError(f'{name} must have exactly one installed RECORD inventory') + record_path = pathlib.Path(dist.locate_file(records[0])) + if not record_path.is_file(): + raise RuntimeError(f'{name} installed RECORD is unavailable') + record_bytes = record_path.read_bytes() + if not record_bytes: + raise RuntimeError(f'{name} installed RECORD is empty') + files = [] + seen = set() + for item in items: + relative = str(item).replace('\\\\', '/') + if not relative or relative in seen: + raise RuntimeError(f'{name} has a duplicate or empty installed path') + seen.add(relative) + path = pathlib.Path(dist.locate_file(item)) + if not path.is_file(): + raise RuntimeError(f'{name} installed file is unavailable: {relative}') + data = path.read_bytes() + files.append({ + 'path': relative, + 'sha256': hashlib.sha256(data).hexdigest(), + 'size_bytes': len(data), + }) + if not files: + raise RuntimeError(f'{name} has no installed files') + files.sort(key=lambda item: item['path']) + return { + 'canonical_name': canonical(dist.metadata['Name']), + 'version': dist.version, + 'record_sha256': hashlib.sha256(record_bytes).hexdigest(), + 'record_size_bytes': len(record_bytes), + 'files': files, + } +pre_inventory = {name: distribution_inventory(name) for name in names} +pre_installed = { + canonical(dist.metadata['Name']): dist.version for dist in metadata.distributions() +} +import nltk +import wonderwords +root = pathlib.Path(wonderwords.__file__).resolve().parent +post_inventory = {name: distribution_inventory(name) for name in names} +post_installed = { + canonical(dist.metadata['Name']): dist.version for dist in metadata.distributions() +} +if post_inventory != pre_inventory or post_installed != pre_installed: + raise RuntimeError('RULER package code changed while imports were active') payload = { 'python': sys.version.split()[0], 'implementation': sys.implementation.name, 'cache_tag': sys.implementation.cache_tag, 'executable': str(pathlib.Path(sys.executable).resolve()), 'platform': platform.platform(), + 'machine': platform.machine(), 'flags': { - 'ignore_environment': sys.flags.ignore_environment, - 'isolated': sys.flags.isolated, - 'no_user_site': sys.flags.no_user_site, + **_recurquant_startup_flags, }, + 'startup_policy': _recurquant_startup_policy, 'packages': {name: metadata.version(name) for name in names}, - 'installed_distributions': { - canonical(dist.metadata['Name']): dist.version for dist in metadata.distributions() - }, + 'installed_distributions': pre_installed, + 'distribution_file_inventory': pre_inventory, 'forbidden_modules': { name: importlib.util.find_spec(name) is not None for name in forbidden }, 'resources': { + 'nltk/punkt/english.pickle': next( + str(pathlib.Path(data_root) / 'tokenizers' / 'punkt' / 'english.pickle') + for data_root in nltk.data.path + if (pathlib.Path(data_root) / 'tokenizers' / 'punkt' / 'english.pickle').is_file() + ), 'nltk/punkt/PY3/english.pickle': str(nltk.data.find('tokenizers/punkt/english.pickle')), 'wonderwords/adjectivelist.txt': str(root / 'assets' / 'adjectivelist.txt'), 'wonderwords/nounlist.txt': str(root / 'assets' / 'nounlist.txt'), @@ -601,13 +1110,38 @@ def verify_runtime(python: Path, nltk_data: Path) -> tuple[dict[str, object], di NLTK_DATA=str(nltk_data.resolve()), TOKENIZERS_PARALLELISM="false", ) - result = subprocess.run( - [str(python), "-I", "-c", code], - check=True, - capture_output=True, - text=True, - env=env, - ) + runtime_root = package_root.resolve().parents[1] + verify_staged_runtime_package_tree(runtime_root, expected=package_tree_manifest) + verify_staged_python_runtime(python_runtime_root, expected=python_runtime_manifest) + with tempfile.TemporaryDirectory(prefix="recurquant-exp013-ruler-pycache-") as temporary: + pycache_prefix = Path(temporary) + _verify_empty_pycache_prefix(pycache_prefix) + try: + result = subprocess.run( + _sealed_python_argv( + python=python, + package_root=package_root, + pycache_prefix=pycache_prefix, + code=SEALED_STARTUP_BOOTSTRAP + "\n" + code, + ), + check=False, + capture_output=True, + text=True, + encoding="utf-8", + errors="strict", + env=env, + timeout=RUNTIME_PROBE_TIMEOUT_SECONDS, + ) + finally: + _verify_empty_pycache_prefix(pycache_prefix) + verify_staged_runtime_package_tree(runtime_root, expected=package_tree_manifest) + verify_staged_python_runtime(python_runtime_root, expected=python_runtime_manifest) + if result.returncode != 0: + diagnostic = result.stderr[-4_000:].strip() + raise RuntimeError( + "sealed RULER runtime probe failed" + + (f": {diagnostic}" if diagnostic else " without stderr") + ) payload = _strict_json(result.stdout.encode(), context="RULER runtime probe") if not isinstance(payload, Mapping): raise ValueError("RULER runtime probe did not return an object") @@ -620,18 +1154,93 @@ def verify_runtime(python: Path, nltk_data: Path) -> tuple[dict[str, object], di } if payload.get("installed_distributions") != expected_distributions: raise ValueError("RULER installed-distribution inventory drifted") + file_inventory = payload.get("distribution_file_inventory") + if not isinstance(file_inventory, Mapping) or set(file_inventory) != set(RUNTIME_PACKAGES): + raise ValueError("RULER distribution-file inventory drifted") + for package_name, version in RUNTIME_PACKAGES.items(): + raw_distribution = file_inventory[package_name] + if not isinstance(raw_distribution, Mapping) or set(raw_distribution) != { + "canonical_name", + "version", + "record_sha256", + "record_size_bytes", + "files", + }: + raise ValueError(f"RULER package-code inventory is malformed for {package_name}") + if ( + raw_distribution["canonical_name"] != _canonical_distribution_name(package_name) + or raw_distribution["version"] != version + ): + raise ValueError(f"RULER package-code identity drifted for {package_name}") + record_sha256 = raw_distribution["record_sha256"] + record_size = raw_distribution["record_size_bytes"] + if ( + not isinstance(record_sha256, str) + or re.fullmatch(r"[0-9a-f]{64}", record_sha256) is None + or isinstance(record_size, bool) + or not isinstance(record_size, int) + or record_size < 1 + ): + raise ValueError(f"RULER package RECORD identity drifted for {package_name}") + files_value = raw_distribution["files"] + if isinstance(files_value, (str, bytes)) or not isinstance(files_value, Sequence): + raise ValueError(f"RULER package file list is malformed for {package_name}") + normalized_files: list[dict[str, object]] = [] + for item in files_value: + if not isinstance(item, Mapping) or set(item) != {"path", "sha256", "size_bytes"}: + raise ValueError(f"RULER package file entry is malformed for {package_name}") + path_value = item["path"] + size_value = item["size_bytes"] + digest_value = item["sha256"] + if ( + not isinstance(path_value, str) + or not path_value + or "\\" in path_value + or "\0" in path_value + or "\n" in path_value + or "\r" in path_value + or not isinstance(digest_value, str) + or re.fullmatch(r"[0-9a-f]{64}", digest_value) is None + or isinstance(size_value, bool) + or not isinstance(size_value, int) + or size_value < 0 + ): + raise ValueError(f"RULER package file identity drifted for {package_name}") + normalized_files.append(dict(item)) + if ( + not normalized_files + or normalized_files != sorted(normalized_files, key=lambda item: str(item["path"])) + or len({str(item["path"]) for item in normalized_files}) != len(normalized_files) + ): + raise ValueError(f"RULER package file ordering drifted for {package_name}") expected_absence = {name: False for name in FORBIDDEN_RUNTIME_MODULES} if payload.get("forbidden_modules") != expected_absence: raise ValueError("RULER runtime contains a forbidden model framework") if payload.get("implementation") != "cpython": raise ValueError("RULER Python implementation drifted") flags = payload.get("flags") - if flags != {"ignore_environment": 1, "isolated": 1, "no_user_site": 1}: + if not _is_exact_one_flag_mapping( + flags, frozenset({"ignore_environment", "isolated", "no_user_site"}) + ): raise ValueError("RULER runtime probe was not isolated") + startup_policy = payload.get("startup_policy") + if ( + startup_policy != SEALED_STARTUP_POLICY + or not isinstance(startup_policy, Mapping) + or any( + type(startup_policy[field]) is not int or startup_policy[field] != 1 + for field in ("dont_write_bytecode", "no_site", "utf8_mode") + ) + or any( + type(startup_policy[field]) is not bool or startup_policy[field] is not False + for field in ("site_loaded", "virtualenv_hook_loaded") + ) + ): + raise ValueError("RULER runtime probe was not sealed") executable = payload.get("executable") if not isinstance(executable, str) or not Path(executable).samefile(python): raise ValueError("RULER runtime probe used a different Python executable") - for field in ("cache_tag", "platform"): + for field in ("cache_tag", "platform", "machine"): if not isinstance(payload.get(field), str) or not payload[field]: raise ValueError(f"RULER runtime probe omitted {field}") resources = payload.get("resources") @@ -642,21 +1251,70 @@ def verify_runtime(python: Path, nltk_data: Path) -> tuple[dict[str, object], di if not isinstance(raw_path, str): raise ValueError("RULER runtime returned a non-string resource path") paths[str(name)] = Path(raw_path) + expected_excluded = [ + {"name": name, "sha256": digest, "size_bytes": size} + for name, (size, digest) in sorted(EXPECTED_EXCLUDED_VIRTUALENV_STARTUP_FILES.items()) + ] + if [dict(item) for item in excluded_startup_files] != expected_excluded: + raise ValueError("RULER excluded startup-file binding drifted") executable_data = python.read_bytes() runtime_manifest = { + "schema": RUNTIME_MANIFEST_SCHEMA, "python": payload["python"], "implementation": payload["implementation"], "cache_tag": payload["cache_tag"], "platform": payload["platform"], + "machine": payload["machine"], "flags": payload["flags"], + "startup_policy": payload["startup_policy"], + "excluded_startup_files": [dict(item) for item in excluded_startup_files], + "source_python": dict(source_python), + "source_pyvenv_config": dict(source_pyvenv_config), + "python_runtime_files": [dict(item) for item in python_runtime_manifest], "executable": _file_entry("python.exe", executable_data), "packages": payload["packages"], "installed_distributions": payload["installed_distributions"], + "distribution_file_inventory": payload["distribution_file_inventory"], "forbidden_modules": payload["forbidden_modules"], } return runtime_manifest, paths +def verify_static_runtime_input_source( + *, tokenizer_dir: Path, nltk_data: Path, capture: Any +) -> dict[str, bytes]: + tokenizer_dir = tokenizer_dir.resolve() + if tokenizer_dir.name != capture.resolver.PRIMARY_MODEL_REVISION: + raise ValueError("tokenizer snapshot directory does not name the frozen revision") + files = {path.name: path for path in tokenizer_dir.iterdir() if path.is_file()} + if set(files) != set(EXPECTED_TOKENIZER_ASSETS): + raise ValueError("tokenizer-only asset inventory drifted") + for path in tokenizer_dir.rglob("*"): + if path.is_file() and capture.FORBIDDEN_MODEL_FILE_RE.search( + path.relative_to(tokenizer_dir).as_posix() + ): + raise ValueError(f"model-weight-like file is forbidden in tokenizer snapshot: {path}") + result: dict[str, bytes] = {} + for name, (size, sha256) in sorted(EXPECTED_TOKENIZER_ASSETS.items()): + result[f"tokenizer/{capture.resolver.PRIMARY_MODEL_REVISION}/{name}"] = _verified_file( + files[name], + size=size, + sha256=sha256, + name=f"tokenizer asset {name}", + ) + for name in ("nltk/punkt/english.pickle", "nltk/punkt/PY3/english.pickle"): + relative = name.removeprefix("nltk/punkt/") + nltk_path = nltk_data.resolve() / "tokenizers" / "punkt" / Path(relative) + size, sha256 = EXPECTED_PACKAGE_RESOURCES[name] + result[f"nltk_data/tokenizers/punkt/{relative}"] = _verified_file( + nltk_path, + size=size, + sha256=sha256, + name=f"RULER NLTK punkt resource {relative}", + ) + return result + + def verify_static_inputs( *, ruler_root: Path, @@ -664,8 +1322,10 @@ def verify_static_inputs( resource_paths: Mapping[str, Path], runtime_manifest: Mapping[str, object], capture: Any, -) -> list[dict[str, object]]: +) -> VerifiedStaticInputs: entries: list[dict[str, object]] = [] + corpus_files: dict[str, bytes] = {} + sealed_runtime_files: dict[str, bytes] = {} corpus_root = ruler_root / "scripts" / "data" / "synthetic" / "json" for name, (size, sha256) in sorted(EXPECTED_CORPORA.items()): data = _verified_file( @@ -675,6 +1335,7 @@ def verify_static_inputs( name=f"RULER corpus {name}", ) entries.append(_file_entry(f"corpora/{name}", data)) + corpus_files[name] = data for name, (size, sha256) in sorted(EXPECTED_PACKAGE_RESOURCES.items()): data = _verified_file( resource_paths[name], @@ -683,6 +1344,9 @@ def verify_static_inputs( name=f"RULER package resource {name}", ) entries.append(_file_entry(f"packages/{name}", data)) + if name.startswith("nltk/punkt/"): + relative = name.removeprefix("nltk/punkt/") + sealed_runtime_files[f"nltk_data/tokenizers/punkt/{relative}"] = data tokenizer_dir = tokenizer_dir.resolve() if tokenizer_dir.name != capture.resolver.PRIMARY_MODEL_REVISION: @@ -703,21 +1367,151 @@ def verify_static_inputs( name=f"tokenizer asset {name}", ) entries.append(_file_entry(f"tokenizer/{name}", data)) + sealed_runtime_files[f"tokenizer/{capture.resolver.PRIMARY_MODEL_REVISION}/{name}"] = data runtime_bytes = _canonical_json_bytes(runtime_manifest) entries.append(_file_entry("runtime/package-manifest.json", runtime_bytes)) entries.append(_file_entry("runtime/requirements.txt", _requirements_bytes())) entries.append(_launcher_source_entry()) - return sorted(entries, key=lambda item: str(item["name"])) + return VerifiedStaticInputs( + entries=tuple(sorted(entries, key=lambda item: str(item["name"]))), + corpus_files=corpus_files, + sealed_runtime_files=dict(sealed_runtime_files), + ) + + +def stage_verified_ruler_source( + root: Path, + *, + checkout: VerifiedRulerCheckout, + static_inputs: VerifiedStaticInputs, +) -> dict[str, dict[str, object]]: + """Materialize only authenticated source/corpus bytes in a new isolated tree.""" + + root = root.resolve() + if root.exists() and any(root.iterdir()): + raise ValueError("isolated RULER source root must start empty") + root.mkdir(parents=True, exist_ok=True) + files: dict[str, bytes] = dict(checkout.source_files) + for name, data in static_inputs.corpus_files.items(): + relative = f"scripts/data/synthetic/json/{name}" + if relative in files: + raise ValueError("isolated RULER source inventory contains a duplicate path") + files[relative] = data + manifest: dict[str, dict[str, object]] = {} + for relative, data in sorted(files.items()): + path = root.joinpath(*relative.split("/")) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("xb") as handle: + handle.write(data) + manifest[relative] = { + "sha256": _sha256_bytes(data), + "size_bytes": len(data), + } + verify_staged_ruler_source(root, expected=manifest) + return manifest + + +def verify_staged_ruler_source( + root: Path, + *, + expected: Mapping[str, Mapping[str, object]], +) -> None: + """Reject added, removed, aliased, or modified staged import inputs.""" + + root = root.resolve() + observed: dict[str, Path] = {} + for path in root.rglob("*"): + if path.is_symlink(): + raise ValueError("isolated RULER source tree may not contain symlinks") + if path.is_file(): + relative = path.relative_to(root).as_posix() + observed[relative] = path + observed_names = set(observed) + expected_names = set(expected) + if observed_names != expected_names: + raise ValueError( + "isolated RULER source inventory drifted; " + f"missing={sorted(expected_names - observed_names)}, " + f"extra={sorted(observed_names - expected_names)}" + ) + for relative, path in observed.items(): + data = path.read_bytes() + identity = expected[relative] + if identity.get("sha256") != _sha256_bytes(data) or identity.get("size_bytes") != len(data): + raise ValueError(f"isolated RULER source bytes drifted: {relative}") + + +def stage_verified_runtime_inputs( + root: Path, *, files: Mapping[str, bytes] +) -> tuple[dict[str, dict[str, object]], Path, Path]: + if root.exists(): + raise FileExistsError(f"refusing to replace staged RULER runtime inputs: {root}") + root.mkdir(parents=True) + manifest: dict[str, dict[str, object]] = {} + for relative, data in sorted(files.items()): + path = root.joinpath(*relative.split("/")) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("xb") as handle: + handle.write(data) + manifest[relative] = _tree_file_entry(relative, data) + verify_staged_runtime_inputs(root, expected=manifest) + tokenizer_parent = root / "tokenizer" + tokenizer_directories = ( + [path for path in tokenizer_parent.iterdir() if path.is_dir()] + if tokenizer_parent.is_dir() + else [] + ) + nltk_data = root / "nltk_data" + if len(tokenizer_directories) != 1 or not nltk_data.is_dir(): + raise ValueError("staged RULER tokenizer or NLTK data is missing") + return manifest, tokenizer_directories[0], nltk_data + + +def verify_staged_runtime_inputs( + root: Path, *, expected: Mapping[str, Mapping[str, object]] +) -> None: + root = root.resolve() + observed: dict[str, Path] = {} + for path in root.rglob("*"): + if _is_reparse_point(path): + raise ValueError("staged RULER runtime inputs contain a redirected path") + if path.is_file(): + observed[path.relative_to(root).as_posix()] = path + if set(observed) != set(expected): + raise ValueError("staged RULER runtime-input inventory drifted") + for name, path in observed.items(): + if _tree_file_entry(name, path.read_bytes()) != expected[name]: + raise ValueError(f"staged RULER runtime-input bytes drifted: {name}") class IndependentTokenizer: """Recompute token counts inside the verified tokenizer-only interpreter.""" - def __init__(self, *, python: Path, tokenizer_dir: Path) -> None: + def __init__( + self, + *, + python: Path, + tokenizer_dir: Path, + package_root: Path, + package_tree_manifest: Sequence[Mapping[str, object]], + runtime_input_root: Path, + runtime_input_manifest: Mapping[str, Mapping[str, object]], + python_runtime_root: Path, + python_runtime_manifest: Sequence[Mapping[str, object]], + ) -> None: self._python = python.resolve() self._tokenizer_dir = tokenizer_dir.resolve() + self._package_root = package_root.resolve() + self._runtime_root = self._package_root.parents[1] + self._package_tree_manifest = tuple(dict(item) for item in package_tree_manifest) + self._runtime_input_root = runtime_input_root.resolve() + self._runtime_input_manifest = { + str(name): dict(item) for name, item in runtime_input_manifest.items() + } + self._python_runtime_root = python_runtime_root.resolve() + self._python_runtime_manifest = tuple(dict(item) for item in python_runtime_manifest) def count_tokens(self, text: str) -> int: code = """ @@ -731,18 +1525,56 @@ def count_tokens(self, text: str) -> int: print(json.dumps({'count': len(tokenizer.tokenize(request['text']))})) """ request = {"text": text, "tokenizer_dir": str(self._tokenizer_dir)} - result = subprocess.run( - [str(self._python), "-I", "-c", code], - check=True, - capture_output=True, - text=True, - input=json.dumps(request, ensure_ascii=False, allow_nan=False), - env=_subprocess_env( - HF_HUB_OFFLINE="1", - TOKENIZERS_PARALLELISM="false", - TRANSFORMERS_OFFLINE="1", - ), + verify_staged_runtime_package_tree(self._runtime_root, expected=self._package_tree_manifest) + verify_staged_runtime_inputs( + self._runtime_input_root, expected=self._runtime_input_manifest + ) + verify_staged_python_runtime( + self._python_runtime_root, expected=self._python_runtime_manifest ) + with tempfile.TemporaryDirectory( + prefix="recurquant-exp013-tokenizer-pycache-" + ) as temporary: + pycache_prefix = Path(temporary) + _verify_empty_pycache_prefix(pycache_prefix) + try: + result = subprocess.run( + _sealed_python_argv( + python=self._python, + package_root=self._package_root, + pycache_prefix=pycache_prefix, + code=SEALED_STARTUP_BOOTSTRAP + "\n" + code, + ), + check=False, + capture_output=True, + text=True, + encoding="utf-8", + errors="strict", + input=json.dumps(request, ensure_ascii=False, allow_nan=False), + env=_subprocess_env( + HF_HUB_OFFLINE="1", + TOKENIZERS_PARALLELISM="false", + TRANSFORMERS_OFFLINE="1", + ), + timeout=TOKENIZER_TIMEOUT_SECONDS, + ) + finally: + _verify_empty_pycache_prefix(pycache_prefix) + verify_staged_runtime_package_tree( + self._runtime_root, expected=self._package_tree_manifest + ) + verify_staged_runtime_inputs( + self._runtime_input_root, expected=self._runtime_input_manifest + ) + verify_staged_python_runtime( + self._python_runtime_root, expected=self._python_runtime_manifest + ) + if result.returncode != 0: + diagnostic = result.stderr[-4_000:].strip() + raise RuntimeError( + "sealed independent tokenizer failed" + + (f": {diagnostic}" if diagnostic else " without stderr") + ) payload = _strict_json(result.stdout.encode(), context="independent tokenizer count") if ( not isinstance(payload, Mapping) @@ -766,7 +1598,8 @@ def _token_count(tokenizer: Any, text: str) -> int: def generator_argv( *, python: Path, - ruler_root: Path, + package_root: Path, + staged_root: Path, raw_root: Path, receipt: Mapping[str, object], ) -> tuple[list[str], list[str]]: @@ -775,37 +1608,48 @@ def generator_argv( spec = TASK_SPECS[config] except KeyError as error: raise ValueError(f"no frozen launcher specification for RULER config {config}") from error - script = ruler_root / "scripts" / "data" / "synthetic" / spec.script - actual = [ - str(python.resolve()), - "-s", - str(script.resolve()), - "--save_dir", - str(raw_root.resolve()), - "--save_name", - config, - "--subset", - "validation", - "--tokenizer_path", - "", - "--tokenizer_type", - "hf", - "--max_seq_length", - str(receipt["configured_length"]), - "--tokens_to_generate", - str(spec.tokens_to_generate), - "--num_samples", - "1", - "--random_seed", - str(receipt["seed"]), - ] + data_root = staged_root / "scripts" / "data" + pycache_prefix = raw_root / ".sealed-pycache" + script_relative = f"synthetic/{spec.script}" + actual = _sealed_python_argv( + python=python, + package_root=package_root, + pycache_prefix=pycache_prefix, + code=ISOLATED_SOURCE_BOOTSTRAP, + arguments=( + str(data_root.resolve()), + script_relative, + "--save_dir", + str(raw_root.resolve()), + "--save_name", + config, + "--subset", + "validation", + "--tokenizer_path", + "", + "--tokenizer_type", + "hf", + "--max_seq_length", + str(receipt["configured_length"]), + "--tokens_to_generate", + str(spec.tokens_to_generate), + "--num_samples", + "1", + "--random_seed", + str(receipt["seed"]), + ), + ) for name, value in spec.arguments: actual.extend((f"--{name}", value)) actual.extend(("--template", spec.template)) portable = list(actual) portable[0] = "" - portable[2] = f"scripts/data/synthetic/{spec.script}" - portable[4] = "" + portable[5] = "pycache_prefix=" + portable[10] = "" + portable[11] = "" + portable[12] = "" + portable[13] = script_relative + portable[15] = "" return actual, portable @@ -952,7 +1796,8 @@ def _load_existing_receipt_result( receipt: Mapping[str, object], capture: Any, python: Path, - ruler_root: Path, + package_root: Path, + staged_root: Path, raw_root: Path, tokenizer: Any, static_entries: Sequence[Mapping[str, object]], @@ -969,31 +1814,25 @@ def _load_existing_receipt_result( configured_length=int(receipt["configured_length"]), seed=int(receipt["seed"]), ) + diagnostic_root = raw_root / path.name.removesuffix(".json") + command_path = diagnostic_root / "command-manifest.json" + raw_path = diagnostic_root / config / "validation.jsonl" + if not command_path.is_file() or not raw_path.is_file(): + raise FileNotFoundError(f"existing RULER receipt lacks raw verification inputs: {path}") + raw_data = raw_path.read_bytes() + raw_lines = raw_data.splitlines() + if len(raw_lines) != 1 or not raw_lines[0]: + raise ValueError("existing RULER raw validation must contain exactly one row") + raw_row = _strict_json(raw_lines[0], context=f"existing raw RULER row {path.name}") normalized = _normalize_output_row( - { - "index": ( - str(value["input"]).find(value["outputs"][0]) if TASK_SPECS[config].niah else 0 - ), - "input": value["input"], - "outputs": value["outputs"], - "length": value["generator_reported_length"], - "length_w_model_temp": value["generator_reported_length"], - "answer_prefix": value["answer_prefix"], - **( - { - "token_position_answer": _token_count( - tokenizer, - str(value["input"])[: str(value["input"]).find(value["outputs"][0])], - ) - } - if TASK_SPECS[config].niah - else {} - ), - }, + raw_row, config=config, configured_length=int(receipt["configured_length"]), tokenizer=tokenizer, ) + for field in ("input", "answer_prefix", "outputs", "generator_reported_length"): + if normalized[field] != value[field]: + raise ValueError(f"existing RULER receipt differs from raw generator row: {field}") auxiliary = value["auxiliary_files"] if not isinstance(auxiliary, Sequence): # capture normalization already rejects this raise ValueError("existing RULER receipt auxiliary inventory is invalid") @@ -1005,7 +1844,8 @@ def _load_existing_receipt_result( raise ValueError("existing RULER receipt has a stale static-input binding") _, portable = generator_argv( python=python, - ruler_root=ruler_root, + package_root=package_root, + staged_root=staged_root, raw_root=raw_root, receipt=receipt, ) @@ -1017,9 +1857,14 @@ def _load_existing_receipt_result( static_entries=static_entries, ) ) + if command_path.read_bytes() != command_bytes: + raise ValueError("existing RULER diagnostic command manifest drifted") expected_command = _file_entry("generator/command-manifest.json", command_bytes) + expected_raw = _file_entry("generator/raw-validation.jsonl", raw_data) if auxiliary_by_name.get("generator/command-manifest.json") != expected_command: raise ValueError("existing RULER receipt has a stale launcher binding") + if auxiliary_by_name.get("generator/raw-validation.jsonl") != expected_raw: + raise ValueError("existing RULER receipt has a stale raw-row binding") expected_names = {str(entry["name"]) for entry in static_entries} | { "generator/command-manifest.json", "generator/raw-validation.jsonl", @@ -1027,20 +1872,116 @@ def _load_existing_receipt_result( if set(auxiliary_by_name) != expected_names: raise ValueError("existing RULER receipt auxiliary inventory drifted") return { + "category": receipt["category"], + "command_manifest": _strict_json(command_bytes, context="command manifest"), + "command_manifest_file": expected_command, + "config": config, + "configured_length": receipt["configured_length"], "filename": path.name, + "generator_reported_length": normalized["generator_reported_length"], "phase": receipt["phase"], + "raw_validation_base64": base64.b64encode(raw_data).decode("ascii"), + "raw_validation_file": expected_raw, + "seed": receipt["seed"], "sha256": _sha256_bytes(payload), "size_bytes": len(payload), - "generator_reported_length": normalized["generator_reported_length"], } +def _verify_owned_orphan_tree( + path: Path, + *, + raw_root: Path, + config: str, + require_complete: bool, +) -> None: + raw_root = raw_root.resolve() + if path.parent.resolve() != raw_root or not path.is_dir() or _is_reparse_point(path): + raise ValueError("RULER orphan candidate is outside the requested raw root") + allowed_directories = {"", config, ".sealed-pycache"} + allowed_files = { + "command-manifest.json", + "runtime-manifest.json", + "stdout.log", + "stderr.log", + f"{config}/validation.jsonl", + } + observed_files: set[str] = set() + for item in path.rglob("*"): + if _is_reparse_point(item): + raise ValueError("RULER orphan candidate contains a redirected path") + relative = item.relative_to(path).as_posix() + if item.is_dir(): + if relative not in allowed_directories: + raise ValueError("RULER orphan candidate contains an unexpected directory") + elif item.is_file(): + if relative not in allowed_files: + raise ValueError("RULER orphan candidate contains an unexpected file") + observed_files.add(relative) + else: + raise ValueError("RULER orphan candidate contains an unsupported filesystem object") + if require_complete and observed_files != allowed_files: + raise ValueError("published RULER diagnostic orphan is incomplete") + + +def recover_owned_receipt_orphans( + *, + filename: str, + config: str, + raw_root: Path, + output_dir: Path, +) -> tuple[str, ...]: + """Remove only exact generator-owned leftovers when no receipt was published.""" + + if Path(filename).name != filename or not filename.endswith(".json"): + raise ValueError("RULER receipt filename is not a canonical basename") + raw_root = Path(os.path.abspath(raw_root)) + output_dir = Path(os.path.abspath(output_dir)) + raw_root.mkdir(parents=True, exist_ok=True) + output_dir.mkdir(parents=True, exist_ok=True) + if _is_reparse_point(raw_root) or _is_reparse_point(output_dir): + raise ValueError("RULER raw or output root is redirected") + output_path = output_dir / filename + if output_path.parent.resolve() != output_dir.resolve(): + raise ValueError("RULER receipt output escapes the requested output root") + if output_path.exists(): + return () + receipt_key = _sha256_bytes(filename.encode("utf-8"))[:12] + staging_pattern = re.compile(rf"^\.rq-{re.escape(receipt_key)}\.[A-Za-z0-9_-]+\.staging$") + candidates = sorted( + (path for path in raw_root.iterdir() if staging_pattern.fullmatch(path.name) is not None), + key=lambda path: path.name, + ) + published = raw_root / filename.removesuffix(".json") + if published.exists(): + candidates.append(published) + recovered: list[str] = [] + for candidate in candidates: + _verify_owned_orphan_tree( + candidate, + raw_root=raw_root, + config=config, + require_complete=candidate == published, + ) + for candidate in candidates: + shutil.rmtree(candidate) + recovered.append(candidate.name) + return tuple(recovered) + + def generate_receipt( *, receipt: Mapping[str, object], capture: Any, python: Path, - ruler_root: Path, + package_root: Path, + package_tree_manifest: Sequence[Mapping[str, object]], + python_runtime_root: Path, + python_runtime_manifest: Sequence[Mapping[str, object]], + runtime_input_root: Path, + runtime_input_manifest: Mapping[str, Mapping[str, object]], + staged_root: Path, + staged_manifest: Mapping[str, Mapping[str, object]], tokenizer_dir: Path, nltk_data: Path, raw_root: Path, @@ -1052,17 +1993,36 @@ def generate_receipt( ) -> dict[str, object]: filename = str(receipt["filename"]) output_path = output_dir / filename + recovered = recover_owned_receipt_orphans( + filename=filename, + config=str(receipt["config"]), + raw_root=raw_root, + output_dir=output_dir, + ) + if recovered: + print( + f"recovered {len(recovered)} generator-owned orphan(s) for {filename}", + flush=True, + ) if output_path.exists(): raise FileExistsError(f"refusing to overwrite RULER receipt: {output_path}") - raw_root.mkdir(parents=True, exist_ok=True) + published_raw_root = raw_root / filename.removesuffix(".json") + if published_raw_root.exists(): + raise FileExistsError( + f"refusing to replace existing RULER diagnostics: {published_raw_root}" + ) receipt_key = _sha256_bytes(filename.encode("utf-8"))[:12] receipt_raw_root = Path( tempfile.mkdtemp(prefix=f".rq-{receipt_key}.", suffix=".staging", dir=raw_root) ) + pycache_prefix = receipt_raw_root / ".sealed-pycache" + pycache_prefix.mkdir() + _verify_empty_pycache_prefix(pycache_prefix) actual, portable = generator_argv( python=python, - ruler_root=ruler_root, + package_root=package_root, + staged_root=staged_root, raw_root=receipt_raw_root, receipt=receipt, ) @@ -1082,19 +2042,34 @@ def generate_receipt( env = _subprocess_env( HF_HUB_OFFLINE="1", NLTK_DATA=str(nltk_data.resolve()), + PYTHONDONTWRITEBYTECODE="1", PYTHONHASHSEED="0", TOKENIZERS_PARALLELISM="false", TRANSFORMERS_OFFLINE="1", ) - result = subprocess.run( - actual, - cwd=ruler_root / "scripts" / "data", - env=env, - capture_output=True, - text=True, - timeout=timeout_seconds, - check=False, - ) + runtime_root = package_root.resolve().parents[1] + verify_staged_runtime_package_tree(runtime_root, expected=package_tree_manifest) + verify_staged_python_runtime(python_runtime_root, expected=python_runtime_manifest) + verify_staged_runtime_inputs(runtime_input_root, expected=runtime_input_manifest) + try: + result = subprocess.run( + actual, + cwd=staged_root / "scripts" / "data", + env=env, + capture_output=True, + text=True, + encoding="utf-8", + errors="strict", + timeout=timeout_seconds, + check=False, + ) + finally: + verify_staged_ruler_source(staged_root, expected=staged_manifest) + verify_staged_runtime_package_tree(runtime_root, expected=package_tree_manifest) + verify_staged_python_runtime(python_runtime_root, expected=python_runtime_manifest) + verify_staged_runtime_inputs(runtime_input_root, expected=runtime_input_manifest) + _verify_empty_pycache_prefix(pycache_prefix) + pycache_prefix.rmdir() (receipt_raw_root / "stdout.log").write_text(result.stdout, encoding="utf-8") (receipt_raw_root / "stderr.log").write_text(result.stderr, encoding="utf-8") if result.returncode != 0: @@ -1144,28 +2119,94 @@ def generate_receipt( seed=int(receipt["seed"]), ) payload = _canonical_json_bytes(receipt_data) + # Publish the diagnostics first. A crash can therefore leave an obvious, + # fail-closed orphan, but can never leave a receipt without the raw inputs + # needed to reproduce it. + receipt_raw_root.rename(published_raw_root) _atomic_publish_new(output_path, payload) - published_raw_root = raw_root / filename.removesuffix(".json") - try: - receipt_raw_root.rename(published_raw_root) - except FileExistsError: - # Raw logs are diagnostics, not publication inputs. Keep this staged - # run rather than replacing an older diagnostic directory. - print( - f"warning: kept raw diagnostics at {receipt_raw_root} because " - f"{published_raw_root} already exists", - file=sys.stderr, - flush=True, - ) + command_entry = _file_entry("generator/command-manifest.json", command_bytes) + raw_entry = _file_entry("generator/raw-validation.jsonl", raw_data) return { + "category": receipt["category"], + "command_manifest": command_manifest, + "command_manifest_file": command_entry, + "config": config, + "configured_length": receipt["configured_length"], "filename": filename, + "generator_reported_length": normalized["generator_reported_length"], "phase": receipt["phase"], + "raw_validation_base64": base64.b64encode(raw_data).decode("ascii"), + "raw_validation_file": raw_entry, + "seed": receipt["seed"], "sha256": _sha256_bytes(payload), "size_bytes": len(payload), - "generator_reported_length": normalized["generator_reported_length"], } +def finalize_generation_manifest_if_complete( + *, + required: Sequence[Mapping[str, object]], + output_dir: Path, + raw_root: Path, + capture: Any, + python: Path, + package_root: Path, + staged_root: Path, + tokenizer: Any, + static_entries: Sequence[Mapping[str, object]], + source_manifest: Sequence[Mapping[str, object]], + runtime_manifest: Mapping[str, object], +) -> dict[str, object] | None: + """Publish the sole manifest only after re-verifying the full 20-file set.""" + + manifest_path = output_dir / "generation-manifest.json" + missing = [ + str(item["filename"]) + for item in required + if not (output_dir / str(item["filename"])).is_file() + ] + if missing: + if manifest_path.exists(): + raise ValueError("complete RULER generation manifest exists beside an incomplete set") + return None + + # Do not trust results retained from the generation loop: independently + # reopen every receipt, raw row, and command binding in canonical order. + results = [ + _load_existing_receipt_result( + path=output_dir / str(receipt["filename"]), + receipt=receipt, + capture=capture, + python=python, + package_root=package_root, + staged_root=staged_root, + raw_root=raw_root, + tokenizer=tokenizer, + static_entries=static_entries, + ) + for receipt in required + ] + launcher_entry = _entry_named(static_entries, "launcher/generate_static_q468_ruler_receipts.py") + source_manifest_value = [dict(item) for item in source_manifest] + manifest: dict[str, object] = { + "schema": GENERATION_MANIFEST_SCHEMA, + "launcher_revision": LAUNCHER_REVISION, + "launcher_source": dict(launcher_entry), + "ruler_revision": capture.resolver.RULER_REVISION, + "source_manifest": source_manifest_value, + "source_manifest_sha256": _sha256_bytes(_canonical_json_bytes(source_manifest_value)), + "runtime_manifest": dict(runtime_manifest), + "runtime_manifest_sha256": _sha256_bytes(_canonical_json_bytes(runtime_manifest)), + "static_inputs": [dict(item) for item in static_entries], + "receipt_count": len(results), + "receipts": results, + } + if len(results) != 20: + raise ValueError("complete RULER generation manifest must contain exactly 20 receipts") + _atomic_publish_same(manifest_path, _canonical_json_bytes(manifest)) + return manifest + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--ruler-root", type=Path, required=True) @@ -1189,23 +2230,14 @@ def main(argv: Sequence[str] | None = None) -> int: if args.timeout_seconds < 1: raise ValueError("--timeout-seconds must be positive") capture = _load_capture_module() - source_manifest = verify_ruler_checkout(args.ruler_root, capture) - runtime_manifest, resource_paths = verify_runtime(args.python, args.nltk_data) - static_entries = verify_static_inputs( - ruler_root=args.ruler_root, + checkout = verify_ruler_checkout(args.ruler_root, capture) + verified_package_tree = verify_runtime_package_source(args.python) + verified_python_runtime = verify_python_runtime_source(args.python) + static_runtime_files = verify_static_runtime_input_source( tokenizer_dir=args.tokenizer_dir, - resource_paths=resource_paths, - runtime_manifest=runtime_manifest, + nltk_data=args.nltk_data, capture=capture, ) - static_entries = sorted( - [ - *static_entries, - _file_entry("ruler/source-manifest.json", _canonical_json_bytes(source_manifest)), - ], - key=lambda item: str(item["name"]), - ) - tokenizer = IndependentTokenizer(python=args.python, tokenizer_dir=args.tokenizer_dir) required = list(capture.required_ruler_receipts()) by_filename = {str(item["filename"]): item for item in required} if len(by_filename) != len(required): @@ -1231,55 +2263,147 @@ def main(argv: Sequence[str] | None = None) -> int: selected = [by_filename[name] for name in args.receipt] else: selected = required - results = [] - for index, receipt in enumerate(selected, start=1): - output_path = args.output_dir / str(receipt["filename"]) - if output_path.exists(): - print( - f"[{index}/{len(selected)}] verifying existing {receipt['filename']}", - flush=True, - ) - result = _load_existing_receipt_result( - path=output_path, - receipt=receipt, - capture=capture, - python=args.python, - ruler_root=args.ruler_root, - raw_root=args.raw_dir, - tokenizer=tokenizer, - static_entries=static_entries, - ) - else: - print(f"[{index}/{len(selected)}] generating {receipt['filename']}", flush=True) - result = generate_receipt( - receipt=receipt, - capture=capture, - python=args.python, - ruler_root=args.ruler_root, - tokenizer_dir=args.tokenizer_dir, - nltk_data=args.nltk_data, - raw_root=args.raw_dir, - output_dir=args.output_dir, - tokenizer=tokenizer, - static_entries=static_entries, - runtime_manifest=runtime_manifest, - timeout_seconds=args.timeout_seconds, + with tempfile.TemporaryDirectory(prefix="recurquant-exp013-ruler-sealed-") as temporary: + temporary_root = Path(temporary) + python_runtime_root = temporary_root / "python-runtime" + staged_python = stage_verified_python_runtime( + python_runtime_root, verified=verified_python_runtime + ) + package_runtime_root = temporary_root / "package-runtime" + package_root = stage_verified_runtime_package_tree( + package_runtime_root, verified=verified_package_tree + ) + runtime_input_root = temporary_root / "runtime-inputs" + runtime_input_manifest, staged_tokenizer_dir, staged_nltk_data = ( + stage_verified_runtime_inputs( + runtime_input_root, + files=static_runtime_files, ) - results.append(result) - launcher_entry = _entry_named(static_entries, "launcher/generate_static_q468_ruler_receipts.py") - manifest = { - "schema": "recurquant.experiment013.ruler-generation-manifest.v1", - "launcher_revision": LAUNCHER_REVISION, - "launcher_source_sha256": launcher_entry["sha256"], - "ruler_revision": capture.resolver.RULER_REVISION, - "source_manifest_sha256": _sha256_bytes(_canonical_json_bytes(source_manifest)), - "runtime_manifest_sha256": _sha256_bytes(_canonical_json_bytes(runtime_manifest)), - "selected_receipts": results, - } - _atomic_publish_same( - args.output_dir / "generation-manifest.json", _canonical_json_bytes(manifest) - ) - print(json.dumps(manifest, indent=2, sort_keys=True), flush=True) + ) + runtime_manifest, resource_paths = verify_runtime( + staged_python, + staged_nltk_data, + package_root=package_root, + package_tree_manifest=verified_package_tree.entries, + excluded_startup_files=verified_package_tree.excluded_startup_files, + python_runtime_root=python_runtime_root, + python_runtime_manifest=verified_python_runtime.entries, + source_python=verified_python_runtime.source_launcher, + source_pyvenv_config=verified_python_runtime.source_pyvenv_config, + ) + verified_static_inputs = verify_static_inputs( + ruler_root=args.ruler_root, + tokenizer_dir=staged_tokenizer_dir, + resource_paths=resource_paths, + runtime_manifest=runtime_manifest, + capture=capture, + ) + if verified_static_inputs.sealed_runtime_files != static_runtime_files: + raise ValueError("staged RULER tokenizer or NLTK bytes drifted during verification") + static_entries = sorted( + [ + *verified_static_inputs.entries, + _file_entry( + "ruler/source-manifest.json", + _canonical_json_bytes(list(checkout.source_manifest)), + ), + ], + key=lambda item: str(item["name"]), + ) + tokenizer = IndependentTokenizer( + python=staged_python, + tokenizer_dir=staged_tokenizer_dir, + package_root=package_root, + package_tree_manifest=verified_package_tree.entries, + runtime_input_root=runtime_input_root, + runtime_input_manifest=runtime_input_manifest, + python_runtime_root=python_runtime_root, + python_runtime_manifest=verified_python_runtime.entries, + ) + staged_root = temporary_root / "ruler-source" + staged_manifest = stage_verified_ruler_source( + staged_root, + checkout=checkout, + static_inputs=verified_static_inputs, + ) + for index, receipt in enumerate(selected, start=1): + output_path = args.output_dir / str(receipt["filename"]) + if output_path.exists(): + print( + f"[{index}/{len(selected)}] verifying existing {receipt['filename']}", + flush=True, + ) + _load_existing_receipt_result( + path=output_path, + receipt=receipt, + capture=capture, + python=staged_python, + package_root=package_root, + staged_root=staged_root, + raw_root=args.raw_dir, + tokenizer=tokenizer, + static_entries=static_entries, + ) + else: + print(f"[{index}/{len(selected)}] generating {receipt['filename']}", flush=True) + generate_receipt( + receipt=receipt, + capture=capture, + python=staged_python, + package_root=package_root, + package_tree_manifest=verified_package_tree.entries, + python_runtime_root=python_runtime_root, + python_runtime_manifest=verified_python_runtime.entries, + runtime_input_root=runtime_input_root, + runtime_input_manifest=runtime_input_manifest, + staged_root=staged_root, + staged_manifest=staged_manifest, + tokenizer_dir=staged_tokenizer_dir, + nltk_data=staged_nltk_data, + raw_root=args.raw_dir, + output_dir=args.output_dir, + tokenizer=tokenizer, + static_entries=static_entries, + runtime_manifest=runtime_manifest, + timeout_seconds=args.timeout_seconds, + ) + manifest = finalize_generation_manifest_if_complete( + required=required, + output_dir=args.output_dir, + raw_root=args.raw_dir, + capture=capture, + python=staged_python, + package_root=package_root, + staged_root=staged_root, + tokenizer=tokenizer, + static_entries=static_entries, + source_manifest=checkout.source_manifest, + runtime_manifest=runtime_manifest, + ) + verify_staged_python_runtime(python_runtime_root, expected=verified_python_runtime.entries) + verify_staged_runtime_package_tree( + package_runtime_root, expected=verified_package_tree.entries + ) + verify_staged_runtime_inputs(runtime_input_root, expected=runtime_input_manifest) + if manifest is None: + present = sum((args.output_dir / str(item["filename"])).is_file() for item in required) + progress = { + "schema": "recurquant.experiment013.ruler-generation-progress.v1", + "complete": False, + "present_receipts": present, + "required_receipts": len(required), + } + print(json.dumps(progress, indent=2, sort_keys=True), flush=True) + else: + manifest_path = args.output_dir / "generation-manifest.json" + summary = { + "schema": "recurquant.experiment013.ruler-generation-success.v1", + "complete": True, + "manifest": str(manifest_path.resolve()), + "manifest_sha256": _sha256_bytes(manifest_path.read_bytes()), + "receipt_count": manifest["receipt_count"], + } + print(json.dumps(summary, sort_keys=True, separators=(",", ":")), flush=True) return 0 diff --git a/scripts/launch_static_q468_calibration.py b/scripts/launch_static_q468_calibration.py new file mode 100644 index 0000000..ef100f9 --- /dev/null +++ b/scripts/launch_static_q468_calibration.py @@ -0,0 +1,1732 @@ +#!/usr/bin/env python3 +"""Launch Experiment 013 in an authenticated, site-free staged Python runtime. + +This host process is metadata-only. It verifies the frozen runtime, source, and +identity bindings, starts the staged interpreter with an exact isolation argv, +and lets the child repeat every material verification before loading the runner. +Model weights and calibration datasets are intentionally outside this module. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import os +import platform +import re +import shutil +import stat +import subprocess +import sys +import tempfile +from collections.abc import Mapping, Sequence +from pathlib import Path, PurePosixPath +from typing import Final + +RUNTIME_MANIFEST_KIND: Final = "recurquant_experiment013_calibration_runtime_manifest" +RUNTIME_MANIFEST_SCHEMA: Final = 3 +IDENTITY_SCHEMA: Final = 4 +BASE_RUNTIME_ROOT_NAME: Final = "base-runtime" +RUNNER_SOURCE_PATH: Final = "scripts/run_static_q468_calibration.py" +RUNNER_MODULE_NAME: Final = "_recurquant_experiment013_sealed_runner" +_WINDOWS_REPARSE_POINT: Final = 0x400 +_SHA256_RE: Final = re.compile(r"[0-9a-f]{64}") +_SHA1_RE: Final = re.compile(r"[0-9a-f]{40}") +_ROOT_NAME_RE: Final = re.compile(r"[a-z][a-z0-9-]{0,63}") +_FORBIDDEN_SUFFIXES: Final = frozenset({".egg-link", ".pth", ".pyc", ".pyo", "._pth"}) +_FORBIDDEN_DIRECTORIES: Final = frozenset({"__pycache__"}) +_FORBIDDEN_FILENAMES: Final = frozenset( + {"pyvenv.cfg", "sitecustomize.py", "usercustomize.py"} +) +_WINDOWS_RESERVED_NAMES: Final = frozenset( + { + "aux", + "clock$", + "con", + "nul", + "prn", + *(f"com{index}" for index in range(1, 10)), + *(f"lpt{index}" for index in range(1, 10)), + } +) +SEALED_LAUNCH_POLICY: Final = { + "bootstrap_mode": "stdlib-only-exact-runner-v1", + "dont_write_bytecode": 1, + "ignore_environment": 1, + "isolated": 1, + "no_site": 1, + "no_user_site": 1, + "package_path_mode": "authenticated-record-only-roots-v1", + "pycache_mode": "new-verified-empty-prefix-v1", + "safe_path": True, + "site_loaded": False, + "sys_path_mode": "staged-base-then-authenticated-packages-v1", + "utf8_mode": 1, + "virtualenv_hook_loaded": False, +} +_BOUND_ARTIFACT_OPTIONS: Final = { + "calibration_runtime_manifest_file_sha256": "--runtime-manifest", + "model_file_manifest_file_sha256": "--model-file-manifest", + "parquet_materialization_manifest_file_sha256": "--parquet-materialization-manifest", + "repository_source_manifest_file_sha256": "--repository-source-manifest", +} +_EXPECTED_BOUND_DIGEST_OPTIONS: Final = { + "calibration_runtime_manifest_file_sha256": "--expected-runtime-manifest-sha256", + "model_file_manifest_file_sha256": "--expected-model-file-manifest-sha256", + "parquet_materialization_manifest_file_sha256": ( + "--expected-parquet-materialization-manifest-sha256" + ), +} +_REQUIRED_RUNNER_OPTIONS: Final = frozenset( + { + "--frozen-identity", + "--repository-root", + *_BOUND_ARTIFACT_OPTIONS.values(), + *_EXPECTED_BOUND_DIGEST_OPTIONS.values(), + } +) +_SENSITIVE_MODULES: Final = frozenset( + { + "_virtualenv", + RUNNER_MODULE_NAME, + "recurquant", + "recurquant.experiment013_calibration_api", + "recurquant.experiment013_qwen35_adapter", + "recurquant.experiment013_source", + "recurquant_experiment013_identity_resolver", + "site", + } +) + + +class SealedLaunchError(RuntimeError): + """Raised when the staged launch cannot be authenticated safely.""" + + +def _canonical_json_bytes(value: object) -> bytes: + return ( + json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + b"\n" + ) + + +def _pretty_json_bytes(value: object) -> bytes: + return (json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n").encode( + "utf-8" + ) + + +def _sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _strict_json(data: bytes, *, context: str) -> dict[str, object]: + if not isinstance(data, bytes): + raise TypeError(f"{context} must be bytes") + + def unique(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise SealedLaunchError(f"{context} contains a duplicate JSON key") + result[key] = value + return result + + def reject_constant(value: str) -> None: + raise SealedLaunchError(f"{context} contains a non-finite JSON constant: {value}") + + try: + value = json.loads( + data.decode("utf-8"), + object_pairs_hook=unique, + parse_constant=reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise SealedLaunchError(f"{context} is not strict UTF-8 JSON") from exc + if not isinstance(value, dict): + raise SealedLaunchError(f"{context} must be a JSON object") + return value + + +def _exact_fields(value: Mapping[str, object], expected: set[str], *, context: str) -> None: + if set(value) != expected: + raise SealedLaunchError(f"{context} fields differ from the frozen schema") + + +def _exact_typed_mapping( + value: object, + expected: Mapping[str, object], + *, + context: str, +) -> None: + if not isinstance(value, Mapping) or set(value) != set(expected): + raise SealedLaunchError(f"{context} fields differ from the frozen schema") + if any( + type(value[name]) is not type(expected[name]) or value[name] != expected[name] + for name in expected + ): + raise SealedLaunchError(f"{context} value or JSON type drifted") + + +def _sha256(value: object, *, context: str) -> str: + if not isinstance(value, str) or _SHA256_RE.fullmatch(value) is None: + raise SealedLaunchError(f"{context} must be a lowercase SHA-256 digest") + return value + + +def _positive_int(value: object, *, context: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise SealedLaunchError(f"{context} must be a positive integer") + return value + + +def _nonnegative_int(value: object, *, context: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise SealedLaunchError(f"{context} must be a non-negative integer") + return value + + +def _canonical_relative_path(value: object, *, context: str) -> str: + if not isinstance(value, str) or not value or value != value.strip(): + raise SealedLaunchError(f"{context} is not a canonical relative path") + if any(character in value for character in ("\\", "\0", "\n", "\r", ":")): + raise SealedLaunchError(f"{context} is not a safe POSIX path") + path = PurePosixPath(value) + if ( + path.is_absolute() + or path.as_posix() != value + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise SealedLaunchError(f"{context} is not a canonical relative path") + for part in path.parts: + if part.endswith((" ", ".")) or part.split(".", 1)[0].casefold() in _WINDOWS_RESERVED_NAMES: + raise SealedLaunchError(f"{context} is unsafe on Windows") + return value + + +def _root_name(value: object, *, context: str) -> str: + if not isinstance(value, str) or _ROOT_NAME_RE.fullmatch(value) is None: + raise SealedLaunchError(f"{context} is not a canonical runtime-root name") + return value + + +def _is_link_or_reparse(path: Path) -> bool: + try: + status = path.lstat() + except OSError as exc: + raise SealedLaunchError(f"required path is unavailable: {path}") from exc + return path.is_symlink() or bool( + getattr(status, "st_file_attributes", 0) & _WINDOWS_REPARSE_POINT + ) + + +def _absolute_directory(path: Path, *, context: str) -> Path: + candidate = Path(os.path.abspath(path)) + if _is_link_or_reparse(candidate): + raise SealedLaunchError(f"{context} is a link or reparse point") + try: + resolved = candidate.resolve(strict=True) + except OSError as exc: + raise SealedLaunchError(f"{context} is unavailable") from exc + if not resolved.is_dir(): + raise SealedLaunchError(f"{context} is not a directory") + return resolved + + +def _safe_join(root: Path, relative: str, *, context: str, directory: bool = False) -> Path: + candidate = root + for part in PurePosixPath(_canonical_relative_path(relative, context=context)).parts: + candidate /= part + if _is_link_or_reparse(candidate): + raise SealedLaunchError(f"{context} traverses a link or reparse point") + try: + resolved = candidate.resolve(strict=True) + resolved.relative_to(root) + except (OSError, ValueError) as exc: + raise SealedLaunchError(f"{context} escapes its authenticated root") from exc + if directory and not resolved.is_dir(): + raise SealedLaunchError(f"{context} is not a directory") + if not directory and not resolved.is_file(): + raise SealedLaunchError(f"{context} is not a regular file") + return resolved + + +def _forbidden_runtime_path(relative: str) -> bool: + path = PurePosixPath(relative) + return ( + any(part.casefold() in _FORBIDDEN_DIRECTORIES for part in path.parts) + or path.name.casefold() in _FORBIDDEN_FILENAMES + or path.suffix.casefold() in _FORBIDDEN_SUFFIXES + ) + + +def _stable_file_record(path: Path, *, relative: str, context: str) -> dict[str, object]: + before = path.stat() + if not stat.S_ISREG(before.st_mode) or _is_link_or_reparse(path): + raise SealedLaunchError(f"{context} is not a stable regular file") + digest = hashlib.sha256() + size = 0 + try: + with path.open("rb") as handle: + while chunk := handle.read(1024 * 1024): + digest.update(chunk) + size += len(chunk) + except OSError as exc: + raise SealedLaunchError(f"cannot read {context}") from exc + after = path.stat() + if ( + not stat.S_ISREG(after.st_mode) + or before.st_size != after.st_size + or before.st_mtime_ns != after.st_mtime_ns + or size != after.st_size + or _is_link_or_reparse(path) + ): + raise SealedLaunchError(f"{context} changed while it was authenticated") + return {"path": relative, "sha256": digest.hexdigest(), "size_bytes": size} + + +def _tree_files(root: Path, *, context: str) -> tuple[dict[str, object], ...]: + root = _absolute_directory(root, context=f"{context} root") + stack: list[tuple[Path, tuple[str, ...]]] = [(root, ())] + files: list[dict[str, object]] = [] + folded: set[str] = set() + while stack: + directory, parents = stack.pop() + try: + entries = sorted(os.scandir(directory), key=lambda item: item.name.casefold()) + except OSError as exc: + raise SealedLaunchError(f"cannot enumerate {context}") from exc + directory_names: set[str] = set() + for entry in entries: + relative = _canonical_relative_path( + PurePosixPath(*parents, entry.name).as_posix(), + context=f"{context} path", + ) + folded_relative = relative.casefold() + if folded_relative in folded or folded_relative in directory_names: + raise SealedLaunchError(f"{context} has a case-insensitive path collision") + directory_names.add(folded_relative) + try: + status = entry.stat(follow_symlinks=False) + except OSError as exc: + raise SealedLaunchError(f"{context} path is unavailable") from exc + if entry.is_symlink() or bool( + getattr(status, "st_file_attributes", 0) & _WINDOWS_REPARSE_POINT + ): + raise SealedLaunchError(f"{context} contains a link or reparse point") + if stat.S_ISDIR(status.st_mode): + if entry.name.casefold() in _FORBIDDEN_DIRECTORIES: + raise SealedLaunchError(f"{context} contains a forbidden cache directory") + stack.append((Path(entry.path), (*parents, entry.name))) + continue + if not stat.S_ISREG(status.st_mode): + raise SealedLaunchError(f"{context} contains a non-regular path") + if _forbidden_runtime_path(relative): + raise SealedLaunchError(f"{context} contains a forbidden runtime file") + folded.add(folded_relative) + files.append( + _stable_file_record( + Path(entry.path), + relative=relative, + context=f"{context} file", + ) + ) + files.sort(key=lambda item: str(item["path"])) + if not files: + raise SealedLaunchError(f"{context} has no files") + return tuple(files) + + +def _parse_runtime_manifest(data: bytes) -> dict[str, object]: + root = _strict_json(data, context="runtime manifest") + _exact_fields( + root, + { + "artifact_kind", + "base_runtime_root", + "base_sys_path", + "distributions", + "interpreter", + "launch_policy", + "machine", + "package_roots", + "python", + "runtime_trees", + "schema_version", + }, + context="runtime manifest", + ) + if _canonical_json_bytes(root) != data: + raise SealedLaunchError("runtime manifest is not canonical JSON") + if ( + root["artifact_kind"] != RUNTIME_MANIFEST_KIND + or type(root["schema_version"]) is not int + or root["schema_version"] != RUNTIME_MANIFEST_SCHEMA + ): + raise SealedLaunchError("runtime manifest kind or schema drifted") + _exact_typed_mapping( + root["launch_policy"], + SEALED_LAUNCH_POLICY, + context="runtime manifest launch policy", + ) + + python_record = root["python"] + if not isinstance(python_record, dict): + raise SealedLaunchError("runtime Python record must be an object") + _exact_fields( + python_record, + {"abi_flags", "cache_tag", "implementation", "version"}, + context="runtime Python record", + ) + for field in ("abi_flags", "cache_tag", "implementation", "version"): + value = python_record[field] + if ( + not isinstance(value, str) + or value != value.strip() + or (field != "abi_flags" and not value) + ): + raise SealedLaunchError("runtime Python identity is invalid") + + machine = root["machine"] + if not isinstance(machine, dict): + raise SealedLaunchError("runtime machine record must be an object") + _exact_fields( + machine, + {"architecture", "byteorder", "machine", "pointer_bits", "system"}, + context="runtime machine record", + ) + for field in ("architecture", "machine", "system"): + value = machine[field] + if not isinstance(value, str) or not value or value != value.strip(): + raise SealedLaunchError("runtime machine identity is invalid") + if machine["byteorder"] not in {"big", "little"}: + raise SealedLaunchError("runtime byte order is invalid") + _positive_int(machine["pointer_bits"], context="runtime pointer bits") + + if _root_name(root["base_runtime_root"], context="base runtime root") != ( + BASE_RUNTIME_ROOT_NAME + ): + raise SealedLaunchError("base runtime root name drifted") + raw_base_sys_path = root["base_sys_path"] + if not isinstance(raw_base_sys_path, list) or not raw_base_sys_path: + raise SealedLaunchError("base sys.path must be a non-empty list") + base_sys_path = [ + _canonical_relative_path(item, context="base sys.path entry") + for item in raw_base_sys_path + ] + if len({item.casefold() for item in base_sys_path}) != len(base_sys_path): + raise SealedLaunchError("base sys.path entries collide") + + raw_package_roots = root["package_roots"] + if not isinstance(raw_package_roots, list) or not raw_package_roots: + raise SealedLaunchError("runtime manifest has no package roots") + package_roots: list[dict[str, str]] = [] + for item in raw_package_roots: + if not isinstance(item, dict): + raise SealedLaunchError("package root record must be an object") + _exact_fields(item, {"import_path", "name"}, context="package root record") + package_roots.append( + { + "name": _root_name(item["name"], context="package root name"), + "import_path": _canonical_relative_path( + item["import_path"], context="package import path" + ), + } + ) + names = [item["name"] for item in package_roots] + if ( + names != sorted(names) + or len(set(names)) != len(names) + or BASE_RUNTIME_ROOT_NAME in names + ): + raise SealedLaunchError("package roots are not unique and sorted") + + raw_trees = root["runtime_trees"] + if not isinstance(raw_trees, list) or len(raw_trees) != len(package_roots) + 1: + raise SealedLaunchError("runtime tree inventory differs from declared roots") + trees: list[dict[str, object]] = [] + expected_tree_names = [BASE_RUNTIME_ROOT_NAME, *names] + for index, raw_tree in enumerate(raw_trees): + if not isinstance(raw_tree, dict): + raise SealedLaunchError("runtime tree record must be an object") + _exact_fields(raw_tree, {"files", "kind", "name"}, context="runtime tree record") + name = _root_name(raw_tree["name"], context="runtime tree name") + kind = raw_tree["kind"] + expected_kind = "base-runtime" if index == 0 else "packages" + if name != expected_tree_names[index] or kind != expected_kind: + raise SealedLaunchError("runtime tree order or kind drifted") + raw_files = raw_tree["files"] + if not isinstance(raw_files, list) or not raw_files: + raise SealedLaunchError("runtime tree has no files") + files: list[dict[str, object]] = [] + for raw_file in raw_files: + if not isinstance(raw_file, dict): + raise SealedLaunchError("runtime file record must be an object") + _exact_fields( + raw_file, + {"path", "sha256", "size_bytes"}, + context="runtime file record", + ) + relative = _canonical_relative_path( + raw_file["path"], context="runtime file path" + ) + if _forbidden_runtime_path(relative): + raise SealedLaunchError("runtime manifest contains a forbidden file") + files.append( + { + "path": relative, + "sha256": _sha256(raw_file["sha256"], context="runtime file SHA-256"), + "size_bytes": _nonnegative_int( + raw_file["size_bytes"], context="runtime file size" + ), + } + ) + paths = [str(item["path"]) for item in files] + if paths != sorted(paths) or len({item.casefold() for item in paths}) != len(paths): + raise SealedLaunchError("runtime file inventory is not unique and sorted") + trees.append({"files": files, "kind": kind, "name": name}) + + interpreter = root["interpreter"] + if not isinstance(interpreter, dict): + raise SealedLaunchError("runtime interpreter record must be an object") + _exact_fields( + interpreter, + {"relative_path", "root", "sha256", "size_bytes"}, + context="runtime interpreter record", + ) + interpreter_path = _canonical_relative_path( + interpreter["relative_path"], context="runtime interpreter path" + ) + normalized_interpreter = { + "relative_path": interpreter_path, + "root": interpreter["root"], + "sha256": _sha256(interpreter["sha256"], context="runtime interpreter SHA-256"), + "size_bytes": _positive_int( + interpreter["size_bytes"], context="runtime interpreter size" + ), + } + if normalized_interpreter["root"] != BASE_RUNTIME_ROOT_NAME: + raise SealedLaunchError("runtime interpreter is not in the base tree") + base_files = {item["path"]: item for item in trees[0]["files"]} + expected_interpreter_file = { + "path": interpreter_path, + "sha256": normalized_interpreter["sha256"], + "size_bytes": normalized_interpreter["size_bytes"], + } + if base_files.get(interpreter_path) != expected_interpreter_file: + raise SealedLaunchError("runtime interpreter differs from its base-tree record") + for entry in base_sys_path: + present = entry in base_files or any( + str(path).startswith(f"{entry}/") for path in base_files + ) + optional_zip = re.fullmatch(r"python[0-9]+\.zip", entry) is not None + if not present and not optional_zip: + raise SealedLaunchError("base sys.path entry is absent from the base tree") + + raw_distributions = root["distributions"] + if not isinstance(raw_distributions, list) or not raw_distributions: + raise SealedLaunchError("runtime manifest has no distributions") + distributions: list[dict[str, object]] = [] + ownership = {name: set() for name in names} + for raw_distribution in raw_distributions: + if not isinstance(raw_distribution, dict): + raise SealedLaunchError("runtime distribution record must be an object") + _exact_fields( + raw_distribution, + {"files", "name", "package_root", "version"}, + context="runtime distribution record", + ) + name = _canonical_distribution_name(raw_distribution["name"]) + if name != raw_distribution["name"]: + raise SealedLaunchError("distribution name is not canonical") + version = raw_distribution["version"] + package_root = _root_name( + raw_distribution["package_root"], context="distribution package root" + ) + if ( + not isinstance(version, str) + or not version + or version != version.strip() + or package_root not in ownership + ): + raise SealedLaunchError("runtime distribution identity is invalid") + raw_files = raw_distribution["files"] + if not isinstance(raw_files, list) or not raw_files: + raise SealedLaunchError("runtime distribution has no RECORD files") + files = [ + _canonical_relative_path(item, context="distribution RECORD path") + for item in raw_files + ] + if files != sorted(files) or len({item.casefold() for item in files}) != len(files): + raise SealedLaunchError("distribution RECORD paths are not unique and sorted") + overlap = ownership[package_root].intersection(files) + if overlap: + raise SealedLaunchError("runtime distributions claim the same installed file") + ownership[package_root].update(files) + distributions.append( + {"files": files, "name": name, "package_root": package_root, "version": version} + ) + distribution_names = [str(item["name"]) for item in distributions] + if distribution_names != sorted(distribution_names) or len(set(distribution_names)) != len( + distribution_names + ): + raise SealedLaunchError("runtime distributions are not unique and sorted") + tree_by_name = {str(item["name"]): item for item in trees} + for name in names: + if ownership[name] != {str(item["path"]) for item in tree_by_name[name]["files"]}: + raise SealedLaunchError("package tree differs from exact RECORD ownership") + + return { + "base_sys_path": base_sys_path, + "distributions": distributions, + "file_sha256": _sha256_bytes(data), + "interpreter": normalized_interpreter, + "machine": machine, + "package_roots": package_roots, + "python": python_record, + "runtime_trees": trees, + } + + +def _canonical_distribution_name(value: object) -> str: + if not isinstance(value, str) or not value.strip(): + raise SealedLaunchError("distribution has no canonical name") + normalized = re.sub(r"[-_.]+", "-", value.strip()).lower() + if re.fullmatch(r"[a-z0-9][a-z0-9-]*", normalized) is None: + raise SealedLaunchError("distribution name is invalid") + return normalized + + +def _parse_package_root(value: str) -> tuple[str, Path]: + name, separator, raw_path = value.partition("=") + if not separator or not raw_path: + raise argparse.ArgumentTypeError("--package-root must use name=absolute-path") + try: + canonical_name = _root_name(name, context="package root name") + except SealedLaunchError as exc: + raise argparse.ArgumentTypeError(str(exc)) from exc + path = Path(raw_path) + if not path.is_absolute(): + raise argparse.ArgumentTypeError("--package-root path must be absolute") + return canonical_name, path + + +def _runtime_roots( + base_runtime_root: Path, + package_roots: Mapping[str, Path], + manifest: Mapping[str, object], +) -> tuple[Path, dict[str, Path], dict[str, str]]: + base = _absolute_directory(base_runtime_root, context="base runtime root") + expected_roots = { + str(item["name"]): str(item["import_path"]) + for item in manifest["package_roots"] # type: ignore[union-attr] + } + if set(package_roots) != set(expected_roots): + raise SealedLaunchError("CLI package roots differ from the runtime manifest") + packages = { + name: _absolute_directory(package_roots[name], context=f"package root {name}") + for name in sorted(package_roots) + } + all_roots = [(BASE_RUNTIME_ROOT_NAME, base), *packages.items()] + for index, (left_name, left) in enumerate(all_roots): + for right_name, right in all_roots[index + 1 :]: + if left == right: + raise SealedLaunchError(f"runtime roots alias: {left_name}, {right_name}") + for outer, inner in ((left, right), (right, left)): + try: + inner.relative_to(outer) + except ValueError: + continue + raise SealedLaunchError("runtime roots must not be nested") + for name, relative in expected_roots.items(): + _safe_join( + packages[name], + relative, + context=f"package root {name} import path", + directory=True, + ) + return base, packages, expected_roots + + +def _distribution_inventory( + package_roots: Mapping[str, Path], + import_paths: Mapping[str, str], +) -> tuple[dict[str, object], ...]: + result: dict[str, dict[str, object]] = {} + for root_name in sorted(package_roots): + search_root = _safe_join( + package_roots[root_name], + import_paths[root_name], + context=f"package root {root_name} import path", + directory=True, + ) + for distribution in importlib.metadata.distributions(path=[str(search_root)]): + name = _canonical_distribution_name(distribution.metadata.get("Name")) + if name in result: + raise SealedLaunchError("staged package roots contain a duplicate distribution") + raw_files = distribution.files + if raw_files is None: + raise SealedLaunchError(f"distribution {name} has no RECORD inventory") + normalized: list[str] = [] + for raw_file in raw_files: + candidate = Path(distribution.locate_file(raw_file)) + try: + resolved = candidate.resolve(strict=True) + relative = resolved.relative_to(package_roots[root_name]).as_posix() + except (OSError, ValueError) as exc: + raise SealedLaunchError( + f"distribution {name} RECORD path escapes its package tree" + ) from exc + _safe_join( + package_roots[root_name], + relative, + context=f"distribution {name} RECORD path", + ) + normalized.append( + _canonical_relative_path(relative, context="normalized RECORD path") + ) + normalized.sort() + if not normalized or len({item.casefold() for item in normalized}) != len(normalized): + raise SealedLaunchError(f"distribution {name} RECORD inventory is invalid") + record_paths = [ + item for item in normalized if item.casefold().endswith(".dist-info/record") + ] + if len(record_paths) != 1 or not distribution.read_text("RECORD"): + raise SealedLaunchError(f"distribution {name} must contain exactly one RECORD") + result[name] = { + "files": normalized, + "name": name, + "package_root": root_name, + "version": str(distribution.version), + } + if not result: + raise SealedLaunchError("staged runtime contains no distributions") + return tuple(result[name] for name in sorted(result)) + + +def _verify_runtime( + manifest: Mapping[str, object], + *, + base_runtime_root: Path, + package_roots: Mapping[str, Path], + require_current_process: bool, +) -> tuple[Path, dict[str, Path], dict[str, str], Path]: + base, packages, import_paths = _runtime_roots( + base_runtime_root, + package_roots, + manifest, + ) + tree_roots = {BASE_RUNTIME_ROOT_NAME: base, **packages} + for expected_tree in manifest["runtime_trees"]: # type: ignore[union-attr] + name = str(expected_tree["name"]) + actual = _tree_files(tree_roots[name], context=f"runtime tree {name}") + if list(actual) != expected_tree["files"]: + raise SealedLaunchError(f"runtime tree {name} differs from its frozen identity") + if list(_distribution_inventory(packages, import_paths)) != manifest["distributions"]: + raise SealedLaunchError("staged distribution identity differs from the runtime manifest") + interpreter = _safe_join( + base, + str(manifest["interpreter"]["relative_path"]), # type: ignore[index] + context="staged interpreter", + ) + interpreter_record = _stable_file_record( + interpreter, + relative=str(manifest["interpreter"]["relative_path"]), # type: ignore[index] + context="staged interpreter", + ) + if interpreter_record != { + "path": manifest["interpreter"]["relative_path"], # type: ignore[index] + "sha256": manifest["interpreter"]["sha256"], # type: ignore[index] + "size_bytes": manifest["interpreter"]["size_bytes"], # type: ignore[index] + }: + raise SealedLaunchError("staged interpreter identity drifted") + if require_current_process: + python = manifest["python"] + machine = manifest["machine"] + import struct + + if ( + platform.python_implementation() != python["implementation"] + or platform.python_version() != python["version"] + or sys.implementation.cache_tag != python["cache_tag"] + or getattr(sys, "abiflags", "") != python["abi_flags"] + or platform.system() != machine["system"] + or f"{8 * struct.calcsize('P')}bit" != machine["architecture"] + or platform.machine() != machine["machine"] + or sys.byteorder != machine["byteorder"] + or 8 * struct.calcsize("P") != machine["pointer_bits"] + or Path(sys.executable).resolve(strict=True) != interpreter + ): + raise SealedLaunchError("point-used Python or machine identity drifted") + return base, packages, import_paths, interpreter + + +def _extract_runner_options(arguments: Sequence[str]) -> dict[str, str]: + result: dict[str, str] = {} + index = 0 + while index < len(arguments): + option = arguments[index] + if option in _REQUIRED_RUNNER_OPTIONS: + if option in result or index + 1 >= len(arguments): + raise SealedLaunchError(f"runner option is duplicated or incomplete: {option}") + raw_value = arguments[index + 1] + if raw_value.startswith("--"): + raise SealedLaunchError(f"runner option has no value: {option}") + result[option] = raw_value + index += 2 + continue + index += 1 + missing = sorted(_REQUIRED_RUNNER_OPTIONS - set(result)) + if missing: + raise SealedLaunchError(f"runner arguments omit required sealed inputs: {missing}") + return result + + +def _parse_identity(data: bytes) -> dict[str, str]: + root = _strict_json(data, context="frozen identity") + _exact_fields(root, {"canonical_evidence_sha256", "evidence"}, context="frozen identity") + if _canonical_json_bytes(root) != data: + raise SealedLaunchError("frozen identity is not canonical JSON") + evidence = root["evidence"] + if not isinstance(evidence, dict): + raise SealedLaunchError("frozen identity evidence is missing") + if ( + type(evidence.get("schema_version")) is not int + or evidence.get("schema_version") != IDENTITY_SCHEMA + or evidence.get("status") != "frozen" + or evidence.get("phase") != "calibration" + or evidence.get("identity_only") is not True + or evidence.get("promotion_required") is not False + ): + raise SealedLaunchError("frozen identity state or schema drifted") + claimed = _sha256( + root["canonical_evidence_sha256"], context="identity evidence SHA-256" + ) + if claimed != _sha256_bytes(_canonical_json_bytes(evidence)): + raise SealedLaunchError("frozen identity evidence hash drifted") + bindings = evidence.get("execution_bindings") + if not isinstance(bindings, dict): + raise SealedLaunchError("frozen identity execution bindings are missing") + _exact_fields(bindings, set(_BOUND_ARTIFACT_OPTIONS), context="identity execution bindings") + return { + name: _sha256(bindings[name], context=f"identity binding {name}") + for name in sorted(bindings) + } + + +def _parse_source_manifest(data: bytes) -> dict[str, object]: + root = _strict_json(data, context="source manifest") + _exact_fields( + root, + { + "canonical_manifest_sha256", + "object_format", + "paths", + "profile", + "repository_binding", + "schema", + "source_commit", + }, + context="source manifest", + ) + if _pretty_json_bytes(root) != data: + raise SealedLaunchError("source manifest is not canonical JSON") + if root["schema"] != "recurquant.experiment013.source-manifest.v1": + raise SealedLaunchError("source manifest schema drifted") + if root["profile"] != "experiment-013-static-q468-frozen-source-v1": + raise SealedLaunchError("source manifest profile drifted") + if root["object_format"] != "sha1" or not isinstance(root["source_commit"], str): + raise SealedLaunchError("source manifest Git identity drifted") + if _SHA1_RE.fullmatch(str(root["source_commit"])) is None: + raise SealedLaunchError("source manifest commit is invalid") + payload = dict(root) + claimed = _sha256(payload.pop("canonical_manifest_sha256"), context="source self-hash") + if claimed != _sha256_bytes(_pretty_json_bytes(payload)): + raise SealedLaunchError("source manifest self-hash drifted") + raw_paths = root["paths"] + if not isinstance(raw_paths, list) or not raw_paths: + raise SealedLaunchError("source manifest has no paths") + paths: list[dict[str, object]] = [] + for raw_entry in raw_paths: + if not isinstance(raw_entry, dict): + raise SealedLaunchError("source path record must be an object") + _exact_fields( + raw_entry, + {"git_blob_oid", "index_blob_oid", "mode", "path", "raw_sha256", "worktree_blob_oid"}, + context="source path record", + ) + relative = _canonical_relative_path(raw_entry["path"], context="source path") + for oid_field in ("git_blob_oid", "index_blob_oid", "worktree_blob_oid"): + if not isinstance(raw_entry[oid_field], str) or _SHA1_RE.fullmatch( + raw_entry[oid_field] + ) is None: + raise SealedLaunchError("source Git object identity is invalid") + git_object_ids = { + raw_entry[name] + for name in ("git_blob_oid", "index_blob_oid", "worktree_blob_oid") + } + if len(git_object_ids) != 1: + raise SealedLaunchError("source Git object identities disagree") + if raw_entry["mode"] not in {"100644", "100755"}: + raise SealedLaunchError("source file mode is invalid") + paths.append( + { + "path": relative, + "raw_sha256": _sha256(raw_entry["raw_sha256"], context="source raw SHA-256"), + } + ) + rendered = [str(item["path"]) for item in paths] + if rendered != sorted(rendered) or len({item.casefold() for item in rendered}) != len(rendered): + raise SealedLaunchError("source path inventory is not unique and sorted") + if RUNNER_SOURCE_PATH not in rendered: + raise SealedLaunchError("source manifest omits the calibration runner") + return {"file_sha256": _sha256_bytes(data), "paths": paths} + + +def _verify_source(source_manifest: Mapping[str, object], repository_root: Path) -> Path: + root = _absolute_directory(repository_root, context="repository root") + runner_path: Path | None = None + for entry in source_manifest["paths"]: # type: ignore[union-attr] + relative = str(entry["path"]) + path = _safe_join(root, relative, context=f"source file {relative}") + actual = _stable_file_record(path, relative=relative, context=f"source file {relative}") + if actual["sha256"] != entry["raw_sha256"]: + raise SealedLaunchError(f"source bytes drifted: {relative}") + if relative == RUNNER_SOURCE_PATH: + runner_path = path + assert runner_path is not None + return runner_path + + +def _verify_bound_artifacts( + runner_options: Mapping[str, str], + *, + runtime_manifest_path: Path, +) -> tuple[dict[str, str], dict[str, object], Path]: + identity_path = Path(runner_options["--frozen-identity"]) + identity_bytes = identity_path.read_bytes() + bindings = _parse_identity(identity_bytes) + for binding, option in _BOUND_ARTIFACT_OPTIONS.items(): + artifact_path = Path(runner_options[option]) + try: + artifact_bytes = artifact_path.read_bytes() + except OSError as exc: + raise SealedLaunchError(f"bound artifact is unavailable: {option}") from exc + if _sha256_bytes(artifact_bytes) != bindings[binding]: + raise SealedLaunchError(f"identity binding mismatch: {option}") + for binding, option in _EXPECTED_BOUND_DIGEST_OPTIONS.items(): + expected = _sha256(runner_options[option], context=f"runner option {option}") + if expected != bindings[binding]: + raise SealedLaunchError(f"runner digest binding mismatch: {option}") + try: + if runtime_manifest_path.resolve(strict=True) != Path( + runner_options["--runtime-manifest"] + ).resolve(strict=True): + raise SealedLaunchError("host and runner runtime-manifest paths differ") + except OSError as exc: + raise SealedLaunchError("runtime manifest path is unavailable") from exc + source_bytes = Path(runner_options["--repository-source-manifest"]).read_bytes() + source_manifest = _parse_source_manifest(source_bytes) + runner_path = _verify_source( + source_manifest, + Path(runner_options["--repository-root"]), + ) + return bindings, source_manifest, runner_path + + +def _verify_empty_pycache(path: Path) -> Path: + root = _absolute_directory(path, context="pycache prefix") + try: + if any(os.scandir(root)): + raise SealedLaunchError("pycache prefix is not empty") + except OSError as exc: + raise SealedLaunchError("cannot enumerate pycache prefix") from exc + return root + + +def _sealed_environment() -> dict[str, str]: + return { + key: value + for key, value in os.environ.items() + if not key.upper().startswith("PYTHON") + and key.upper() not in {"VIRTUAL_ENV", "VIRTUAL_ENV_PROMPT"} + } + + +def _sealed_argv( + *, + interpreter: Path, + runtime_manifest: Path, + base_runtime_root: Path, + package_roots: Mapping[str, Path], + pycache_prefix: Path, + runner_arguments: Sequence[str], +) -> list[str]: + serialized_roots = json.dumps( + {name: str(package_roots[name]) for name in sorted(package_roots)}, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + return [ + str(interpreter), + "-I", + "-S", + "-B", + "-X", + f"pycache_prefix={pycache_prefix}", + "-X", + "utf8", + "-c", + SEALED_BOOTSTRAP, + str(runtime_manifest), + str(base_runtime_root), + serialized_roots, + str(pycache_prefix), + *runner_arguments, + ] + + +# This is intentionally standalone. The child starts with no repository or +# package path and cannot import this host module safely. Keep its verification +# semantics aligned with the host functions above and test both boundaries. +SEALED_BOOTSTRAP: Final = r''' +import sys as _s +_sensitive = { + "site", "_virtualenv", "recurquant", + "recurquant.experiment013_calibration_api", + "recurquant.experiment013_qwen35_adapter", + "recurquant.experiment013_source", + "recurquant_experiment013_identity_resolver", + "_recurquant_experiment013_sealed_runner", +} +if _sensitive.intersection(_s.modules): + raise RuntimeError("sealed bootstrap found a preloaded sensitive module") +if ( + _s.flags.isolated != 1 + or _s.flags.ignore_environment != 1 + or _s.flags.no_user_site != 1 + or _s.flags.no_site != 1 + or _s.flags.dont_write_bytecode != 1 + or _s.flags.safe_path is not True + or _s.flags.utf8_mode != 1 +): + raise RuntimeError("sealed bootstrap startup flags drifted") +if ( + set(_s._xoptions) != {"pycache_prefix", "utf8"} + or _s._xoptions.get("utf8") is not True + or not isinstance(_s._xoptions.get("pycache_prefix"), str) +): + raise RuntimeError("sealed bootstrap -X options drifted") + +import hashlib as _h +import importlib.metadata as _md +import json as _j +import os as _o +import pathlib as _p +import platform as _platform +import re as _re +import stat as _stat +import struct as _struct +import types as _types + +_rp = 0x400 +_sha = _re.compile(r"[0-9a-f]{64}") +_root_re = _re.compile(r"[a-z][a-z0-9-]{0,63}") +_bad_suffix = {".egg-link", ".pth", ".pyc", ".pyo", "._pth"} +_bad_dir = {"__pycache__"} +_bad_name = {"pyvenv.cfg", "sitecustomize.py", "usercustomize.py"} +_reserved = { + "aux", "clock$", "con", "nul", "prn", + *{"com" + str(i) for i in range(1, 10)}, + *{"lpt" + str(i) for i in range(1, 10)}, +} +_policy = { + "bootstrap_mode": "stdlib-only-exact-runner-v1", + "dont_write_bytecode": 1, + "ignore_environment": 1, + "isolated": 1, + "no_site": 1, + "no_user_site": 1, + "package_path_mode": "authenticated-record-only-roots-v1", + "pycache_mode": "new-verified-empty-prefix-v1", + "safe_path": True, + "site_loaded": False, + "sys_path_mode": "staged-base-then-authenticated-packages-v1", + "utf8_mode": 1, + "virtualenv_hook_loaded": False, +} +_binding_options = { + "calibration_runtime_manifest_file_sha256": "--runtime-manifest", + "model_file_manifest_file_sha256": "--model-file-manifest", + "parquet_materialization_manifest_file_sha256": "--parquet-materialization-manifest", + "repository_source_manifest_file_sha256": "--repository-source-manifest", +} +_expected_digest_options = { + "calibration_runtime_manifest_file_sha256": "--expected-runtime-manifest-sha256", + "model_file_manifest_file_sha256": "--expected-model-file-manifest-sha256", + "parquet_materialization_manifest_file_sha256": + "--expected-parquet-materialization-manifest-sha256", +} + +def _fail(message): + raise RuntimeError(message) + +def _canonical(value): + return (_j.dumps(value, ensure_ascii=False, allow_nan=False, sort_keys=True, + separators=(",", ":")) + "\n").encode("utf-8") + +def _pretty(value): + return (_j.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n").encode("utf-8") + +def _json(data, context): + def unique(pairs): + out = {} + for key, value in pairs: + if key in out: + _fail(context + " contains a duplicate JSON key") + out[key] = value + return out + def constant(value): + _fail(context + " contains a non-finite JSON constant") + try: + value = _j.loads(data.decode("utf-8"), object_pairs_hook=unique, + parse_constant=constant) + except (UnicodeDecodeError, _j.JSONDecodeError) as error: + raise RuntimeError(context + " is not strict UTF-8 JSON") from error + if not isinstance(value, dict): + _fail(context + " must be an object") + return value + +def _fields(value, expected, context): + if set(value) != set(expected): + _fail(context + " fields differ from the frozen schema") + +def _typed(value, expected, context): + if not isinstance(value, dict) or set(value) != set(expected): + _fail(context + " fields differ from the frozen schema") + if any(type(value[key]) is not type(expected[key]) or value[key] != expected[key] + for key in expected): + _fail(context + " value or JSON type drifted") + +def _digest(value, context): + if not isinstance(value, str) or _sha.fullmatch(value) is None: + _fail(context + " is not a SHA-256 digest") + return value + +def _positive(value, context): + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + _fail(context + " is not a positive integer") + return value + +def _nonnegative(value, context): + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + _fail(context + " is not a non-negative integer") + return value + +def _relative(value, context): + if not isinstance(value, str) or not value or value != value.strip(): + _fail(context + " is not a canonical relative path") + if any(c in value for c in ("\\", "\0", "\n", "\r", ":")): + _fail(context + " is not a safe POSIX path") + path = _p.PurePosixPath(value) + if path.is_absolute() or path.as_posix() != value or any( + part in {"", ".", ".."} for part in path.parts + ): + _fail(context + " is not a canonical relative path") + for part in path.parts: + if part.endswith((" ", ".")) or part.split(".", 1)[0].casefold() in _reserved: + _fail(context + " is unsafe on Windows") + return value + +def _link(path): + try: + status = path.lstat() + except OSError as error: + raise RuntimeError("required path is unavailable") from error + return path.is_symlink() or bool(getattr(status, "st_file_attributes", 0) & _rp) + +def _directory(raw, context): + path = _p.Path(_o.path.abspath(raw)) + if _link(path): + _fail(context + " is a link or reparse point") + try: + result = path.resolve(strict=True) + except OSError as error: + raise RuntimeError(context + " is unavailable") from error + if not result.is_dir(): + _fail(context + " is not a directory") + return result + +def _join(root, relative, context, directory=False): + path = root + for part in _p.PurePosixPath(_relative(relative, context)).parts: + path /= part + if _link(path): + _fail(context + " traverses a link or reparse point") + try: + result = path.resolve(strict=True) + result.relative_to(root) + except (OSError, ValueError) as error: + raise RuntimeError(context + " escapes its authenticated root") from error + if directory and not result.is_dir(): + _fail(context + " is not a directory") + if not directory and not result.is_file(): + _fail(context + " is not a regular file") + return result + +def _forbidden(relative): + path = _p.PurePosixPath(relative) + return (any(part.casefold() in _bad_dir for part in path.parts) + or path.name.casefold() in _bad_name + or path.suffix.casefold() in _bad_suffix) + +def _file(path, relative, context): + before = path.stat() + if not _stat.S_ISREG(before.st_mode) or _link(path): + _fail(context + " is not a stable regular file") + digest = _h.sha256() + size = 0 + with path.open("rb") as handle: + while True: + chunk = handle.read(1024 * 1024) + if not chunk: + break + digest.update(chunk) + size += len(chunk) + after = path.stat() + if (before.st_size != after.st_size or before.st_mtime_ns != after.st_mtime_ns + or size != after.st_size or _link(path)): + _fail(context + " changed during authentication") + return {"path": relative, "sha256": digest.hexdigest(), "size_bytes": size} + +def _tree(root, context): + root = _directory(root, context + " root") + stack = [(root, ())] + files = [] + folded = set() + while stack: + directory, parents = stack.pop() + entries = sorted(_o.scandir(directory), key=lambda item: item.name.casefold()) + for entry in entries: + relative = _relative(_p.PurePosixPath(*parents, entry.name).as_posix(), + context + " path") + status = entry.stat(follow_symlinks=False) + if entry.is_symlink() or bool(getattr(status, "st_file_attributes", 0) & _rp): + _fail(context + " contains a link or reparse point") + if _stat.S_ISDIR(status.st_mode): + if entry.name.casefold() in _bad_dir: + _fail(context + " contains a forbidden cache directory") + stack.append((_p.Path(entry.path), (*parents, entry.name))) + continue + if not _stat.S_ISREG(status.st_mode) or _forbidden(relative): + _fail(context + " contains a forbidden or non-regular file") + if relative.casefold() in folded: + _fail(context + " contains a case-insensitive path collision") + folded.add(relative.casefold()) + files.append(_file(_p.Path(entry.path), relative, context + " file")) + files.sort(key=lambda item: item["path"]) + if not files: + _fail(context + " has no files") + return files + +def _name(value): + if not isinstance(value, str) or not value.strip(): + _fail("distribution has no canonical name") + result = _re.sub(r"[-_.]+", "-", value.strip()).lower() + if _re.fullmatch(r"[a-z0-9][a-z0-9-]*", result) is None: + _fail("distribution name is invalid") + return result + +def _options(arguments): + required = {"--frozen-identity", "--repository-root", *_binding_options.values(), + *_expected_digest_options.values()} + result = {} + index = 0 + while index < len(arguments): + item = arguments[index] + if item in required: + if ( + item in result + or index + 1 >= len(arguments) + or arguments[index + 1].startswith("--") + ): + _fail("runner option is duplicated or incomplete: " + item) + result[item] = arguments[index + 1] + index += 2 + else: + index += 1 + if set(result) != required: + _fail("runner arguments omit required sealed inputs") + return result + +def _identity(data): + root = _json(data, "frozen identity") + _fields(root, {"canonical_evidence_sha256", "evidence"}, "frozen identity") + if _canonical(root) != data: + _fail("frozen identity is not canonical JSON") + evidence = root["evidence"] + if (not isinstance(evidence, dict) or type(evidence.get("schema_version")) is not int + or evidence.get("schema_version") != 4 + or evidence.get("status") != "frozen" or evidence.get("phase") != "calibration" + or evidence.get("identity_only") is not True + or evidence.get("promotion_required") is not False): + _fail("frozen identity state or schema drifted") + if _digest(root["canonical_evidence_sha256"], "identity evidence hash") != _h.sha256( + _canonical(evidence) + ).hexdigest(): + _fail("frozen identity evidence hash drifted") + bindings = evidence.get("execution_bindings") + if not isinstance(bindings, dict): + _fail("frozen identity bindings are missing") + _fields(bindings, set(_binding_options), "identity bindings") + return {key: _digest(value, "identity binding") for key, value in bindings.items()} + +def _source(data): + root = _json(data, "source manifest") + _fields(root, {"canonical_manifest_sha256", "object_format", "paths", "profile", + "repository_binding", "schema", "source_commit"}, "source manifest") + if _pretty(root) != data: + _fail("source manifest is not canonical JSON") + payload = dict(root) + claimed = _digest(payload.pop("canonical_manifest_sha256"), "source self-hash") + if claimed != _h.sha256(_pretty(payload)).hexdigest(): + _fail("source manifest self-hash drifted") + paths = [] + for entry in root["paths"]: + _fields(entry, {"git_blob_oid", "index_blob_oid", "mode", "path", "raw_sha256", + "worktree_blob_oid"}, "source path") + paths.append({"path": _relative(entry["path"], "source path"), + "raw_sha256": _digest(entry["raw_sha256"], "source SHA-256")}) + rendered = [item["path"] for item in paths] + if rendered != sorted(rendered) or "scripts/run_static_q468_calibration.py" not in rendered: + _fail("source path inventory drifted") + return {"file_sha256": _h.sha256(data).hexdigest(), "paths": paths} + +def _verify_source(manifest, root): + root = _directory(root, "repository root") + runner = None + for entry in manifest["paths"]: + path = _join(root, entry["path"], "source file") + record = _file(path, entry["path"], "source file") + if record["sha256"] != entry["raw_sha256"]: + _fail("source bytes drifted") + if entry["path"] == "scripts/run_static_q468_calibration.py": + runner = path + if runner is None: + _fail("source manifest omitted runner") + return runner + +def _distributions(packages, imports): + found = {} + for root_name in sorted(packages): + search = _join(packages[root_name], imports[root_name], "package import path", True) + for distribution in _md.distributions(path=[str(search)]): + name = _name(distribution.metadata.get("Name")) + if name in found or distribution.files is None: + _fail("staged distribution inventory is invalid") + files = [] + for item in distribution.files: + candidate = _p.Path(distribution.locate_file(item)).resolve(strict=True) + try: + relative = candidate.relative_to(packages[root_name]).as_posix() + except ValueError as error: + raise RuntimeError("RECORD path escapes package tree") from error + _join(packages[root_name], relative, "RECORD path") + files.append(_relative(relative, "normalized RECORD path")) + files.sort() + if len([item for item in files if item.casefold().endswith(".dist-info/record")]) != 1: + _fail("distribution must contain exactly one RECORD") + found[name] = {"files": files, "name": name, "package_root": root_name, + "version": str(distribution.version)} + return [found[name] for name in sorted(found)] + +def _manifest(data): + root = _json(data, "runtime manifest") + _fields(root, {"artifact_kind", "base_runtime_root", "base_sys_path", "distributions", + "interpreter", "launch_policy", "machine", "package_roots", "python", + "runtime_trees", "schema_version"}, "runtime manifest") + if _canonical(root) != data or root["artifact_kind"] != ( + "recurquant_experiment013_calibration_runtime_manifest" + ) or type(root["schema_version"]) is not int or root["schema_version"] != 3: + _fail("runtime manifest identity or policy drifted") + _typed(root["launch_policy"], _policy, "runtime launch policy") + if root["base_runtime_root"] != "base-runtime": + _fail("base runtime name drifted") + + python = root["python"] + if not isinstance(python, dict): + _fail("runtime Python record is invalid") + _fields(python, {"abi_flags", "cache_tag", "implementation", "version"}, + "runtime Python record") + for field in ("abi_flags", "cache_tag", "implementation", "version"): + value = python[field] + if (not isinstance(value, str) or value != value.strip() + or (field != "abi_flags" and not value)): + _fail("runtime Python identity is invalid") + + machine = root["machine"] + if not isinstance(machine, dict): + _fail("runtime machine record is invalid") + _fields(machine, {"architecture", "byteorder", "machine", "pointer_bits", "system"}, + "runtime machine record") + for field in ("architecture", "machine", "system"): + value = machine[field] + if not isinstance(value, str) or not value or value != value.strip(): + _fail("runtime machine identity is invalid") + if machine["byteorder"] not in {"big", "little"}: + _fail("runtime machine byte order is invalid") + _positive(machine["pointer_bits"], "runtime pointer bits") + + raw_roots = root["package_roots"] + if not isinstance(raw_roots, list) or not raw_roots: + _fail("runtime package roots are invalid") + roots = [] + for item in raw_roots: + if not isinstance(item, dict): + _fail("package root is not an object") + _fields(item, {"import_path", "name"}, "package root") + name = item["name"] + if not isinstance(name, str) or _root_re.fullmatch(name) is None: + _fail("package root name is invalid") + roots.append({"name": name, "import_path": _relative(item["import_path"], "import path")}) + names = [item["name"] for item in roots] + if (names != sorted(names) or len(set(names)) != len(names) + or "base-runtime" in names): + _fail("package roots drifted") + + raw_base_paths = root["base_sys_path"] + if not isinstance(raw_base_paths, list) or not raw_base_paths: + _fail("base sys.path is invalid") + base_paths = [_relative(item, "base sys.path") for item in raw_base_paths] + if len({item.casefold() for item in base_paths}) != len(base_paths): + _fail("base sys.path entries collide") + + trees = root["runtime_trees"] + if not isinstance(trees, list) or len(trees) != len(roots) + 1: + _fail("runtime tree count drifted") + expected_names = ["base-runtime", *names] + normalized_trees = [] + for index, tree in enumerate(trees): + if not isinstance(tree, dict): + _fail("runtime tree is not an object") + _fields(tree, {"files", "kind", "name"}, "runtime tree") + if tree["name"] != expected_names[index] or tree["kind"] != ( + "base-runtime" if index == 0 else "packages" + ): + _fail("runtime tree order drifted") + if not isinstance(tree["files"], list) or not tree["files"]: + _fail("runtime tree file inventory is invalid") + files = [] + for item in tree["files"]: + if not isinstance(item, dict): + _fail("runtime file is not an object") + _fields(item, {"path", "sha256", "size_bytes"}, "runtime file") + relative = _relative(item["path"], "runtime file") + if _forbidden(relative): + _fail("runtime manifest contains forbidden file") + files.append({"path": relative, "sha256": _digest(item["sha256"], "file hash"), + "size_bytes": _nonnegative(item["size_bytes"], "file size")}) + paths = [item["path"] for item in files] + if paths != sorted(paths) or len({item.casefold() for item in paths}) != len(paths): + _fail("runtime file inventory is not unique and sorted") + normalized_trees.append({"name": tree["name"], "kind": tree["kind"], "files": files}) + + interpreter = root["interpreter"] + if not isinstance(interpreter, dict): + _fail("runtime interpreter is not an object") + _fields(interpreter, {"relative_path", "root", "sha256", "size_bytes"}, "interpreter") + normalized_interpreter = { + "relative_path": _relative(interpreter["relative_path"], "interpreter path"), + "root": interpreter["root"], + "sha256": _digest(interpreter["sha256"], "interpreter hash"), + "size_bytes": _positive(interpreter["size_bytes"], "interpreter size"), + } + if normalized_interpreter["root"] != "base-runtime": + _fail("runtime interpreter root drifted") + base_files = {item["path"]: item for item in normalized_trees[0]["files"]} + expected_interpreter = { + "path": normalized_interpreter["relative_path"], + "sha256": normalized_interpreter["sha256"], + "size_bytes": normalized_interpreter["size_bytes"], + } + if base_files.get(normalized_interpreter["relative_path"]) != expected_interpreter: + _fail("runtime interpreter differs from the base tree") + for entry in base_paths: + present = entry in base_files or any(path.startswith(entry + "/") for path in base_files) + if not present and _re.fullmatch(r"python[0-9]+\.zip", entry) is None: + _fail("base sys.path entry is absent from the base tree") + + raw_distributions = root["distributions"] + if not isinstance(raw_distributions, list) or not raw_distributions: + _fail("runtime distributions are invalid") + distributions = [] + ownership = {name: set() for name in names} + for item in raw_distributions: + if not isinstance(item, dict): + _fail("runtime distribution is not an object") + _fields(item, {"files", "name", "package_root", "version"}, + "runtime distribution") + name = _name(item["name"]) + package_root = item["package_root"] + version = item["version"] + if (name != item["name"] or package_root not in ownership + or not isinstance(version, str) or not version + or version != version.strip()): + _fail("runtime distribution identity is invalid") + raw_files = item["files"] + if not isinstance(raw_files, list) or not raw_files: + _fail("runtime distribution files are invalid") + files = [_relative(path, "distribution RECORD path") for path in raw_files] + if files != sorted(files) or len({path.casefold() for path in files}) != len(files): + _fail("distribution RECORD paths are not unique and sorted") + if ownership[package_root].intersection(files): + _fail("runtime distributions have overlapping RECORD ownership") + ownership[package_root].update(files) + distributions.append({"files": files, "name": name, + "package_root": package_root, "version": version}) + distribution_names = [item["name"] for item in distributions] + if (distribution_names != sorted(distribution_names) + or len(set(distribution_names)) != len(distribution_names)): + _fail("runtime distributions are not unique and sorted") + tree_by_name = {item["name"]: item for item in normalized_trees} + for name in names: + if ownership[name] != {item["path"] for item in tree_by_name[name]["files"]}: + _fail("package tree differs from exact RECORD ownership") + + return {"base_sys_path": base_paths, "distributions": distributions, + "file_sha256": _h.sha256(data).hexdigest(), + "interpreter": normalized_interpreter, + "machine": machine, "package_roots": roots, "python": python, + "runtime_trees": normalized_trees} + +def _verify_runtime(manifest, base_raw, package_raw, packages_appended=False): + base = _directory(base_raw, "base runtime") + declared = {item["name"]: item["import_path"] for item in manifest["package_roots"]} + if set(package_raw) != set(declared): + _fail("CLI package roots differ from manifest") + packages = {name: _directory(package_raw[name], "package root") for name in sorted(package_raw)} + tree_roots = {"base-runtime": base, **packages} + for tree in manifest["runtime_trees"]: + if _tree(tree_roots[tree["name"]], "runtime tree") != tree["files"]: + _fail("complete runtime tree identity drifted") + if _distributions(packages, declared) != manifest["distributions"]: + _fail("distribution RECORD identity drifted") + interpreter = _join(base, manifest["interpreter"]["relative_path"], "interpreter") + if _file( + interpreter, + manifest["interpreter"]["relative_path"], + "interpreter", + ) != { + "path": manifest["interpreter"]["relative_path"], + "sha256": manifest["interpreter"]["sha256"], + "size_bytes": manifest["interpreter"]["size_bytes"], + }: + _fail("point-used interpreter identity drifted") + if _p.Path(_s.executable).resolve(strict=True) != interpreter: + _fail("point-used interpreter path drifted") + python = manifest["python"] + machine = manifest["machine"] + if (_platform.python_implementation() != python["implementation"] + or _platform.python_version() != python["version"] + or _s.implementation.cache_tag != python["cache_tag"] + or getattr(_s, "abiflags", "") != python["abi_flags"] + or _platform.system() != machine["system"] + or f"{8 * _struct.calcsize('P')}bit" != machine["architecture"] + or _platform.machine() != machine["machine"] + or _s.byteorder != machine["byteorder"] + or 8 * _struct.calcsize("P") != machine["pointer_bits"]): + _fail("point-used Python or machine identity drifted") + imports = {name: str(_join(packages[name], declared[name], "import path", True)) + for name in sorted(packages)} + expected_path = [str(base / _p.PurePosixPath(item)) + for item in manifest["base_sys_path"]] + if packages_appended: + expected_path.extend(imports[name] for name in sorted(imports)) + if [_o.path.normcase(_o.path.abspath(item)) for item in _s.path] != [ + _o.path.normcase(_o.path.abspath(item)) for item in expected_path + ]: + _fail("authenticated sys.path differs from the frozen runtime") + return base, packages, declared, imports, interpreter + +_runtime_path = _p.Path(_s.argv[1]) +_base_raw = _p.Path(_s.argv[2]) +try: + _package_raw = {name: _p.Path(path) for name, path in _j.loads(_s.argv[3]).items()} +except Exception as error: + raise RuntimeError("package-root bootstrap argument is invalid") from error +_pycache = _directory(_s.argv[4], "pycache prefix") +if any(_pycache.iterdir()) or _s.pycache_prefix is None or _p.Path( + _s.pycache_prefix +).resolve(strict=True) != _pycache: + _fail("sealed pycache prefix is not exact and empty") +_runner_args = list(_s.argv[5:]) +_runner_options = _options(_runner_args) +_runtime_bytes = _runtime_path.read_bytes() +_runtime = _manifest(_runtime_bytes) +_bindings = _identity(_p.Path(_runner_options["--frozen-identity"]).read_bytes()) +for _binding, _option in _binding_options.items(): + if _h.sha256(_p.Path(_runner_options[_option]).read_bytes()).hexdigest() != _bindings[_binding]: + _fail("identity binding mismatch: " + _option) +for _binding, _option in _expected_digest_options.items(): + if _digest(_runner_options[_option], "runner digest binding") != _bindings[_binding]: + _fail("runner digest binding mismatch: " + _option) +if _runtime_path.resolve(strict=True) != _p.Path( + _runner_options["--runtime-manifest"] +).resolve(strict=True): + _fail("host and runner runtime manifests differ") +_source_manifest = _source( + _p.Path(_runner_options["--repository-source-manifest"]).read_bytes() +) +_repository_root = _p.Path(_runner_options["--repository-root"]) +_runner_path = _verify_source(_source_manifest, _repository_root) +_base, _packages, _import_rel, _imports, _interpreter = _verify_runtime( + _runtime, _base_raw, _package_raw +) +_s.path.extend(_imports[name] for name in sorted(_imports)) +if _s.path != [str(_base / _p.PurePosixPath(item)) for item in _runtime["base_sys_path"]] + [ + _imports[name] for name in sorted(_imports) +]: + _fail("authenticated package paths were not appended exactly") +if _sensitive.intersection(_s.modules): + _fail("sensitive module appeared before exact runner load") +_before = _file(_runner_path, "scripts/run_static_q468_calibration.py", "runner source") +_payload = _runner_path.read_bytes() +if _h.sha256(_payload).hexdigest() != _before["sha256"]: + _fail("runner source changed before compile") +_module = _types.ModuleType("_recurquant_experiment013_sealed_runner") +_module.__file__ = str(_runner_path) +_module.__package__ = "" +_s.modules[_module.__name__] = _module +try: + try: + _code = compile(_payload, str(_runner_path), "exec", dont_inherit=True) + exec(_code, _module.__dict__) + if _file( + _runner_path, + "scripts/run_static_q468_calibration.py", + "runner source", + ) != _before: + _fail("runner source changed during exact load") + _sealed_main = getattr(_module, "sealed_main", None) + if not callable(_sealed_main): + _fail("authenticated runner has no sealed_main entrypoint") + _result = _sealed_main( + _runner_args, + base_runtime_root=_base, + package_roots=_packages, + package_import_paths=_import_rel, + interpreter_path=_interpreter, + pycache_prefix=_pycache, + ) + finally: + if any(_pycache.iterdir()): + _fail("sealed pycache prefix changed during calibration") + _verify_source(_source_manifest, _repository_root) + _verify_runtime(_runtime, _base, _packages, packages_appended=True) +finally: + _s.modules.pop("_recurquant_experiment013_sealed_runner", None) +if not isinstance(_result, int) or isinstance(_result, bool): + _fail("sealed_main returned a non-integer status") +raise SystemExit(_result) +'''.strip() + + +def _split_host_and_runner_args(argv: Sequence[str]) -> tuple[list[str], list[str]]: + positions = [index for index, value in enumerate(argv) if value == "--"] + if len(positions) != 1: + raise SealedLaunchError("launcher requires exactly one -- separator") + separator = positions[0] + host = list(argv[:separator]) + runner = list(argv[separator + 1 :]) + if not runner: + raise SealedLaunchError("launcher requires exact runner arguments after --") + return host, runner + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-runtime-root", required=True, type=Path) + parser.add_argument("--package-root", required=True, action="append", type=_parse_package_root) + parser.add_argument("--runtime-manifest", required=True, type=Path) + return parser + + +def launch(argv: Sequence[str]) -> int: + if list(argv) in (["-h"], ["--help"]): + _parser().print_help() + print("\nAppend -- followed by the exact run_static_q468_calibration.py arguments.") + return 0 + host_arguments, runner_arguments = _split_host_and_runner_args(argv) + args = _parser().parse_args(host_arguments) + package_roots: dict[str, Path] = {} + for name, path in args.package_root: + if name in package_roots or name == BASE_RUNTIME_ROOT_NAME: + raise SealedLaunchError(f"duplicate or reserved package root: {name}") + package_roots[name] = path + + try: + runtime_bytes = args.runtime_manifest.read_bytes() + except OSError as exc: + raise SealedLaunchError("runtime manifest is unavailable") from exc + runtime_manifest = _parse_runtime_manifest(runtime_bytes) + runner_options = _extract_runner_options(runner_arguments) + _verify_bound_artifacts( + runner_options, + runtime_manifest_path=args.runtime_manifest, + ) + base, packages, _import_paths, interpreter = _verify_runtime( + runtime_manifest, + base_runtime_root=args.base_runtime_root, + package_roots=package_roots, + require_current_process=False, + ) + + pycache_parent = Path(tempfile.mkdtemp(prefix="recurquant-exp013-sealed-pycache-")) + pycache = _verify_empty_pycache(pycache_parent) + try: + command = _sealed_argv( + interpreter=interpreter, + runtime_manifest=args.runtime_manifest.resolve(strict=True), + base_runtime_root=base, + package_roots=packages, + pycache_prefix=pycache, + runner_arguments=runner_arguments, + ) + completed = subprocess.run( + command, + check=False, + cwd=base, + env=_sealed_environment(), + ) + _verify_empty_pycache(pycache) + _verify_bound_artifacts( + runner_options, + runtime_manifest_path=args.runtime_manifest, + ) + _verify_runtime( + runtime_manifest, + base_runtime_root=base, + package_roots=packages, + require_current_process=False, + ) + return int(completed.returncode) + finally: + _verify_empty_pycache(pycache) + shutil.rmtree(pycache, ignore_errors=False) + + +def main(argv: Sequence[str] | None = None) -> int: + return launch(list(sys.argv[1:] if argv is None else argv)) + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/scripts/resolve_static_q468_identity.py b/scripts/resolve_static_q468_identity.py index fa5f68f..7f28f54 100644 --- a/scripts/resolve_static_q468_identity.py +++ b/scripts/resolve_static_q468_identity.py @@ -29,11 +29,14 @@ from pathlib import Path from typing import Any, Final -INPUT_SCHEMA: Final = "recurquant.experiment013.identity-input.v2" -CANDIDATE_SCHEMA: Final = "recurquant.experiment013.identity-candidate.v2" -FROZEN_SCHEMA: Final = "recurquant.experiment013.identity-frozen.v2" +INPUT_SCHEMA: Final = "recurquant.experiment013.identity-input.v4" +CANDIDATE_SCHEMA: Final = "recurquant.experiment013.identity-candidate.v4" +FROZEN_SCHEMA: Final = "recurquant.experiment013.identity-frozen.v4" ARTIFACT_KIND: Final = "recurquant_static_rht_q468_identity" -RESOLVER_VERSION: Final = 2 +RESOLVER_VERSION: Final = 4 +PARQUET_MATERIALIZATION_MANIFEST_FILE_SHA256: Final = ( + "ee5628e50e5d3516fd79077542d355fd915455ac0e53128d372f4177ad63d39c" +) PRIMARY_MODEL_ID: Final = "Qwen/Qwen3.5-0.8B-Base" PRIMARY_MODEL_REVISION: Final = "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68" @@ -126,7 +129,8 @@ ) CLAIM_BOUNDARY: Final = ( - "This artifact freezes Experiment 013 data and tokenizer identity only. " + "This artifact freezes Experiment 013 data, tokenizer, source, runtime, and " + "metadata-only model-file identity. " "It is not quality, latency, novelty, state-of-the-art, deployment, or " "breakthrough evidence." ) @@ -194,6 +198,14 @@ "static_k29334_policy_file_sha256", } ) +EXECUTION_BINDING_FIELDS: Final = frozenset( + { + "repository_source_manifest_file_sha256", + "calibration_runtime_manifest_file_sha256", + "model_file_manifest_file_sha256", + "parquet_materialization_manifest_file_sha256", + } +) FROZEN_EVIDENCE_FIELDS: Final = frozenset( { "schema_version", @@ -205,6 +217,7 @@ "identity_only", "claim_boundary", "source_manifest_sha256", + "execution_bindings", "model_contracts", "datasets", "upstream_tool_contracts", @@ -957,6 +970,19 @@ def _validate_calibration_binding(value: object) -> dict[str, str]: } +def _validate_execution_bindings(value: object) -> dict[str, str]: + bindings = require_mapping(value, context="execution_bindings") + require_exact_fields( + bindings, + EXECUTION_BINDING_FIELDS, + context="execution_bindings", + ) + return { + key: require_sha256(bindings[key], context=f"execution_bindings.{key}") + for key in sorted(EXECUTION_BINDING_FIELDS) + } + + def build_candidate( source: Mapping[str, Any], *, @@ -972,6 +998,7 @@ def build_candidate( "datasets", "tokenizer", "records", + "execution_bindings", "model_weights_loaded", } if phase == "stage_a": @@ -981,7 +1008,7 @@ def build_candidate( raise ValueError("identity input schema drifted") if phase not in ALLOWED_PHASES: if phase in PROTECTED_STAGES: - raise PermissionError(f"{phase} is protected and unavailable in resolver v2") + raise PermissionError(f"{phase} is protected and unavailable in resolver v4") raise ValueError(f"unsupported identity phase: {phase!r}") if source["model_weights_loaded"] is not False: raise ValueError("identity resolution must occur before model weights") @@ -1004,6 +1031,15 @@ def build_candidate( raise ValueError("CLI dataset revisions do not match the frozen upstream commits") datasets = _validate_dataset_contracts(source["datasets"], expected_revisions=revisions) tokenizer = _validate_tokenizer(source["tokenizer"]) + execution_bindings = _validate_execution_bindings(source["execution_bindings"]) + parquet_materialization_manifest_file_sha256 = execution_bindings[ + "parquet_materialization_manifest_file_sha256" + ] + if ( + parquet_materialization_manifest_file_sha256 + != PARQUET_MATERIALIZATION_MANIFEST_FILE_SHA256 + ): + raise ValueError("Parquet materialization manifest file SHA-256 drifted") raw_records = require_sequence(source["records"], context="records") records = [ _normalize_record( @@ -1029,7 +1065,7 @@ def build_candidate( content_manifest_hash = sha256_bytes(canonical_json_bytes(records)) source_hash = sha256_bytes(canonical_json_bytes(source)) evidence: dict[str, Any] = { - "schema_version": 2, + "schema_version": 4, "artifact_kind": ARTIFACT_KIND, "identity_schema": CANDIDATE_SCHEMA, "resolver_version": RESOLVER_VERSION, @@ -1038,6 +1074,7 @@ def build_candidate( "identity_only": True, "claim_boundary": CLAIM_BOUNDARY, "source_manifest_sha256": source_hash, + "execution_bindings": execution_bindings, "model_contracts": { "primary": {"id": PRIMARY_MODEL_ID, "revision": PRIMARY_MODEL_REVISION}, "conditional_scale_check": { @@ -1099,7 +1136,7 @@ def validate_candidate_artifact(artifact: Mapping[str, Any]) -> None: raise ValueError("candidate canonical evidence SHA-256 drifted") phase = evidence["phase"] exact_scalars = { - "schema_version": 2, + "schema_version": 4, "artifact_kind": ARTIFACT_KIND, "identity_schema": CANDIDATE_SCHEMA, "resolver_version": RESOLVER_VERSION, @@ -1109,7 +1146,10 @@ def validate_candidate_artifact(artifact: Mapping[str, Any]) -> None: "promotion_required": True, } for name, expected in exact_scalars.items(): - if evidence[name] != expected: + if ( + isinstance(expected, (bool, int)) + and type(evidence[name]) is not type(expected) + ) or evidence[name] != expected: raise ValueError(f"candidate {name} drifted") if phase not in ALLOWED_PHASES: raise ValueError("candidate phase drifted") @@ -1117,6 +1157,14 @@ def validate_candidate_artifact(artifact: Mapping[str, Any]) -> None: evidence["source_manifest_sha256"], context="candidate source manifest SHA-256", ) + execution_bindings = _validate_execution_bindings(evidence["execution_bindings"]) + if dict(evidence["execution_bindings"]) != execution_bindings: + raise ValueError("candidate execution bindings are not canonical") + if ( + execution_bindings["parquet_materialization_manifest_file_sha256"] + != PARQUET_MATERIALIZATION_MANIFEST_FILE_SHA256 + ): + raise ValueError("candidate Parquet materialization manifest file SHA-256 drifted") models = require_mapping(evidence["model_contracts"], context="model contracts") require_exact_fields( @@ -1229,11 +1277,15 @@ def validate_candidate_artifact(artifact: Mapping[str, Any]) -> None: raise ValueError("candidate calibration binding is not canonical") protected = require_mapping(evidence["protected_identity"], context="protected identity") - if protected != { - "stage_b_read": False, - "stage_c_read": False, - "ordinary_tests_may_read_protected_content": False, - }: + protected_fields = frozenset( + { + "stage_b_read", + "stage_c_read", + "ordinary_tests_may_read_protected_content", + } + ) + require_exact_fields(protected, protected_fields, context="protected identity") + if any(protected[field] is not False for field in protected_fields): raise ValueError("protected identity boundary drifted") @@ -1288,6 +1340,8 @@ class FrozenCalibrationIdentityArtifact: "assignment", "assignment_sha256", "tokenizer_manifest_sha256", + "parquet_materialization_manifest_file_sha256", + "execution_bindings", ) def __init__( @@ -1299,6 +1353,8 @@ def __init__( assignment: tuple[dict[str, Any], ...], assignment_sha256: str, tokenizer_manifest_sha256: str, + parquet_materialization_manifest_file_sha256: str, + execution_bindings: dict[str, str], ) -> None: self.file_sha256 = file_sha256 self.canonical_evidence_sha256 = canonical_evidence_sha256 @@ -1306,6 +1362,10 @@ def __init__( self.assignment = assignment self.assignment_sha256 = assignment_sha256 self.tokenizer_manifest_sha256 = tokenizer_manifest_sha256 + self.parquet_materialization_manifest_file_sha256 = ( + parquet_materialization_manifest_file_sha256 + ) + self.execution_bindings = execution_bindings class FrozenStageAIdentityArtifact: @@ -1317,6 +1377,8 @@ class FrozenStageAIdentityArtifact: "records", "tokenizer_manifest_sha256", "calibration_binding", + "parquet_materialization_manifest_file_sha256", + "execution_bindings", ) def __init__( @@ -1327,12 +1389,18 @@ def __init__( records: tuple[dict[str, Any], ...], tokenizer_manifest_sha256: str, calibration_binding: dict[str, str], + parquet_materialization_manifest_file_sha256: str, + execution_bindings: dict[str, str], ) -> None: self.file_sha256 = file_sha256 self.canonical_evidence_sha256 = canonical_evidence_sha256 self.records = records self.tokenizer_manifest_sha256 = tokenizer_manifest_sha256 self.calibration_binding = calibration_binding + self.parquet_materialization_manifest_file_sha256 = ( + parquet_materialization_manifest_file_sha256 + ) + self.execution_bindings = execution_bindings class StageACalibrationBindingArtifact: @@ -1397,7 +1465,7 @@ def deserialize_frozen_calibration_identity_artifact( if canonical_evidence_sha256 != sha256_bytes(canonical_json_bytes(evidence)): raise ValueError("frozen identity canonical evidence SHA-256 drifted") exact_scalars = { - "schema_version": 2, + "schema_version": 4, "artifact_kind": ARTIFACT_KIND, "identity_schema": FROZEN_SCHEMA, "resolver_version": RESOLVER_VERSION, @@ -1409,12 +1477,23 @@ def deserialize_frozen_calibration_identity_artifact( "promotion_required": False, } for name, expected in exact_scalars.items(): - if evidence[name] != expected: + if ( + isinstance(expected, (bool, int)) + and type(evidence[name]) is not type(expected) + ) or evidence[name] != expected: raise ValueError(f"frozen identity {name} drifted") require_sha256( evidence["source_manifest_sha256"], context="frozen identity source manifest SHA-256", ) + execution_bindings = _validate_execution_bindings(evidence["execution_bindings"]) + if dict(evidence["execution_bindings"]) != execution_bindings: + raise ValueError("frozen execution bindings are not canonical") + parquet_manifest_sha256 = execution_bindings[ + "parquet_materialization_manifest_file_sha256" + ] + if parquet_manifest_sha256 != PARQUET_MATERIALIZATION_MANIFEST_FILE_SHA256: + raise ValueError("frozen Parquet materialization manifest file SHA-256 drifted") models = require_mapping(evidence["model_contracts"], context="model contracts") require_exact_fields( @@ -1515,12 +1594,23 @@ def deserialize_frozen_calibration_identity_artifact( } if evidence["upstream_tool_contracts"] != expected_upstream: raise ValueError("frozen upstream tool contracts drifted") - expected_protected = { - "stage_b_read": False, - "stage_c_read": False, - "ordinary_tests_may_read_protected_content": False, - } - if evidence["protected_identity"] != expected_protected: + protected = require_mapping( + evidence["protected_identity"], + context="frozen protected identity", + ) + protected_fields = frozenset( + { + "stage_b_read", + "stage_c_read", + "ordinary_tests_may_read_protected_content", + } + ) + require_exact_fields( + protected, + protected_fields, + context="frozen protected identity", + ) + if any(protected[field] is not False for field in protected_fields): raise ValueError("frozen protected identity boundary drifted") split = require_mapping( @@ -1587,6 +1677,8 @@ def deserialize_frozen_calibration_identity_artifact( assignment=assignments, assignment_sha256=str(expected_split["assignment_sha256"]), tokenizer_manifest_sha256=str(tokenizer_manifest_sha256), + parquet_materialization_manifest_file_sha256=parquet_manifest_sha256, + execution_bindings=execution_bindings, ) @@ -1691,12 +1783,20 @@ def deserialize_frozen_stage_a_identity_artifact( candidate_evidence["tokenizer"]["file_manifest_sha256"], context="frozen Stage-A tokenizer manifest SHA-256", ) + execution_bindings = _validate_execution_bindings(candidate_evidence["execution_bindings"]) + parquet_manifest_sha256 = execution_bindings[ + "parquet_materialization_manifest_file_sha256" + ] + if parquet_manifest_sha256 != PARQUET_MATERIALIZATION_MANIFEST_FILE_SHA256: + raise ValueError("frozen Stage-A Parquet materialization manifest file SHA-256 drifted") return FrozenStageAIdentityArtifact( file_sha256=file_sha256, canonical_evidence_sha256=canonical_evidence_sha256, records=records, tokenizer_manifest_sha256=tokenizer_manifest_sha256, calibration_binding=dict(verified_binding), + parquet_materialization_manifest_file_sha256=parquet_manifest_sha256, + execution_bindings=execution_bindings, ) @@ -2094,7 +2194,7 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: def _reject_protected_before_input(phase: str) -> None: if phase in PROTECTED_STAGES: raise PermissionError( - f"{phase} is protected; resolver v2 refuses it before reading --input" + f"{phase} is protected; resolver v3 refuses it before reading --input" ) diff --git a/scripts/run_static_q468_calibration.py b/scripts/run_static_q468_calibration.py new file mode 100644 index 0000000..b19a35a --- /dev/null +++ b/scripts/run_static_q468_calibration.py @@ -0,0 +1,4292 @@ +#!/usr/bin/env python3 +"""Run the fail-closed Experiment 013 static-Q468 calibration. + +The orchestration core is deliberately separated from the live Qwen adapter. +It authenticates the promoted calibration identity, repository source, exact +local model files, and every materialized token sequence before a model is +loaded. A live adapter must expose one-token causal observations; this module +does not silently fall back to chunked inference or an unauthenticated Hub +download. + +The command line accepts only the reviewed Qwen3.5 adapter. Generic adapters +remain test-injection surfaces. Unit tests exercise the complete ordering, EMA, +anchor, output, and failure semantics with small in-memory adapters and never +load model weights or data. +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import importlib +import importlib.metadata +import json +import os +import platform +import re +import shutil +import stat +import struct +import subprocess +import sys +import tempfile +import time +import urllib.parse +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from io import StringIO +from pathlib import Path, PurePosixPath +from types import ModuleType +from typing import Any, Final, Protocol, cast + +REPOSITORY_ROOT: Final = Path(__file__).resolve().parents[1] +IDENTITY_RESOLVER_PATH: Final = REPOSITORY_ROOT / "scripts" / "resolve_static_q468_identity.py" +IDENTITY_RESOLVER_MODULE: Final = "recurquant_experiment013_identity_resolver" +CALIBRATION_API_MODULE: Final = "recurquant.experiment013_calibration_api" +CALIBRATION_API_PATH: Final = "src/recurquant/experiment013_calibration_api.py" +SOURCE_VERIFIER_PATH: Final = "src/recurquant/experiment013_source.py" +SOURCE_CAPTURE_MODULE: Final = "recurquant_experiment013_source_capture" +RUNNER_SOURCE_PATH: Final = "scripts/run_static_q468_calibration.py" +IDENTITY_RESOLVER_SOURCE_PATH: Final = "scripts/resolve_static_q468_identity.py" +CANONICAL_ADAPTER_SPEC: Final = "recurquant.experiment013_qwen35_adapter:create_adapter" +CANONICAL_ADAPTER_MODULE: Final = "recurquant.experiment013_qwen35_adapter" +CANONICAL_ADAPTER_PATH: Final = "src/recurquant/experiment013_qwen35_adapter.py" + +RUNNER_REVISION: Final = "experiment-013-static-q468-calibration-runner-v1" +MODEL_FILE_MANIFEST_KIND: Final = "recurquant_experiment013_model_file_manifest" +MODEL_FILE_MANIFEST_SCHEMA: Final = 1 +MODEL_FILE_MANIFEST_DERIVATION: Final = "huggingface-hub-pinned-tree-lfs-v1" +MODEL_FILE_SELECTION_PROFILE: Final = "qwen35-config-index-safetensors-v1" +RUNTIME_MANIFEST_KIND: Final = "recurquant_experiment013_calibration_runtime_manifest" +RUNTIME_MANIFEST_SCHEMA: Final = 3 +RUN_REPORT_KIND: Final = "recurquant_experiment013_calibration_run" +RUN_REPORT_SCHEMA: Final = 1 + +QUERY_EMA_DECAY: Final = 2.0 ** (-1.0 / 32.0) +QUERY_ENERGY_EPSILON: Final = 1.0e-6 +RHT_SEED: Final = 2_339 + +_SHA256_RE: Final = re.compile(r"[0-9a-f]{64}") +_GIT_REVISION_RE: Final = re.compile(r"[0-9a-f]{40}") +_SAFE_MODEL_FILE_RE: Final = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]*") +_WEIGHT_FILE_RE: Final = re.compile( + r"(?:^|/)(?:model(?:-[0-9]+-of-[0-9]+)?|" + r"model\.safetensors-[0-9]+-of-[0-9]+)\.safetensors$" +) + +SCORE_FILENAME: Final = "calibration-scores.json" +SPLIT_FILENAME: Final = "split-half-stability.json" +K27030_FILENAME: Final = "static-k27030-policy.json" +K29334_FILENAME: Final = "static-k29334-policy.json" +Q48_FILENAME: Final = "static-q48-p14739-policy.json" +BINDING_FILENAME: Final = "stage-a-calibration-binding.json" +REPORT_FILENAME: Final = "calibration-run-report.json" +COMPLETE_FILENAME: Final = "CALIBRATION_COMPLETE" +PREPARED_RUNTIME_MANIFEST_FILENAME: Final = "calibration-runtime-manifest.json" +PREPARED_RUNTIME_COMPLETE_FILENAME: Final = "RUNTIME_PREPARED" +DEFAULT_PACKAGE_RUNTIME_ROOT_NAME: Final = "calibration-packages" +DEFAULT_PACKAGE_IMPORT_PATH: Final = "Lib/site-packages" +STAGING_EXCLUDED_DISTRIBUTIONS: Final = frozenset({"pip", "setuptools"}) + +_WINDOWS_REPARSE_POINT: Final = 0x400 +_RUNTIME_ROOT_NAME_RE: Final = re.compile(r"[a-z][a-z0-9-]{0,63}") +BASE_RUNTIME_ROOT_NAME: Final = "base-runtime" +_FORBIDDEN_RUNTIME_SUFFIXES: Final = frozenset( + {"._pth", ".egg-link", ".pth", ".pyc", ".pyo"} +) +_FORBIDDEN_RUNTIME_DIRECTORY_NAMES: Final = frozenset({"__pycache__"}) +_FORBIDDEN_RUNTIME_FILENAMES: Final = frozenset( + {"pyvenv.cfg", "sitecustomize.py", "usercustomize.py"} +) +SEALED_LAUNCH_POLICY: Final = { + "bootstrap_mode": "stdlib-only-exact-runner-v1", + "dont_write_bytecode": 1, + "ignore_environment": 1, + "isolated": 1, + "no_site": 1, + "no_user_site": 1, + "package_path_mode": "authenticated-record-only-roots-v1", + "pycache_mode": "new-verified-empty-prefix-v1", + "safe_path": True, + "site_loaded": False, + "sys_path_mode": "staged-base-then-authenticated-packages-v1", + "utf8_mode": 1, + "virtualenv_hook_loaded": False, +} + + +class CalibrationRunError(RuntimeError): + """Raised when the authenticated calibration cannot safely continue.""" + + +class CalibrationStabilityFailure(CalibrationRunError): + """Raised after publishing a failure-only report for an unstable policy.""" + + +_AUTHENTICATED_IDENTITY_RESOLVER: ModuleType | None = None +_AUTHENTICATED_SOURCE_VERIFIER: ModuleType | None = None +_AUTHENTICATED_CALIBRATION_API: ModuleType | None = None + + +def canonical_json_bytes(value: object) -> bytes: + """Return the newline-terminated canonical JSON used by run evidence.""" + + return ( + json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + b"\n" + ) + + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _strict_json_bytes(data: bytes, *, context: str) -> dict[str, object]: + if not isinstance(data, bytes): + raise TypeError(f"{context} must be bytes") + + def unique(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"{context} contains duplicate JSON key {key!r}") + result[key] = value + return result + + def reject_constant(value: str) -> None: + raise ValueError(f"{context} contains non-finite JSON constant {value}") + + try: + value = json.loads( + data.decode("utf-8"), + object_pairs_hook=unique, + parse_constant=reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError(f"{context} is not valid UTF-8 JSON") from exc + if not isinstance(value, dict): + raise ValueError(f"{context} must be a JSON object") + return value + + +def _exact_fields(value: Mapping[str, object], expected: set[str], *, context: str) -> None: + actual = set(value) + if actual != expected: + raise ValueError( + f"{context} fields differ; missing={sorted(expected - actual)}, " + f"extra={sorted(actual - expected)}" + ) + + +def _exact_typed_mapping( + value: object, + expected: Mapping[str, object], + *, + context: str, +) -> None: + if not isinstance(value, Mapping) or set(value) != set(expected): + raise ValueError(f"{context} fields differ from the frozen schema") + if any( + type(value[name]) is not type(expected[name]) or value[name] != expected[name] + for name in expected + ): + raise ValueError(f"{context} value or JSON type drifted") + + +def _sha256(value: object, *, context: str) -> str: + if not isinstance(value, str) or _SHA256_RE.fullmatch(value) is None: + raise ValueError(f"{context} must be a lowercase SHA-256 digest") + return value + + +def _git_revision(value: object, *, context: str) -> str: + if not isinstance(value, str) or _GIT_REVISION_RE.fullmatch(value) is None: + raise ValueError(f"{context} must be an immutable lowercase 40-hex revision") + return value + + +def _positive_int(value: object, *, context: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{context} must be a positive integer") + return value + + +def _nonnegative_int(value: object, *, context: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ValueError(f"{context} must be a non-negative integer") + return value + + +@dataclass(frozen=True, slots=True) +class BootstrapIdentityBindings: + repository_source_manifest_file_sha256: str + runtime_manifest_file_sha256: str + model_file_manifest_file_sha256: str + parquet_materialization_manifest_file_sha256: str + + +@dataclass(frozen=True, slots=True) +class BootstrapSource: + manifest: dict[str, object] + source_commit: str + entries: dict[str, dict[str, object]] + + +def _bootstrap_identity_bindings(data: bytes) -> BootstrapIdentityBindings: + """Strictly extract only the v4 execution bindings using stdlib code. + + Full semantic decoding remains the authenticated resolver's job. This + minimal pass exists solely to authenticate the code that performs it. + """ + + root = _strict_json_bytes(data, context="frozen calibration identity bootstrap") + _exact_fields( + root, + {"canonical_evidence_sha256", "evidence"}, + context="frozen calibration identity bootstrap wrapper", + ) + if canonical_json_bytes(root) != data: + raise CalibrationRunError("frozen calibration identity bootstrap bytes are not canonical") + evidence = root["evidence"] + if not isinstance(evidence, dict): + raise CalibrationRunError("frozen calibration identity bootstrap evidence is missing") + if ( + type(evidence.get("schema_version")) is not int + or evidence.get("schema_version") != 4 + or evidence.get("status") != "frozen" + or evidence.get("phase") != "calibration" + or evidence.get("identity_only") is not True + or evidence.get("promotion_required") is not False + ): + raise CalibrationRunError("frozen calibration identity bootstrap state is invalid") + canonical_evidence_sha256 = _sha256( + root["canonical_evidence_sha256"], + context="frozen identity canonical evidence SHA-256", + ) + if canonical_evidence_sha256 != sha256_bytes(canonical_json_bytes(evidence)): + raise CalibrationRunError("frozen identity canonical evidence SHA-256 drifted") + bindings = evidence.get("execution_bindings") + if not isinstance(bindings, dict): + raise CalibrationRunError("frozen identity execution bindings are missing") + _exact_fields( + bindings, + { + "calibration_runtime_manifest_file_sha256", + "model_file_manifest_file_sha256", + "parquet_materialization_manifest_file_sha256", + "repository_source_manifest_file_sha256", + }, + context="frozen identity bootstrap execution bindings", + ) + return BootstrapIdentityBindings( + repository_source_manifest_file_sha256=_sha256( + bindings["repository_source_manifest_file_sha256"], + context="bootstrap repository source manifest file SHA-256", + ), + runtime_manifest_file_sha256=_sha256( + bindings["calibration_runtime_manifest_file_sha256"], + context="bootstrap runtime manifest file SHA-256", + ), + model_file_manifest_file_sha256=_sha256( + bindings["model_file_manifest_file_sha256"], + context="bootstrap model manifest file SHA-256", + ), + parquet_materialization_manifest_file_sha256=_sha256( + bindings["parquet_materialization_manifest_file_sha256"], + context="bootstrap parquet materialization manifest file SHA-256", + ), + ) + + +def _source_manifest_canonical_sha256(value: Mapping[str, object]) -> str: + payload = dict(value) + payload.pop("canonical_manifest_sha256", None) + encoded = (json.dumps(payload, indent=2, sort_keys=True, allow_nan=False) + "\n").encode( + "utf-8" + ) + return sha256_bytes(encoded) + + +def _canonical_relative_path(value: object, *, context: str) -> str: + if not isinstance(value, str) or not value or value != value.strip(): + raise CalibrationRunError(f"{context} must be a non-empty canonical path") + if "\\" in value or "\0" in value or "\n" in value or "\r" in value: + raise CalibrationRunError(f"{context} must be a single-line POSIX path") + path = PurePosixPath(value) + if ( + path.is_absolute() + or path.parts[0].endswith(":") + or any(part in {"", ".", ".."} for part in path.parts) + or path.as_posix() != value + ): + raise CalibrationRunError(f"{context} must be repository-relative") + return value + + +def _is_link_or_reparse(path: Path) -> bool: + try: + stat_result = path.lstat() + except OSError as exc: + raise CalibrationRunError(f"required path is unavailable: {path}") from exc + return path.is_symlink() or bool( + getattr(stat_result, "st_file_attributes", 0) & _WINDOWS_REPARSE_POINT + ) + + +def _assert_no_link_components(root: Path, relative: PurePosixPath) -> Path: + candidate = root + if _is_link_or_reparse(candidate): + raise CalibrationRunError(f"authenticated root is a link or reparse point: {root}") + for part in relative.parts: + candidate = candidate / part + if _is_link_or_reparse(candidate): + raise CalibrationRunError( + f"authenticated path traverses a link or reparse point: {relative}" + ) + try: + resolved_root = root.resolve(strict=True) + resolved = candidate.resolve(strict=True) + resolved.relative_to(resolved_root) + except (OSError, ValueError) as exc: + raise CalibrationRunError(f"authenticated path escapes its root: {relative}") from exc + if not resolved.is_file(): + raise CalibrationRunError(f"authenticated path is not a regular file: {relative}") + return resolved + + +def _bootstrap_source_manifest( + data: bytes, + *, + repository_root: Path, + require_adapter: bool, +) -> BootstrapSource: + manifest = _strict_json_bytes(data, context="repository source manifest bootstrap") + _exact_fields( + manifest, + { + "canonical_manifest_sha256", + "object_format", + "paths", + "profile", + "repository_binding", + "schema", + "source_commit", + }, + context="repository source manifest bootstrap", + ) + claimed = _sha256( + manifest["canonical_manifest_sha256"], + context="repository source manifest canonical SHA-256", + ) + if claimed != _source_manifest_canonical_sha256(manifest): + raise CalibrationRunError("repository source manifest canonical SHA-256 drifted") + source_commit = _git_revision( + manifest["source_commit"], + context="repository source manifest source commit", + ) + raw_paths = manifest["paths"] + if not isinstance(raw_paths, list) or not raw_paths: + raise CalibrationRunError("repository source manifest paths must be non-empty") + entries: dict[str, dict[str, object]] = {} + for index, raw_entry in enumerate(raw_paths): + if not isinstance(raw_entry, dict): + raise CalibrationRunError(f"repository source paths[{index}] must be an object") + _exact_fields( + raw_entry, + { + "git_blob_oid", + "index_blob_oid", + "mode", + "path", + "raw_sha256", + "worktree_blob_oid", + }, + context=f"repository source paths[{index}]", + ) + relative = _canonical_relative_path( + raw_entry["path"], context=f"repository source paths[{index}].path" + ) + if relative in entries: + raise CalibrationRunError(f"duplicate repository source path: {relative}") + _sha256(raw_entry["raw_sha256"], context=f"repository source {relative} SHA-256") + entries[relative] = dict(raw_entry) + required = { + RUNNER_SOURCE_PATH, + IDENTITY_RESOLVER_SOURCE_PATH, + SOURCE_VERIFIER_PATH, + CALIBRATION_API_PATH, + } + if require_adapter: + required.add(CANONICAL_ADAPTER_PATH) + missing = sorted(required - set(entries)) + if missing: + raise CalibrationRunError(f"repository source manifest omits bootstrap paths: {missing}") + root = Path(os.path.abspath(repository_root)) + if not root.is_dir(): + raise CalibrationRunError("repository root is unavailable") + for relative in sorted(required): + source_path = _assert_no_link_components(root, PurePosixPath(relative)) + digest, _size = _stream_file_sha256(source_path) + if digest != entries[relative]["raw_sha256"]: + raise CalibrationRunError(f"bootstrap source bytes drifted: {relative}") + runner_path = _assert_no_link_components(root, PurePosixPath(RUNNER_SOURCE_PATH)) + if Path(__file__).resolve(strict=True) != runner_path: + raise CalibrationRunError("executing runner is not the authenticated repository runner") + return BootstrapSource(manifest=manifest, source_commit=source_commit, entries=entries) + + +def _load_exact_source_module( + module_name: str, + relative_path: str, + *, + repository_root: Path, + entry: Mapping[str, object], +) -> ModuleType: + if module_name in sys.modules: + raise CalibrationRunError(f"refusing preloaded authenticated module: {module_name}") + source_path = _assert_no_link_components( + Path(os.path.abspath(repository_root)), PurePosixPath(relative_path) + ) + expected_sha256 = _sha256(entry.get("raw_sha256"), context=f"{module_name} source SHA-256") + source_bytes = source_path.read_bytes() + actual_sha256 = sha256_bytes(source_bytes) + if actual_sha256 != expected_sha256: + raise CalibrationRunError( + f"authenticated module bytes drifted before import: {module_name}" + ) + try: + code = compile(source_bytes, str(source_path), "exec", dont_inherit=True) + except (SyntaxError, ValueError) as exc: + raise CalibrationRunError(f"cannot compile authenticated source: {module_name}") from exc + module = ModuleType(module_name) + module.__file__ = str(source_path) + module.__package__ = module_name.rpartition(".")[0] + module.__loader__ = None + module.__spec__ = importlib.machinery.ModuleSpec( + module_name, + loader=None, + origin=str(source_path), + ) + sys.modules[module_name] = module + try: + exec(code, module.__dict__) + declared = Path(cast(str, getattr(module, "__file__", ""))).resolve(strict=True) + after_sha256, _after_size = _stream_file_sha256(source_path) + if declared != source_path or after_sha256 != expected_sha256: + raise CalibrationRunError( + f"authenticated module identity drifted on import: {module_name}" + ) + except BaseException: + sys.modules.pop(module_name, None) + raise + return module + + +def _load_source_capture_module(repository_root: Path) -> ModuleType: + """Load the committed source-manifest implementation without package import hooks.""" + + if SOURCE_CAPTURE_MODULE in sys.modules: + raise CalibrationRunError("refusing a preloaded source-manifest capture module") + root = Path(os.path.abspath(repository_root)) + source_path = _assert_no_link_components(root, PurePosixPath(SOURCE_VERIFIER_PATH)) + source_bytes = source_path.read_bytes() + before_sha256 = sha256_bytes(source_bytes) + before_size = len(source_bytes) + try: + code = compile(source_bytes, str(source_path), "exec", dont_inherit=True) + except (SyntaxError, ValueError) as exc: + raise CalibrationRunError("cannot compile source-manifest implementation") from exc + module = ModuleType(SOURCE_CAPTURE_MODULE) + module.__file__ = str(source_path) + module.__package__ = "" + module.__loader__ = None + module.__spec__ = importlib.machinery.ModuleSpec( + SOURCE_CAPTURE_MODULE, + loader=None, + origin=str(source_path), + ) + sys.modules[SOURCE_CAPTURE_MODULE] = module + try: + exec(code, module.__dict__) + declared = Path(cast(str, getattr(module, "__file__", ""))).resolve(strict=True) + after_sha256, after_size = _stream_file_sha256(source_path) + if declared != source_path or after_sha256 != before_sha256 or after_size != before_size: + raise CalibrationRunError("source-manifest implementation changed during import") + except BaseException: + sys.modules.pop(SOURCE_CAPTURE_MODULE, None) + raise + return module + + +def _sanitized_git_environment() -> dict[str, str]: + environment = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} + environment.update( + { + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_OPTIONAL_LOCKS": "0", + "GIT_TERMINAL_PROMPT": "0", + } + ) + return environment + + +def _assert_source_manifest_output_location(repository_root: Path, output: Path) -> Path: + """Allow source manifests only outside the repository or at an ignored path.""" + + try: + root = Path(repository_root).resolve(strict=True) + except OSError as exc: + raise CalibrationRunError("repository root is unavailable") from exc + if not root.is_dir() or _is_link_or_reparse(root): + raise CalibrationRunError("repository root is not a regular directory") + resolved_output = Path(output).resolve(strict=False) + if resolved_output.exists() or resolved_output.is_symlink(): + raise FileExistsError(f"refusing to overwrite existing artifact: {resolved_output}") + try: + relative = resolved_output.relative_to(root) + except ValueError: + return resolved_output + if not relative.parts or relative.parts[0].casefold() == ".git": + raise CalibrationRunError("source manifest output cannot be repository metadata") + process = subprocess.run( + ["git", "check-ignore", "--quiet", "--no-index", "--", relative.as_posix()], + cwd=root, + check=False, + capture_output=True, + env=_sanitized_git_environment(), + ) + if process.returncode == 0: + return resolved_output + if process.returncode == 1: + raise CalibrationRunError( + "source manifest output inside the repository must be ignored by Git" + ) + raise CalibrationRunError("cannot verify source manifest output ignore status") + + +def _torch_runtime() -> Any: + """Import torch only after the caller has authenticated the runtime.""" + + return importlib.import_module("torch") + + +@dataclass(frozen=True, slots=True) +class Geometry: + """Minimal recurrent-state geometry required by the causal runner.""" + + layer_indices: tuple[int, ...] + heads: int + key_rows: int + value_width: int + + def __post_init__(self) -> None: + if ( + not self.layer_indices + or any(index < 0 for index in self.layer_indices) + or len(set(self.layer_indices)) != len(self.layer_indices) + ): + raise ValueError("layer_indices must be unique non-negative integers") + for name in ("heads", "key_rows", "value_width"): + _positive_int(getattr(self, name), context=name) + if self.value_width & (self.value_width - 1): + raise ValueError("value_width must be a power of two") + + @property + def layers(self) -> int: + return len(self.layer_indices) + + @property + def rows_per_layer(self) -> int: + return self.heads * self.key_rows + + @property + def total_rows(self) -> int: + return self.layers * self.rows_per_layer + + +@dataclass(frozen=True, slots=True) +class FrozenCalibrationIdentity: + """Runner-facing view of the resolver's strictly decoded frozen identity.""" + + file_sha256: str + canonical_evidence_sha256: str + records: tuple[dict[str, object], ...] + assignment: tuple[dict[str, object], ...] + assignment_sha256: str + tokenizer_manifest_sha256: str + identity_input_manifest_sha256: str + repository_source_manifest_file_sha256: str + runtime_manifest_file_sha256: str + model_file_manifest_file_sha256: str + parquet_materialization_manifest_file_sha256: str + model_id: str + model_revision: str + transformers_version: str + artifact_bytes: bytes + + +@dataclass(frozen=True, slots=True) +class CapturedSequence: + """Only the current sequence's frozen anchor tensors, all on CPU FP64.""" + + anchor_positions: tuple[int, ...] + query_energy: Any + q4_mse: Any + q6_mse: Any + q8_mse: Any + + +@dataclass(frozen=True, slots=True) +class ModelFileRecord: + name: str + size_bytes: int + sha256: str | None + git_blob_oid: str + lfs_sha256: str | None + lfs_size_bytes: int | None + + +@dataclass(frozen=True, slots=True) +class ModelFileManifest: + model_id: str + revision: str + transformers_version: str + files: tuple[ModelFileRecord, ...] + hub_tree_manifest_sha256: str + file_sha256: str + + +@dataclass(frozen=True, slots=True) +class RuntimeFileRecord: + path: str + size_bytes: int + sha256: str + + +@dataclass(frozen=True, slots=True) +class RuntimeDistributionRecord: + name: str + version: str + package_root: str + files: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class RuntimeTreeRecord: + name: str + kind: str + files: tuple[RuntimeFileRecord, ...] + + +@dataclass(frozen=True, slots=True) +class RuntimePackageRootRecord: + name: str + import_path: str + + +@dataclass(frozen=True, slots=True) +class RuntimeInterpreterProbe: + python_implementation: str + python_version: str + python_cache_tag: str + python_abi_flags: str + machine_system: str + machine_architecture: str + machine_name: str + machine_byteorder: str + machine_pointer_bits: int + base_sys_path: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class RuntimeRequirement: + name: str + version: str + + +@dataclass(frozen=True, slots=True) +class CalibrationRuntimeManifest: + python_implementation: str + python_version: str + python_cache_tag: str + python_abi_flags: str + machine_system: str + machine_architecture: str + machine_name: str + machine_byteorder: str + machine_pointer_bits: int + launch_policy: Mapping[str, object] + base_sys_path: tuple[str, ...] + base_runtime_root: str + package_roots: tuple[RuntimePackageRootRecord, ...] + interpreter_root: str + interpreter_relative_path: str + interpreter_size_bytes: int + interpreter_sha256: str + runtime_trees: tuple[RuntimeTreeRecord, ...] + distributions: tuple[RuntimeDistributionRecord, ...] + file_sha256: str + + +@dataclass(frozen=True, slots=True) +class AuthenticatedRuntime: + manifest_file_sha256: str + python_implementation: str + python_version: str + python_cache_tag: str + interpreter_sha256: str + machine_name: str + base_runtime_file_count: int + package_root_count: int + distributions: tuple[tuple[str, str], ...] + distribution_count: int + file_count: int + + +@dataclass(frozen=True, slots=True) +class SealedRuntimeContext: + manifest_file_sha256: str + base_runtime_root: Path + package_roots: Mapping[str, Path] + package_import_paths: Mapping[str, str] + pycache_prefix: Path + + +@dataclass(frozen=True, slots=True) +class CalibrationArtifacts: + """In-memory pass artifacts; no publication occurs until all are built.""" + + score: bytes + split_half: bytes + static_k27030: bytes + static_k29334: bytes + static_q48: bytes + stage_a_binding: bytes + stability: Mapping[str, object] + calibration_scores_sha256: str + sequence_score_manifest_sha256: str + + +@dataclass(frozen=True, slots=True) +class FinalizationResult: + passed: bool + stability: Mapping[str, object] + artifacts: CalibrationArtifacts | None + + +@dataclass(frozen=True, slots=True) +class CalibrationRunConfig: + frozen_identity_bytes: bytes + repository_source_manifest_bytes: bytes + model_file_manifest_bytes: bytes + parquet_materialization_manifest_bytes: bytes + runtime_manifest_bytes: bytes + model_root: Path + repository_root: Path + expected_source_commit: str + expected_model_file_manifest_sha256: str + expected_parquet_materialization_manifest_sha256: str + expected_runtime_manifest_sha256: str + output_dir: Path + require_cuda: bool = True + + +class CalibrationBackend(Protocol): + geometry: Geometry + + def decode_identity(self, data: bytes) -> FrozenCalibrationIdentity: ... + + def reduce_sequence( + self, + record: Mapping[str, object], + token_ids: tuple[int, ...], + captured: CapturedSequence, + ) -> object: ... + + def finalize( + self, + scores: Sequence[object], + *, + identity: FrozenCalibrationIdentity, + source_commit: str, + ) -> FinalizationResult: ... + + +SourceVerifier = Callable[[Mapping[str, object], Path], tuple[dict[str, object], str]] +AdapterValidator = Callable[[Any], None] +DistortionFunction = Callable[[Any, Geometry], tuple[Any, Any, Any]] +ModelAuthenticator = Callable[[Path, ModelFileManifest], Any] +RuntimeAuthenticator = Callable[[CalibrationRuntimeManifest], AuthenticatedRuntime] + + +@dataclass(frozen=True, slots=True) +class RunnerServices: + backend: CalibrationBackend + calibration_api: ModuleType + verify_repository_source: SourceVerifier + validate_adapter: AdapterValidator + distortion_function: DistortionFunction + authenticate_model_files: ModelAuthenticator + authenticate_runtime: RuntimeAuthenticator + + +def parse_model_file_manifest(data: bytes) -> ModelFileManifest: + """Strictly decode a canonical, immutable local-model file manifest.""" + + root = _strict_json_bytes(data, context="model file manifest") + _exact_fields( + root, + { + "artifact_kind", + "files", + "hub_tree_manifest_sha256", + "metadata_derivation", + "selection_profile", + "model_id", + "revision", + "schema_version", + "transformers_version", + }, + context="model file manifest", + ) + if canonical_json_bytes(root) != data: + raise ValueError("model file manifest is not canonical newline-terminated JSON") + if ( + root["artifact_kind"] != MODEL_FILE_MANIFEST_KIND + or type(root["schema_version"]) is not int + or root["schema_version"] != MODEL_FILE_MANIFEST_SCHEMA + ): + raise ValueError("model file manifest kind or schema drifted") + if root["metadata_derivation"] != MODEL_FILE_MANIFEST_DERIVATION: + raise ValueError("model file manifest metadata derivation drifted") + if root["selection_profile"] != MODEL_FILE_SELECTION_PROFILE: + raise ValueError("model file manifest selection profile drifted") + model_id = root["model_id"] + transformers_version = root["transformers_version"] + if not isinstance(model_id, str) or not model_id or model_id != model_id.strip(): + raise ValueError("model file manifest model_id is invalid") + revision = _git_revision(root["revision"], context="model file manifest revision") + if not isinstance(transformers_version, str) or not re.fullmatch( + r"[0-9]+\.[0-9]+\.[0-9]+", transformers_version + ): + raise ValueError("model file manifest Transformers version must be exact semver") + raw_files = root["files"] + if not isinstance(raw_files, list) or not raw_files: + raise ValueError("model file manifest files must be a non-empty list") + files: list[ModelFileRecord] = [] + for index, item in enumerate(raw_files): + if not isinstance(item, dict): + raise ValueError(f"model file manifest files[{index}] must be an object") + _exact_fields( + item, + { + "git_blob_oid", + "lfs_sha256", + "lfs_size_bytes", + "name", + "sha256", + "size_bytes", + }, + context=f"files[{index}]", + ) + name = item["name"] + if not isinstance(name, str) or _SAFE_MODEL_FILE_RE.fullmatch(name) is None: + raise ValueError(f"model file manifest files[{index}].name is invalid") + path = PurePosixPath(name) + if path.is_absolute() or ".." in path.parts or "." in path.parts or "\\" in name: + raise ValueError("model file names must be canonical relative POSIX paths") + size_bytes = _positive_int(item["size_bytes"], context=f"files[{index}].size_bytes") + raw_file_sha256 = item["sha256"] + git_blob_oid = _git_revision(item["git_blob_oid"], context=f"files[{index}].git_blob_oid") + raw_lfs_sha256 = item["lfs_sha256"] + raw_lfs_size = item["lfs_size_bytes"] + if raw_lfs_sha256 is None and raw_lfs_size is None: + lfs_sha256 = None + lfs_size_bytes = None + if raw_file_sha256 is not None: + raise ValueError("ordinary Git blobs must use null SHA-256 and their Git blob OID") + file_sha256 = None + elif raw_lfs_sha256 is not None and raw_lfs_size is not None: + lfs_sha256 = _sha256(raw_lfs_sha256, context=f"files[{index}].lfs_sha256") + file_sha256 = _sha256(raw_file_sha256, context=f"files[{index}].sha256") + lfs_size_bytes = _positive_int( + raw_lfs_size, + context=f"files[{index}].lfs_size_bytes", + ) + if lfs_sha256 != file_sha256 or lfs_size_bytes != size_bytes: + raise ValueError("model LFS identity must equal the local content contract") + else: + raise ValueError("model LFS SHA-256 and size must either both be null or both be set") + if _WEIGHT_FILE_RE.search(name) and lfs_sha256 is None: + raise ValueError("safetensors weights require a pinned Hub LFS identity") + files.append( + ModelFileRecord( + name=name, + size_bytes=size_bytes, + sha256=file_sha256, + git_blob_oid=git_blob_oid, + lfs_sha256=lfs_sha256, + lfs_size_bytes=lfs_size_bytes, + ) + ) + names = [item.name for item in files] + if names != sorted(names) or len(names) != len(set(names)): + raise ValueError("model file manifest names must be unique and sorted") + if "config.json" not in names: + raise ValueError("model file manifest must authenticate config.json") + if not any(_WEIGHT_FILE_RE.search(name) for name in names): + raise ValueError( + "model file manifest must authenticate at least one safetensors weight file" + ) + tree_payload = [ + { + "git_blob_oid": item.git_blob_oid, + "lfs_sha256": item.lfs_sha256, + "lfs_size_bytes": item.lfs_size_bytes, + "name": item.name, + } + for item in files + ] + hub_tree_manifest_sha256 = _sha256( + root["hub_tree_manifest_sha256"], + context="model Hub tree manifest SHA-256", + ) + if hub_tree_manifest_sha256 != sha256_bytes(canonical_json_bytes(tree_payload)): + raise ValueError("model Hub tree metadata manifest SHA-256 drifted") + return ModelFileManifest( + model_id=model_id, + revision=revision, + transformers_version=transformers_version, + files=tuple(files), + hub_tree_manifest_sha256=hub_tree_manifest_sha256, + file_sha256=sha256_bytes(data), + ) + + +def _hub_value(value: object, name: str, *, default: object = None) -> object: + if isinstance(value, Mapping): + return value.get(name, default) + return getattr(value, name, default) + + +def _selected_model_tree_path(path: str) -> bool: + if "/" in path: + return False + return path in {"config.json", "model.safetensors.index.json"} or bool( + _WEIGHT_FILE_RE.fullmatch(path) + ) + + +def capture_model_file_manifest_from_hub( + model_id: str, + revision: str, + *, + transformers_version: str, + api: object | None = None, + tree_entries: Sequence[object] | None = None, + resolved_revision: str | None = None, + token: str | bool | None = None, +) -> bytes: + """Build the local-file contract using only pinned Hub tree/LFS metadata. + + The function never downloads or opens model files. Ordinary files are + authenticated by their Git blob OID. LFS files additionally bind the + content SHA-256 and byte size advertised by the pinned Hub revision. + """ + + if not isinstance(model_id, str) or not model_id or model_id != model_id.strip(): + raise ValueError("model_id must be a non-empty canonical string") + pinned_revision = _git_revision(revision, context="model Hub revision") + if ( + not isinstance(transformers_version, str) + or re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", transformers_version) is None + ): + raise ValueError("Transformers version must be exact semver") + if tree_entries is None: + if api is None: + from huggingface_hub import HfApi + + api = HfApi() + info = api.model_info( # type: ignore[attr-defined] + model_id, + revision=pinned_revision, + files_metadata=False, + token=token, + ) + resolved = _git_revision( + getattr(info, "sha", None), + context="resolved Hub model revision", + ) + if resolved != pinned_revision: + raise CalibrationRunError("Hub resolved a different immutable model revision") + tree_entries = tuple( + api.list_repo_tree( # type: ignore[attr-defined] + model_id, + recursive=True, + expand=False, + revision=pinned_revision, + repo_type="model", + token=token, + ) + ) + else: + resolved = _git_revision( + resolved_revision, + context="mocked resolved Hub model revision", + ) + if resolved != pinned_revision: + raise CalibrationRunError("Hub tree metadata revision differs from the pinned revision") + + selected: list[dict[str, object]] = [] + seen_paths: set[str] = set() + for index, entry in enumerate(tree_entries): + raw_path = _hub_value(entry, "path") + if not isinstance(raw_path, str) or not raw_path or "\\" in raw_path or "\0" in raw_path: + raise ValueError(f"Hub tree entry {index} has an invalid path") + path = PurePosixPath(raw_path) + if path.is_absolute() or ".." in path.parts or "." in path.parts: + raise ValueError("Hub tree paths must be canonical repository-relative POSIX paths") + if raw_path in seen_paths: + raise ValueError(f"Hub tree metadata contains duplicate path: {raw_path}") + seen_paths.add(raw_path) + if not _selected_model_tree_path(raw_path): + continue + size_bytes = _positive_int( + _hub_value(entry, "size"), + context=f"Hub tree {raw_path} size", + ) + git_blob_oid = _git_revision( + _hub_value(entry, "blob_id"), + context=f"Hub tree {raw_path} Git blob OID", + ) + raw_lfs = _hub_value(entry, "lfs") + if raw_lfs is None: + lfs_sha256 = None + lfs_size_bytes = None + file_sha256 = None + else: + lfs_sha256 = _sha256( + _hub_value(raw_lfs, "sha256"), + context=f"Hub tree {raw_path} LFS SHA-256", + ) + lfs_size_bytes = _positive_int( + _hub_value(raw_lfs, "size"), + context=f"Hub tree {raw_path} LFS size", + ) + if lfs_size_bytes != size_bytes: + raise ValueError(f"Hub tree {raw_path} size differs from LFS metadata") + file_sha256 = lfs_sha256 + if _WEIGHT_FILE_RE.fullmatch(raw_path) and lfs_sha256 is None: + raise ValueError(f"safetensors weight lacks pinned LFS metadata: {raw_path}") + selected.append( + { + "git_blob_oid": git_blob_oid, + "lfs_sha256": lfs_sha256, + "lfs_size_bytes": lfs_size_bytes, + "name": raw_path, + "sha256": file_sha256, + "size_bytes": size_bytes, + } + ) + selected.sort(key=lambda item: cast(str, item["name"])) + names = [cast(str, item["name"]) for item in selected] + if "config.json" not in names: + raise ValueError("pinned Hub tree has no root config.json") + if not any(_WEIGHT_FILE_RE.fullmatch(name) for name in names): + raise ValueError("pinned Hub tree has no root safetensors weight files") + tree_payload = [ + { + "git_blob_oid": item["git_blob_oid"], + "lfs_sha256": item["lfs_sha256"], + "lfs_size_bytes": item["lfs_size_bytes"], + "name": item["name"], + } + for item in selected + ] + document = { + "artifact_kind": MODEL_FILE_MANIFEST_KIND, + "files": selected, + "hub_tree_manifest_sha256": sha256_bytes(canonical_json_bytes(tree_payload)), + "metadata_derivation": MODEL_FILE_MANIFEST_DERIVATION, + "model_id": model_id, + "revision": pinned_revision, + "schema_version": MODEL_FILE_MANIFEST_SCHEMA, + "selection_profile": MODEL_FILE_SELECTION_PROFILE, + "transformers_version": transformers_version, + } + payload = canonical_json_bytes(document) + parse_model_file_manifest(payload) + return payload + + +def _stream_file_sha256(path: Path) -> tuple[str, int]: + before = path.stat() + digest = hashlib.sha256() + size = 0 + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + size += len(chunk) + after = path.stat() + if ( + before.st_size != after.st_size + or before.st_mtime_ns != after.st_mtime_ns + or before.st_dev != after.st_dev + or before.st_ino != after.st_ino + or size != after.st_size + ): + raise CalibrationRunError(f"model file changed while hashing: {path}") + return digest.hexdigest(), size + + +def _stream_model_file_identity(path: Path) -> tuple[str, str, int]: + before = path.stat() + size = before.st_size + sha256 = hashlib.sha256() + git_blob = hashlib.sha1(usedforsecurity=False) + git_blob.update(f"blob {size}\0".encode("ascii")) + streamed = 0 + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + sha256.update(chunk) + git_blob.update(chunk) + streamed += len(chunk) + after = path.stat() + if ( + streamed != size + or before.st_size != after.st_size + or before.st_mtime_ns != after.st_mtime_ns + or before.st_dev != after.st_dev + or before.st_ino != after.st_ino + ): + raise CalibrationRunError(f"model file changed while hashing: {path}") + return sha256.hexdigest(), git_blob.hexdigest(), size + + +def authenticate_local_model_files( + model_root: Path, + manifest: ModelFileManifest, + *, + calibration_api: ModuleType, +) -> Any: + """Hash every exact local model file immediately before model loading.""" + + root = Path(os.path.abspath(model_root)) + if not root.is_dir(): + raise ValueError("model_root must be a directory") + if _is_link_or_reparse(root): + raise CalibrationRunError("model_root must not be a symlink or reparse point") + actual_names: list[str] = [] + for path in root.rglob("*"): + relative = path.relative_to(root).as_posix() + if _is_link_or_reparse(path): + raise CalibrationRunError(f"local model path is a link or reparse point: {relative}") + if path.is_file(): + actual_names.append(relative) + elif not path.is_dir(): + raise CalibrationRunError( + f"local model path is not a regular file/directory: {relative}" + ) + actual_names.sort() + expected_names = [item.name for item in manifest.files] + if actual_names != expected_names: + raise CalibrationRunError( + "local model file set differs from the authenticated manifest; " + f"missing={sorted(set(expected_names) - set(actual_names))}, " + f"extra={sorted(set(actual_names) - set(expected_names))}" + ) + for item in manifest.files: + candidate = _assert_no_link_components(root, PurePosixPath(item.name)) + sha256, git_blob_oid, size = _stream_model_file_identity(candidate) + content_matches = ( + sha256 == item.lfs_sha256 + if item.lfs_sha256 is not None + else git_blob_oid == item.git_blob_oid + ) + if size != item.size_bytes or not content_matches: + raise CalibrationRunError(f"local model file authentication failed: {item.name}") + identities = tuple( + calibration_api.ModelFileIdentity( + name=item.name, + size_bytes=item.size_bytes, + sha256=item.sha256, + git_blob_oid=item.git_blob_oid, + lfs_sha256=item.lfs_sha256, + lfs_size_bytes=item.lfs_size_bytes, + ) + for item in manifest.files + ) + return calibration_api.AuthenticatedModelFiles( + model_root=root, + model_id=manifest.model_id, + revision=manifest.revision, + transformers_version=manifest.transformers_version, + files=identities, + hub_tree_manifest_sha256=manifest.hub_tree_manifest_sha256, + manifest_file_sha256=manifest.file_sha256, + ) + + +def _normalized_distribution_name(value: object) -> str: + if not isinstance(value, str) or not value.strip(): + raise CalibrationRunError("installed distribution has no canonical name") + normalized = re.sub(r"[-_.]+", "-", value.strip()).lower() + if re.fullmatch(r"[a-z0-9][a-z0-9-]*", normalized) is None: + raise CalibrationRunError(f"installed distribution name is invalid: {value!r}") + return normalized + + +def _runtime_root_name(value: object, *, context: str) -> str: + if not isinstance(value, str) or _RUNTIME_ROOT_NAME_RE.fullmatch(value) is None: + raise CalibrationRunError(f"{context} is not a canonical runtime-root name") + return value + + +def _absolute_runtime_root(path: Path, *, context: str) -> Path: + root = Path(os.path.abspath(path)) + if _is_link_or_reparse(root): + raise CalibrationRunError(f"{context} is a link or reparse point") + try: + resolved = root.resolve(strict=True) + except OSError as exc: + raise CalibrationRunError(f"{context} is unavailable") from exc + if not resolved.is_dir(): + raise CalibrationRunError(f"{context} is not a directory") + return resolved + + +def _runtime_root_map( + base_runtime_root: Path, + package_roots: Mapping[str, Path], +) -> dict[str, Path]: + if not isinstance(package_roots, Mapping) or not package_roots: + raise CalibrationRunError("at least one explicit package root is required") + roots = { + BASE_RUNTIME_ROOT_NAME: _absolute_runtime_root( + base_runtime_root, + context="base runtime root", + ) + } + for raw_name, raw_path in sorted(package_roots.items()): + name = _runtime_root_name(raw_name, context="package root name") + if name == BASE_RUNTIME_ROOT_NAME or name in roots: + raise CalibrationRunError(f"duplicate or reserved package root name: {name}") + roots[name] = _absolute_runtime_root( + Path(raw_path), + context=f"package root {name}", + ) + resolved = list(roots.items()) + for index, (left_name, left) in enumerate(resolved): + for right_name, right in resolved[index + 1 :]: + if left == right: + raise CalibrationRunError( + f"runtime roots resolve to the same directory: {left_name}, {right_name}" + ) + for outer_name, outer, inner_name, inner in ( + (left_name, left, right_name, right), + (right_name, right, left_name, left), + ): + try: + inner.relative_to(outer) + except ValueError: + continue + raise CalibrationRunError( + f"runtime roots must not be nested: {outer_name}, {inner_name}" + ) + return roots + + +def _normalized_package_import_paths( + package_roots: Mapping[str, Path], + package_import_paths: Mapping[str, str], +) -> dict[str, str]: + if not isinstance(package_import_paths, Mapping) or set(package_import_paths) != set( + package_roots + ): + raise CalibrationRunError( + "package import paths must exactly match the named package roots" + ) + normalized: dict[str, str] = {} + for name in sorted(package_roots): + relative = _canonical_relative_path( + package_import_paths[name], + context=f"package root {name} import path", + ) + candidate = package_roots[name] + for part in PurePosixPath(relative).parts: + candidate /= part + if _is_link_or_reparse(candidate): + raise CalibrationRunError( + f"package root {name} import path traverses a link or reparse" + ) + try: + resolved = candidate.resolve(strict=True) + resolved.relative_to(package_roots[name]) + except (OSError, ValueError) as exc: + raise CalibrationRunError( + f"package root {name} import path is outside its runtime tree" + ) from exc + if not resolved.is_dir(): + raise CalibrationRunError(f"package root {name} import path is not a directory") + normalized[name] = relative + return normalized + + +def _capture_base_sys_path( + base_runtime_root: Path, + supplied: Sequence[str] | None, +) -> tuple[str, ...]: + if supplied is not None: + values = tuple( + _canonical_relative_path(item, context="base sys.path entry") for item in supplied + ) + else: + root = base_runtime_root.resolve(strict=True) + if Path(sys.prefix).resolve(strict=True) != root or Path(sys.base_prefix).resolve( + strict=True + ) != root: + raise CalibrationRunError( + "runtime capture must run from the staged base interpreter with no virtualenv" + ) + captured: list[str] = [] + for raw_entry in sys.path: + if not isinstance(raw_entry, str) or not raw_entry or not Path(raw_entry).is_absolute(): + raise CalibrationRunError("isolated base sys.path contains a relative entry") + try: + relative = Path(os.path.abspath(raw_entry)).relative_to(root).as_posix() + except ValueError as exc: + raise CalibrationRunError( + "isolated base sys.path escapes the staged base runtime" + ) from exc + captured.append( + _canonical_relative_path(relative, context="base sys.path entry") + ) + values = tuple(captured) + if not values or len(set(item.casefold() for item in values)) != len(values): + raise CalibrationRunError("base sys.path entries must be non-empty and unique") + return values + + +def _verify_runtime_capture_launch_state() -> None: + if ( + sys.flags.isolated != 1 + or sys.flags.ignore_environment != 1 + or sys.flags.no_user_site != 1 + or sys.flags.no_site != 1 + or sys.flags.dont_write_bytecode != 1 + or sys.flags.utf8_mode != 1 + or sys.flags.safe_path is not True + ): + raise CalibrationRunError( + "runtime capture requires -I -S -B -X utf8 on the staged interpreter" + ) + if sys._xoptions != {"utf8": True}: + raise CalibrationRunError("runtime capture Python -X options drifted") + if "site" in sys.modules or "_virtualenv" in sys.modules: + raise CalibrationRunError("runtime capture loaded site or virtualenv startup hooks") + + +def _runtime_path_is_forbidden(relative: str) -> bool: + path = PurePosixPath(relative) + return ( + any(part.casefold() in _FORBIDDEN_RUNTIME_DIRECTORY_NAMES for part in path.parts) + or path.name.casefold() in _FORBIDDEN_RUNTIME_FILENAMES + or path.suffix.casefold() in _FORBIDDEN_RUNTIME_SUFFIXES + ) + + +def _runtime_tree_files(root: Path, *, kind: str) -> tuple[RuntimeFileRecord, ...]: + """Hash one complete staged tree without following links or reparses.""" + + root = _absolute_runtime_root(root, context=f"{kind} tree root") + stack: list[tuple[Path, tuple[str, ...]]] = [(root, ())] + files: list[RuntimeFileRecord] = [] + casefolded_paths: set[str] = set() + while stack: + directory, relative_parts = stack.pop() + try: + entries = sorted(os.scandir(directory), key=lambda item: item.name) + except OSError as exc: + raise CalibrationRunError(f"cannot enumerate {kind} tree") from exc + for entry in entries: + parts = (*relative_parts, entry.name) + relative = _canonical_relative_path( + PurePosixPath(*parts).as_posix(), + context=f"{kind} tree path", + ) + try: + status = entry.stat(follow_symlinks=False) + except OSError as exc: + raise CalibrationRunError( + f"runtime tree path is unavailable: {relative}" + ) from exc + if entry.is_symlink() or ( + getattr(status, "st_file_attributes", 0) & _WINDOWS_REPARSE_POINT + ): + raise CalibrationRunError( + f"runtime tree path is a link or reparse point: {relative}" + ) + if stat.S_ISDIR(status.st_mode): + if entry.name.casefold() in _FORBIDDEN_RUNTIME_DIRECTORY_NAMES: + raise CalibrationRunError( + f"runtime tree contains forbidden cache directory: {relative}" + ) + stack.append((Path(entry.path), parts)) + continue + if not stat.S_ISREG(status.st_mode): + raise CalibrationRunError(f"runtime tree path is not regular: {relative}") + if _runtime_path_is_forbidden(relative): + raise CalibrationRunError(f"runtime tree contains forbidden file: {relative}") + folded = relative.casefold() + if folded in casefolded_paths: + raise CalibrationRunError( + f"runtime tree contains a case-insensitive duplicate path: {relative}" + ) + casefolded_paths.add(folded) + path = Path(entry.path) + digest, size = _stream_file_sha256(path) + if _is_link_or_reparse(path): + raise CalibrationRunError( + f"runtime tree path became a link or reparse point: {relative}" + ) + try: + path.resolve(strict=True).relative_to(root) + except (OSError, ValueError) as exc: + raise CalibrationRunError( + f"runtime tree path escapes its root: {relative}" + ) from exc + files.append(RuntimeFileRecord(path=relative, size_bytes=size, sha256=digest)) + files.sort(key=lambda item: item.path) + if not files: + raise CalibrationRunError(f"{kind} tree contains no files") + return tuple(files) + + +def _raw_record_path(value: object, *, context: str) -> str: + if not isinstance(value, str) or not value or value != value.strip(): + raise CalibrationRunError(f"{context} is empty or non-canonical") + if "\\" in value or "\0" in value or "\n" in value or "\r" in value: + raise CalibrationRunError(f"{context} is not a POSIX path") + path = PurePosixPath(value) + if path.is_absolute() or path.parts[0].endswith(":") or path.as_posix() != value: + raise CalibrationRunError(f"{context} is absolute or non-canonical") + return value + + +def _record_paths(distribution: Any, *, name: str) -> tuple[str, ...]: + read_text = getattr(distribution, "read_text", None) + if not callable(read_text): + raise CalibrationRunError(f"distribution has no RECORD reader: {name}") + record_text = read_text("RECORD") + if not isinstance(record_text, str) or not record_text: + raise CalibrationRunError(f"distribution has no wheel RECORD inventory: {name}") + parsed: list[str] = [] + try: + for index, row in enumerate(csv.reader(StringIO(record_text, newline=""))): + if len(row) != 3: + raise CalibrationRunError( + f"distribution {name} RECORD row {index} is malformed" + ) + parsed.append( + _raw_record_path( + row[0], + context=f"distribution {name} RECORD path", + ) + ) + except csv.Error as exc: + raise CalibrationRunError(f"distribution {name} RECORD is malformed") from exc + if not parsed: + raise CalibrationRunError(f"distribution has an empty RECORD inventory: {name}") + if len({path.casefold() for path in parsed}) != len(parsed): + raise CalibrationRunError( + f"distribution {name} RECORD paths must be case-insensitively unique" + ) + raw_files = getattr(distribution, "files", None) + if raw_files is None: + raise CalibrationRunError(f"distribution has no RECORD file inventory: {name}") + advertised = tuple( + _raw_record_path( + PurePosixPath(str(path)).as_posix(), + context=f"distribution {name} metadata path", + ) + for path in raw_files + ) + if sorted(advertised) != sorted(parsed): + raise CalibrationRunError( + f"distribution {name} metadata files differ from its exact RECORD" + ) + return tuple(sorted(parsed)) + + +def _distribution_record( + distribution: Any, + *, + name: str, + package_roots: Mapping[str, Path], + package_import_paths: Mapping[str, str], +) -> RuntimeDistributionRecord: + raw_paths = _record_paths(distribution, name=name) + selected_root: str | None = None + rendered_paths: list[str] = [] + for record_path in raw_paths: + located = Path(distribution.locate_file(record_path)) + matches: list[tuple[str, Path]] = [] + for root_name, root in package_roots.items(): + try: + candidate = located.resolve(strict=True) + candidate.relative_to(root) + except (OSError, ValueError): + continue + if _is_link_or_reparse(candidate) or not candidate.is_file(): + raise CalibrationRunError( + f"distribution {name} RECORD path is not a regular authenticated file" + ) + matches.append((root_name, candidate)) + if len(matches) != 1: + raise CalibrationRunError( + f"distribution {name} RECORD path is outside or ambiguous across package roots" + ) + matched_root, matched_path = matches[0] + if selected_root is None: + selected_root = matched_root + elif selected_root != matched_root: + raise CalibrationRunError( + f"distribution {name} spans more than one authenticated package root" + ) + rendered_paths.append(matched_path.relative_to(package_roots[matched_root]).as_posix()) + assert selected_root is not None + import_root = package_roots[selected_root] / PurePosixPath( + package_import_paths[selected_root] + ) + try: + import_root.resolve(strict=True).relative_to(package_roots[selected_root]) + except (OSError, ValueError) as exc: + raise CalibrationRunError( + f"distribution {name} package import path escapes its tree root" + ) from exc + if len({path.casefold() for path in rendered_paths}) != len(rendered_paths): + raise CalibrationRunError( + f"distribution {name} RECORD paths collide after tree-root normalization" + ) + version = str(distribution.version) + if not version or version != version.strip(): + raise CalibrationRunError(f"distribution {name} has an invalid version") + return RuntimeDistributionRecord( + name=name, + version=version, + package_root=selected_root, + files=tuple(sorted(rendered_paths)), + ) + + +def _installed_distribution_map( + distributions: Sequence[Any] | None = None, + *, + package_roots: Mapping[str, Path] | None = None, + package_import_paths: Mapping[str, str] | None = None, +) -> dict[str, Any]: + if distributions is None: + if not package_roots or not package_import_paths: + raise CalibrationRunError( + "installed distribution discovery requires explicit package roots" + ) + selected = list( + importlib.metadata.distributions( + path=[ + str(package_roots[name] / PurePosixPath(package_import_paths[name])) + for name in sorted(package_roots) + ] + ) + ) + else: + selected = list(distributions) + result: dict[str, Any] = {} + for distribution in selected: + name = _normalized_distribution_name(distribution.metadata.get("Name")) + if name in result: + raise CalibrationRunError(f"duplicate installed distribution identity: {name}") + result[name] = distribution + if not result: + raise CalibrationRunError("calibration runtime contains no installed distributions") + return result + + +def capture_calibration_runtime_manifest( + *, + base_runtime_root: Path, + package_roots: Mapping[str, Path], + package_import_paths: Mapping[str, str], + base_sys_path: Sequence[str] | None = None, + interpreter_relative_path: str | None = None, + distributions: Sequence[Any] | None = None, + interpreter_path: Path | None = None, + runtime_probe: RuntimeInterpreterProbe | None = None, +) -> bytes: + """Capture complete staged base and RECORD-only package trees. + + Capture itself is not authorization to load weights. The canonical bytes + must be frozen into the promoted identity before ``run_calibration`` can + accept them. + """ + + roots = _runtime_root_map(base_runtime_root, package_roots) + packages = {name: roots[name] for name in sorted(roots) if name != BASE_RUNTIME_ROOT_NAME} + import_paths = _normalized_package_import_paths(packages, package_import_paths) + if runtime_probe is not None and not isinstance(runtime_probe, RuntimeInterpreterProbe): + raise TypeError("runtime_probe must be RuntimeInterpreterProbe") + if runtime_probe is not None and base_sys_path is not None: + raise CalibrationRunError("runtime probe and explicit base sys.path are mutually exclusive") + if runtime_probe is None and base_sys_path is None: + _verify_runtime_capture_launch_state() + frozen_base_sys_path = _capture_base_sys_path( + roots[BASE_RUNTIME_ROOT_NAME], + runtime_probe.base_sys_path if runtime_probe is not None else base_sys_path, + ) + trees: list[RuntimeTreeRecord] = [ + RuntimeTreeRecord( + name=BASE_RUNTIME_ROOT_NAME, + kind="base-runtime", + files=_runtime_tree_files(roots[BASE_RUNTIME_ROOT_NAME], kind="base-runtime"), + ) + ] + for name, root in packages.items(): + trees.append( + RuntimeTreeRecord( + name=name, + kind="packages", + files=_runtime_tree_files(root, kind=f"package root {name}"), + ) + ) + selected_interpreter = ( + Path(sys.executable) if interpreter_path is None else Path(interpreter_path) + ) + try: + resolved_interpreter = selected_interpreter.resolve(strict=True) + derived_relative = resolved_interpreter.relative_to( + roots[BASE_RUNTIME_ROOT_NAME] + ).as_posix() + except (OSError, ValueError) as exc: + raise CalibrationRunError("interpreter is outside the staged base runtime root") from exc + relative_interpreter = ( + derived_relative + if interpreter_relative_path is None + else _canonical_relative_path( + interpreter_relative_path, + context="runtime interpreter relative path", + ) + ) + if relative_interpreter != derived_relative: + raise CalibrationRunError("interpreter relative path differs from its staged location") + base_files = {item.path: item for item in trees[0].files} + if relative_interpreter not in base_files: + raise CalibrationRunError("staged interpreter is absent from the complete base tree") + + installed = _installed_distribution_map( + distributions, + package_roots=packages, + package_import_paths=import_paths, + ) + distribution_records: list[RuntimeDistributionRecord] = [] + for name, distribution in sorted(installed.items()): + distribution_records.append( + _distribution_record( + distribution, + name=name, + package_roots=packages, + package_import_paths=import_paths, + ) + ) + tree_by_name = {tree.name: tree for tree in trees} + for root_name in packages: + expected = {item.path for item in tree_by_name[root_name].files} + claimed: set[str] = set() + for distribution in distribution_records: + if distribution.package_root != root_name: + continue + overlap = claimed.intersection(distribution.files) + if overlap: + raise CalibrationRunError( + f"package root {root_name} has duplicate RECORD ownership: {sorted(overlap)}" + ) + claimed.update(distribution.files) + if claimed != expected: + raise CalibrationRunError( + f"package root {root_name} differs from the complete RECORD inventory; " + f"missing={sorted(expected - claimed)}, extra={sorted(claimed - expected)}" + ) + interpreter = base_files[relative_interpreter] + current_machine = _current_machine_identity() + machine_identity = ( + ( + runtime_probe.machine_system, + runtime_probe.machine_architecture, + runtime_probe.machine_name, + runtime_probe.machine_byteorder, + runtime_probe.machine_pointer_bits, + ) + if runtime_probe is not None + else current_machine + ) + python_identity = ( + ( + runtime_probe.python_implementation, + runtime_probe.python_version, + runtime_probe.python_cache_tag, + runtime_probe.python_abi_flags, + ) + if runtime_probe is not None + else ( + platform.python_implementation(), + platform.python_version(), + sys.implementation.cache_tag, + getattr(sys, "abiflags", ""), + ) + ) + document = { + "artifact_kind": RUNTIME_MANIFEST_KIND, + "base_sys_path": list(frozen_base_sys_path), + "base_runtime_root": BASE_RUNTIME_ROOT_NAME, + "distributions": [ + { + "files": list(item.files), + "name": item.name, + "package_root": item.package_root, + "version": item.version, + } + for item in distribution_records + ], + "interpreter": { + "relative_path": relative_interpreter, + "root": BASE_RUNTIME_ROOT_NAME, + "sha256": interpreter.sha256, + "size_bytes": interpreter.size_bytes, + }, + "launch_policy": dict(SEALED_LAUNCH_POLICY), + "machine": dict( + zip( + ("system", "architecture", "machine", "byteorder", "pointer_bits"), + machine_identity, + strict=True, + ) + ), + "package_roots": [ + {"import_path": import_paths[name], "name": name} for name in packages + ], + "python": { + "abi_flags": python_identity[3], + "cache_tag": python_identity[2], + "implementation": python_identity[0], + "version": python_identity[1], + }, + "runtime_trees": [ + { + "files": [ + { + "path": file.path, + "sha256": file.sha256, + "size_bytes": file.size_bytes, + } + for file in tree.files + ], + "kind": tree.kind, + "name": tree.name, + } + for tree in trees + ], + "schema_version": RUNTIME_MANIFEST_SCHEMA, + } + payload = canonical_json_bytes(document) + parse_calibration_runtime_manifest(payload) + return payload + + +def _parse_runtime_tree(raw_tree: object, *, index: int) -> RuntimeTreeRecord: + if not isinstance(raw_tree, dict): + raise ValueError(f"runtime_trees[{index}] must be an object") + _exact_fields(raw_tree, {"files", "kind", "name"}, context=f"runtime_trees[{index}]") + name = _runtime_root_name(raw_tree["name"], context=f"runtime_trees[{index}].name") + kind = raw_tree["kind"] + if kind not in {"base-runtime", "packages"}: + raise ValueError(f"runtime_trees[{index}].kind is invalid") + raw_files = raw_tree["files"] + if not isinstance(raw_files, list) or not raw_files: + raise ValueError(f"runtime_trees[{index}].files must be non-empty") + files: list[RuntimeFileRecord] = [] + for file_index, raw_file in enumerate(raw_files): + if not isinstance(raw_file, dict): + raise ValueError(f"runtime_trees[{index}].files[{file_index}] must be an object") + _exact_fields( + raw_file, + {"path", "sha256", "size_bytes"}, + context=f"runtime_trees[{index}].files[{file_index}]", + ) + path = _canonical_relative_path( + raw_file["path"], + context=f"runtime tree {name} file path", + ) + if _runtime_path_is_forbidden(path): + raise ValueError(f"runtime tree {name} contains a forbidden file") + files.append( + RuntimeFileRecord( + path=path, + size_bytes=_nonnegative_int( + raw_file["size_bytes"], context=f"runtime tree {name} file size" + ), + sha256=_sha256( + raw_file["sha256"], context=f"runtime tree {name} file SHA-256" + ), + ) + ) + if [item.path for item in files] != sorted(item.path for item in files) or len( + {item.path.casefold() for item in files} + ) != len(files): + raise ValueError(f"runtime tree {name} file paths must be unique and sorted") + return RuntimeTreeRecord(name=name, kind=cast(str, kind), files=tuple(files)) + + +def parse_calibration_runtime_manifest(data: bytes) -> CalibrationRuntimeManifest: + root = _strict_json_bytes(data, context="calibration runtime manifest") + _exact_fields( + root, + { + "artifact_kind", + "base_runtime_root", + "base_sys_path", + "distributions", + "interpreter", + "launch_policy", + "machine", + "package_roots", + "python", + "runtime_trees", + "schema_version", + }, + context="calibration runtime manifest", + ) + if canonical_json_bytes(root) != data: + raise ValueError("calibration runtime manifest is not canonical JSON") + if ( + root["artifact_kind"] != RUNTIME_MANIFEST_KIND + or type(root["schema_version"]) is not int + or root["schema_version"] != RUNTIME_MANIFEST_SCHEMA + ): + raise ValueError("calibration runtime manifest kind or schema drifted") + _exact_typed_mapping( + root["launch_policy"], + SEALED_LAUNCH_POLICY, + context="calibration runtime launch policy", + ) + + python_record = root["python"] + if not isinstance(python_record, dict): + raise ValueError("calibration runtime python record must be an object") + _exact_fields( + python_record, + {"abi_flags", "cache_tag", "implementation", "version"}, + context="calibration runtime python record", + ) + for field in ("abi_flags", "cache_tag", "implementation", "version"): + value = python_record[field] + invalid_empty = field != "abi_flags" and (not value or value != value.strip()) + if not isinstance(value, str) or invalid_empty: + raise ValueError(f"calibration runtime python {field} is invalid") + + machine = root["machine"] + if not isinstance(machine, dict): + raise ValueError("calibration runtime machine record must be an object") + _exact_fields( + machine, + {"architecture", "byteorder", "machine", "pointer_bits", "system"}, + context="calibration runtime machine record", + ) + for field in ("architecture", "machine", "system"): + value = machine[field] + if not isinstance(value, str) or not value or value != value.strip(): + raise ValueError(f"calibration runtime machine {field} is invalid") + if machine["byteorder"] not in {"little", "big"}: + raise ValueError("calibration runtime machine byteorder is invalid") + pointer_bits = _positive_int(machine["pointer_bits"], context="machine pointer_bits") + + base_root = _runtime_root_name(root["base_runtime_root"], context="base runtime root") + if base_root != BASE_RUNTIME_ROOT_NAME: + raise ValueError("base runtime root name drifted") + raw_base_sys_path = root["base_sys_path"] + if not isinstance(raw_base_sys_path, list) or not raw_base_sys_path: + raise ValueError("base_sys_path must be a non-empty list") + base_sys_path = tuple( + _canonical_relative_path(item, context="base sys.path entry") + for item in raw_base_sys_path + ) + if len({item.casefold() for item in base_sys_path}) != len(base_sys_path): + raise ValueError("base_sys_path entries must be case-insensitively unique") + raw_package_roots = root["package_roots"] + if not isinstance(raw_package_roots, list) or not raw_package_roots: + raise ValueError("package_roots must be a non-empty list") + parsed_package_roots: list[RuntimePackageRootRecord] = [] + for index, item in enumerate(raw_package_roots): + if not isinstance(item, dict): + raise ValueError(f"package_roots[{index}] must be an object") + _exact_fields( + item, + {"import_path", "name"}, + context=f"package_roots[{index}]", + ) + parsed_package_roots.append( + RuntimePackageRootRecord( + name=_runtime_root_name(item["name"], context="package root name"), + import_path=_canonical_relative_path( + item["import_path"], + context="package root import path", + ), + ) + ) + package_root_names = tuple(item.name for item in parsed_package_roots) + if ( + list(package_root_names) != sorted(package_root_names) + or len(set(package_root_names)) != len(package_root_names) + or base_root in package_root_names + ): + raise ValueError("package_roots must be unique, sorted, and distinct from base runtime") + + raw_trees = root["runtime_trees"] + if not isinstance(raw_trees, list): + raise ValueError("runtime_trees must be a list") + trees = tuple(_parse_runtime_tree(item, index=index) for index, item in enumerate(raw_trees)) + if tuple(item.name for item in trees) != (base_root, *package_root_names): + raise ValueError("runtime tree order or exact root inventory drifted") + if trees[0].kind != "base-runtime" or any(tree.kind != "packages" for tree in trees[1:]): + raise ValueError("runtime tree kinds drifted") + + interpreter = root["interpreter"] + if not isinstance(interpreter, dict): + raise ValueError("calibration runtime interpreter record must be an object") + _exact_fields( + interpreter, + {"relative_path", "root", "sha256", "size_bytes"}, + context="runtime interpreter", + ) + if interpreter["root"] != base_root: + raise ValueError("runtime interpreter is not bound to the base runtime root") + interpreter_path = _canonical_relative_path( + interpreter["relative_path"], context="runtime interpreter relative path" + ) + interpreter_sha256 = _sha256( + interpreter["sha256"], context="runtime interpreter SHA-256" + ) + interpreter_size = _positive_int( + interpreter["size_bytes"], context="runtime interpreter size" + ) + base_files = {item.path: item for item in trees[0].files} + for sys_path_entry in base_sys_path: + present = sys_path_entry in base_files or any( + path.startswith(f"{sys_path_entry}/") for path in base_files + ) + optional_zip = re.fullmatch(r"python[0-9]+\.zip", sys_path_entry) is not None + if not present and not optional_zip: + raise ValueError("base_sys_path entry is absent from the complete base tree") + if base_files.get(interpreter_path) != RuntimeFileRecord( + interpreter_path, + interpreter_size, + interpreter_sha256, + ): + raise ValueError("runtime interpreter identity differs from the complete base tree") + + raw_distributions = root["distributions"] + if not isinstance(raw_distributions, list) or not raw_distributions: + raise ValueError("calibration runtime distributions must be a non-empty list") + parsed: list[RuntimeDistributionRecord] = [] + ownership: dict[str, set[str]] = {name: set() for name in package_root_names} + for index, raw_distribution in enumerate(raw_distributions): + if not isinstance(raw_distribution, dict): + raise ValueError(f"runtime distributions[{index}] must be an object") + _exact_fields( + raw_distribution, + {"files", "name", "package_root", "version"}, + context=f"runtime distributions[{index}]", + ) + name = _normalized_distribution_name(raw_distribution["name"]) + if name != raw_distribution["name"]: + raise ValueError("runtime distribution names must already be canonical") + version = raw_distribution["version"] + if not isinstance(version, str) or not version or version != version.strip(): + raise ValueError(f"runtime distribution {name} version is invalid") + package_root = _runtime_root_name( + raw_distribution["package_root"], + context=f"runtime distribution {name} package_root", + ) + if package_root not in ownership: + raise ValueError(f"runtime distribution {name} uses an unknown package root") + raw_files = raw_distribution["files"] + if not isinstance(raw_files, list) or not raw_files: + raise ValueError(f"runtime distribution {name} files must be non-empty") + files = tuple( + _canonical_relative_path(path, context=f"runtime distribution {name} file path") + for path in raw_files + ) + if list(files) != sorted(files) or len({path.casefold() for path in files}) != len(files): + raise ValueError(f"runtime distribution {name} files must be unique and sorted") + overlap = ownership[package_root].intersection(files) + if overlap: + raise ValueError( + f"runtime package file has duplicate distribution ownership: {sorted(overlap)}" + ) + ownership[package_root].update(files) + parsed.append(RuntimeDistributionRecord(name, cast(str, version), package_root, files)) + if [item.name for item in parsed] != sorted(item.name for item in parsed) or len( + {item.name for item in parsed} + ) != len(parsed): + raise ValueError("runtime distributions must be unique and sorted") + tree_by_name = {item.name: item for item in trees} + for root_name in package_root_names: + if ownership[root_name] != {item.path for item in tree_by_name[root_name].files}: + raise ValueError( + f"package tree {root_name} differs from its complete distribution inventory" + ) + + return CalibrationRuntimeManifest( + python_implementation=cast(str, python_record["implementation"]), + python_version=cast(str, python_record["version"]), + python_cache_tag=cast(str, python_record["cache_tag"]), + python_abi_flags=cast(str, python_record["abi_flags"]), + machine_system=cast(str, machine["system"]), + machine_architecture=cast(str, machine["architecture"]), + machine_name=cast(str, machine["machine"]), + machine_byteorder=cast(str, machine["byteorder"]), + machine_pointer_bits=pointer_bits, + launch_policy=dict(SEALED_LAUNCH_POLICY), + base_sys_path=base_sys_path, + base_runtime_root=base_root, + package_roots=tuple(parsed_package_roots), + interpreter_root=base_root, + interpreter_relative_path=interpreter_path, + interpreter_size_bytes=interpreter_size, + interpreter_sha256=interpreter_sha256, + runtime_trees=trees, + distributions=tuple(parsed), + file_sha256=sha256_bytes(data), + ) + + +def authenticate_calibration_runtime( + manifest: CalibrationRuntimeManifest, + *, + base_runtime_root: Path, + package_roots: Mapping[str, Path], + distributions: Sequence[Any] | None = None, + interpreter_path: Path | None = None, +) -> AuthenticatedRuntime: + """Rehash both complete staged trees and exact RECORD inventories.""" + + if not isinstance(manifest, CalibrationRuntimeManifest): + raise TypeError("manifest must be CalibrationRuntimeManifest") + roots = _runtime_root_map(Path(base_runtime_root), package_roots) + packages = {name: roots[name] for name in sorted(roots) if name != BASE_RUNTIME_ROOT_NAME} + manifest_package_names = tuple(item.name for item in manifest.package_roots) + if tuple(packages) != manifest_package_names: + raise CalibrationRunError("point-used package roots differ from the frozen manifest") + import_paths = _normalized_package_import_paths( + packages, + {item.name: item.import_path for item in manifest.package_roots}, + ) + if ( + manifest.python_implementation != platform.python_implementation() + or manifest.python_version != platform.python_version() + or manifest.python_cache_tag != sys.implementation.cache_tag + or manifest.python_abi_flags != getattr(sys, "abiflags", "") + ): + raise CalibrationRunError("Python runtime differs from the frozen runtime manifest") + if _current_machine_identity() != ( + manifest.machine_system, + manifest.machine_architecture, + manifest.machine_name, + manifest.machine_byteorder, + manifest.machine_pointer_bits, + ): + raise CalibrationRunError("machine identity differs from the frozen runtime manifest") + + actual_trees: list[RuntimeTreeRecord] = [ + RuntimeTreeRecord( + BASE_RUNTIME_ROOT_NAME, + "base-runtime", + _runtime_tree_files(roots[BASE_RUNTIME_ROOT_NAME], kind="base-runtime"), + ) + ] + actual_trees.extend( + RuntimeTreeRecord( + name, + "packages", + _runtime_tree_files(path, kind=f"package root {name}"), + ) + for name, path in packages.items() + ) + if tuple(actual_trees) != manifest.runtime_trees: + raise CalibrationRunError("complete staged runtime tree identity drifted") + + expected_interpreter = _assert_no_link_components( + roots[BASE_RUNTIME_ROOT_NAME], + PurePosixPath(manifest.interpreter_relative_path), + ) + actual_interpreter = ( + Path(sys.executable).resolve(strict=True) + if interpreter_path is None + else Path(interpreter_path).resolve(strict=True) + ) + if actual_interpreter != expected_interpreter: + raise CalibrationRunError("point-used interpreter path differs from the runtime manifest") + interpreter_sha256, interpreter_size = _stream_file_sha256(actual_interpreter) + if ( + interpreter_sha256 != manifest.interpreter_sha256 + or interpreter_size != manifest.interpreter_size_bytes + ): + raise CalibrationRunError("Python interpreter bytes differ from the runtime manifest") + + installed = _installed_distribution_map( + distributions, + package_roots=packages, + package_import_paths=import_paths, + ) + if sorted(installed) != [item.name for item in manifest.distributions]: + raise CalibrationRunError("installed distribution set differs from the frozen manifest") + for expected in manifest.distributions: + actual = _distribution_record( + installed[expected.name], + name=expected.name, + package_roots=packages, + package_import_paths=import_paths, + ) + if actual != expected: + raise CalibrationRunError(f"installed distribution drifted: {expected.name}") + total_files = sum(len(tree.files) for tree in manifest.runtime_trees) + return AuthenticatedRuntime( + manifest_file_sha256=manifest.file_sha256, + python_implementation=manifest.python_implementation, + python_version=manifest.python_version, + python_cache_tag=manifest.python_cache_tag, + interpreter_sha256=manifest.interpreter_sha256, + machine_name=manifest.machine_name, + base_runtime_file_count=len(manifest.runtime_trees[0].files), + package_root_count=len(manifest.package_roots), + distributions=tuple((item.name, item.version) for item in manifest.distributions), + distribution_count=len(manifest.distributions), + file_count=total_files, + ) + + +def _current_machine_identity() -> tuple[str, str, str, str, int]: + pointer_bits = 8 * struct.calcsize("P") + return ( + platform.system(), + f"{pointer_bits}bit", + platform.machine(), + sys.byteorder, + pointer_bits, + ) + + +def _parse_named_path_arguments(values: Sequence[str], *, context: str) -> dict[str, Path]: + result: dict[str, Path] = {} + for value in values: + if not isinstance(value, str) or value.count("=") != 1: + raise CalibrationRunError(f"{context} must use NAME=PATH") + raw_name, raw_path = value.split("=", 1) + name = _runtime_root_name(raw_name, context=f"{context} name") + if name in result or not raw_path or raw_path != raw_path.strip(): + raise CalibrationRunError(f"{context} contains a duplicate name or invalid path") + path = Path(raw_path) + if not path.is_absolute(): + raise CalibrationRunError(f"{context} paths must be absolute") + result[name] = path + if not result: + raise CalibrationRunError(f"{context} must contain at least one entry") + return {name: result[name] for name in sorted(result)} + + +def _parse_named_import_path_arguments( + values: Sequence[str], + *, + context: str, +) -> dict[str, str]: + result: dict[str, str] = {} + for value in values: + if not isinstance(value, str) or value.count("=") != 1: + raise CalibrationRunError(f"{context} must use NAME=RELATIVE_PATH") + raw_name, raw_path = value.split("=", 1) + name = _runtime_root_name(raw_name, context=f"{context} name") + if name in result: + raise CalibrationRunError(f"{context} contains a duplicate name") + result[name] = _canonical_relative_path( + raw_path, + context=f"{context} relative path", + ) + if not result: + raise CalibrationRunError(f"{context} must contain at least one entry") + return {name: result[name] for name in sorted(result)} + + +def _read_pyvenv_config(path: Path) -> dict[str, str]: + if _is_link_or_reparse(path): + raise CalibrationRunError("source pyvenv.cfg is a link or reparse point") + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + raise CalibrationRunError("source pyvenv.cfg is not readable UTF-8") from exc + result: dict[str, str] = {} + for index, raw_line in enumerate(text.splitlines(), start=1): + line = raw_line.strip() + if not line: + continue + if "=" not in line: + raise CalibrationRunError(f"source pyvenv.cfg line {index} is malformed") + raw_key, raw_value = line.split("=", 1) + key = raw_key.strip().casefold() + value = raw_value.strip() + if not key or not value or key in result: + raise CalibrationRunError("source pyvenv.cfg has a duplicate or empty field") + result[key] = value + return result + + +def _source_runtime_layout(source_python: Path) -> tuple[Path, Path, str]: + executable = Path(os.path.abspath(source_python)) + if _is_link_or_reparse(executable): + raise CalibrationRunError("source interpreter is a link or reparse point") + try: + executable = executable.resolve(strict=True) + except OSError as exc: + raise CalibrationRunError("source interpreter is unavailable") from exc + if not executable.is_file(): + raise CalibrationRunError("source interpreter is not a regular file") + + virtual_root = executable.parent.parent + config_path = virtual_root / "pyvenv.cfg" + if executable.parent.name.casefold() == "scripts" and config_path.is_file(): + _assert_no_link_components(virtual_root, PurePosixPath("pyvenv.cfg")) + config = _read_pyvenv_config(config_path) + raw_home = config.get("home") + if raw_home is None or not Path(raw_home).is_absolute(): + raise CalibrationRunError("source pyvenv.cfg has no absolute base home") + try: + physical_home = Path(raw_home).resolve(strict=True) + except OSError as exc: + raise CalibrationRunError("source pyvenv.cfg base home is unavailable") from exc + base_root = _absolute_runtime_root(physical_home, context="source base runtime root") + package_root = _absolute_runtime_root(virtual_root, context="source virtualenv root") + else: + base_root = _absolute_runtime_root(executable.parent, context="source base runtime root") + package_root = base_root + import_relative = DEFAULT_PACKAGE_IMPORT_PATH + import_root = package_root / PurePosixPath(import_relative) + if not import_root.is_dir() or _is_link_or_reparse(import_root): + raise CalibrationRunError("source environment has no regular Lib/site-packages") + return base_root, package_root, import_relative + + +def _wheel_requirement_version(name: str, url: str) -> str: + parsed = urllib.parse.urlsplit(url) + fragment = urllib.parse.parse_qs(parsed.fragment, strict_parsing=True) + hashes = fragment.get("sha256") + if parsed.scheme != "https" or hashes is None or len(hashes) != 1: + raise CalibrationRunError(f"direct requirement {name} must have one HTTPS SHA-256") + _sha256(hashes[0], context=f"direct requirement {name} SHA-256") + filename = PurePosixPath(urllib.parse.unquote(parsed.path)).name + match = re.fullmatch( + r"(?P.+)-(?P[^-]+)-[^-]+-[^-]+-[^-]+\.whl", + filename, + ) + if match is None or _normalized_distribution_name(match.group("distribution")) != name: + raise CalibrationRunError(f"direct requirement {name} has an invalid wheel URL") + return match.group("version") + + +def _parse_runtime_requirements(path: Path) -> tuple[RuntimeRequirement, ...]: + requirements_path = Path(os.path.abspath(path)) + if _is_link_or_reparse(requirements_path): + raise CalibrationRunError("runtime requirements file is a link or reparse point") + try: + text = requirements_path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + raise CalibrationRunError("runtime requirements file is not readable UTF-8") from exc + selected: dict[str, RuntimeRequirement] = {} + for line_number, raw_line in enumerate(text.splitlines(), start=1): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + pinned = re.fullmatch(r"([A-Za-z0-9][A-Za-z0-9_.-]*)==([^\s;]+)", line) + direct = re.fullmatch(r"([A-Za-z0-9][A-Za-z0-9_.-]*)\s*@\s*(\S+)", line) + if pinned is not None: + name = _normalized_distribution_name(pinned.group(1)) + version = pinned.group(2) + elif direct is not None: + name = _normalized_distribution_name(direct.group(1)) + version = _wheel_requirement_version(name, direct.group(2)) + else: + raise CalibrationRunError( + f"runtime requirement line {line_number} is not one exact pin" + ) + if name in selected: + raise CalibrationRunError(f"runtime requirements repeat distribution {name}") + if name not in STAGING_EXCLUDED_DISTRIBUTIONS: + selected[name] = RuntimeRequirement(name=name, version=version) + if not selected: + raise CalibrationRunError("runtime requirements select no loadable distributions") + return tuple(selected[name] for name in sorted(selected)) + + +def _source_regular_file(root: Path, relative: PurePosixPath) -> Path: + candidate = root + if _is_link_or_reparse(candidate): + raise CalibrationRunError("source runtime root is a link or reparse point") + for part in relative.parts: + candidate /= part + if _is_link_or_reparse(candidate): + raise CalibrationRunError( + f"source runtime path traverses a link or reparse point: {relative.as_posix()}" + ) + try: + resolved = candidate.resolve(strict=True) + resolved.relative_to(root) + except (OSError, ValueError) as exc: + raise CalibrationRunError( + f"source runtime path escapes its explicit root: {relative.as_posix()}" + ) from exc + if not resolved.is_file(): + raise CalibrationRunError( + f"source runtime path is not a regular file: {relative.as_posix()}" + ) + return resolved + + +def _copy_independent_runtime_file(source: Path, destination: Path) -> None: + if destination.exists(): + raise CalibrationRunError(f"runtime staging path collision: {destination.name}") + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + if os.path.samefile(source, destination): + raise CalibrationRunError("runtime staging created a shared file instead of copied bytes") + source_sha256, source_size = _stream_file_sha256(source) + destination_sha256, destination_size = _stream_file_sha256(destination) + if (source_sha256, source_size) != (destination_sha256, destination_size): + raise CalibrationRunError("runtime staging changed copied file bytes") + + +def _copy_base_runtime(source_root: Path, destination_root: Path) -> None: + selected_directories = frozenset({"dlls", "lib"}) + stack: list[tuple[Path, tuple[str, ...]]] = [(source_root, ())] + copied = 0 + while stack: + directory, parts = stack.pop() + try: + entries = sorted(os.scandir(directory), key=lambda item: item.name) + except OSError as exc: + raise CalibrationRunError("cannot enumerate source base runtime") from exc + for entry in entries: + child_parts = (*parts, entry.name) + relative = PurePosixPath(*child_parts).as_posix() + try: + status = entry.stat(follow_symlinks=False) + except OSError as exc: + raise CalibrationRunError("source base runtime changed during staging") from exc + if entry.is_symlink() or ( + getattr(status, "st_file_attributes", 0) & _WINDOWS_REPARSE_POINT + ): + raise CalibrationRunError( + f"source base runtime contains a link or reparse point: {relative}" + ) + if stat.S_ISDIR(status.st_mode): + if not parts and entry.name.casefold() not in selected_directories: + continue + if ( + len(parts) == 1 + and parts[0].casefold() == "lib" + and entry.name.casefold() == "site-packages" + ): + continue + if entry.name.casefold() in _FORBIDDEN_RUNTIME_DIRECTORY_NAMES: + continue + stack.append((Path(entry.path), child_parts)) + continue + if not stat.S_ISREG(status.st_mode): + raise CalibrationRunError(f"source base runtime path is not regular: {relative}") + if _runtime_path_is_forbidden(relative): + continue + source = _source_regular_file(source_root, PurePosixPath(*child_parts)) + _copy_independent_runtime_file(source, destination_root / Path(*child_parts)) + copied += 1 + if copied == 0: + raise CalibrationRunError("source base runtime contributed no staged files") + + +def _selected_source_distributions( + source_package_root: Path, + import_path: str, + requirements: Sequence[RuntimeRequirement], +) -> tuple[Any, ...]: + import_root = source_package_root / PurePosixPath(import_path) + installed = _installed_distribution_map( + list(importlib.metadata.distributions(path=[str(import_root)])) + ) + result: list[Any] = [] + for requirement in requirements: + distribution = installed.get(requirement.name) + if distribution is None: + raise CalibrationRunError( + f"source environment lacks required distribution {requirement.name}" + ) + if str(distribution.version) != requirement.version: + raise CalibrationRunError( + f"source distribution version differs from exact pin: {requirement.name}" + ) + result.append(distribution) + return tuple(result) + + +def _copy_record_only_packages( + source_root: Path, + destination_root: Path, + distributions: Sequence[Any], +) -> None: + copied: set[str] = set() + for distribution in distributions: + name = _normalized_distribution_name(distribution.metadata.get("Name")) + for record_path in _record_paths(distribution, name=name): + located = Path(distribution.locate_file(record_path)) + try: + resolved = located.resolve(strict=True) + relative = resolved.relative_to(source_root) + except (OSError, ValueError) as exc: + raise CalibrationRunError( + f"distribution {name} RECORD path escapes the source environment" + ) from exc + posix_relative = _canonical_relative_path( + relative.as_posix(), + context=f"distribution {name} staged path", + ) + if _runtime_path_is_forbidden(posix_relative): + raise CalibrationRunError( + f"distribution {name} owns a forbidden startup or bytecode file" + ) + folded = posix_relative.casefold() + if folded in copied: + raise CalibrationRunError( + f"selected distributions have duplicate RECORD ownership: {posix_relative}" + ) + source = _source_regular_file(source_root, PurePosixPath(posix_relative)) + _copy_independent_runtime_file( + source, + destination_root / Path(*PurePosixPath(posix_relative).parts), + ) + copied.add(folded) + if not copied: + raise CalibrationRunError("selected distributions contributed no staged package files") + + +_RUNTIME_PROBE_SOURCE: Final = r''' +import json +import os +import platform +import struct +import sys + +root = os.path.realpath(sys.argv[1]) +if os.path.realpath(sys.prefix) != root or os.path.realpath(sys.base_prefix) != root: + raise SystemExit("prefix drift") +if "site" in sys.modules or "_virtualenv" in sys.modules: + raise SystemExit("startup hook loaded") +if not ( + sys.flags.isolated == 1 + and sys.flags.ignore_environment == 1 + and sys.flags.no_user_site == 1 + and sys.flags.no_site == 1 + and sys.flags.dont_write_bytecode == 1 + and sys.flags.utf8_mode == 1 + and sys.flags.safe_path is True +): + raise SystemExit("flag drift") +if sys._xoptions != {"utf8": True}: + raise SystemExit("xoption drift") +paths = [] +for item in sys.path: + if not isinstance(item, str) or not item or not os.path.isabs(item): + raise SystemExit("relative sys.path") + relative = os.path.relpath(os.path.realpath(item), root) + if relative == os.pardir or relative.startswith(os.pardir + os.sep): + raise SystemExit("sys.path escape") + paths.append(relative.replace(os.sep, "/")) +bits = 8 * struct.calcsize("P") +payload = { + "base_sys_path": paths, + "machine": { + "architecture": f"{bits}bit", + "byteorder": sys.byteorder, + "machine": platform.machine(), + "pointer_bits": bits, + "system": platform.system(), + }, + "python": { + "abi_flags": getattr(sys, "abiflags", ""), + "cache_tag": sys.implementation.cache_tag, + "implementation": platform.python_implementation(), + "version": platform.python_version(), + }, +} +print(json.dumps(payload, allow_nan=False, sort_keys=True, separators=(",", ":"))) +'''.strip() + + +def _probe_staged_interpreter( + interpreter: Path, + base_runtime_root: Path, +) -> RuntimeInterpreterProbe: + process = subprocess.run( + [ + str(interpreter), + "-I", + "-S", + "-B", + "-X", + "utf8", + "-c", + _RUNTIME_PROBE_SOURCE, + str(base_runtime_root), + ], + check=False, + capture_output=True, + timeout=60, + ) + if process.returncode != 0 or process.stderr: + raise CalibrationRunError("staged interpreter failed the isolated stdlib probe") + root = _strict_json_bytes(process.stdout, context="staged interpreter probe") + _exact_fields(root, {"base_sys_path", "machine", "python"}, context="runtime probe") + raw_python = root["python"] + raw_machine = root["machine"] + raw_paths = root["base_sys_path"] + if not isinstance(raw_python, dict) or not isinstance(raw_machine, dict): + raise CalibrationRunError("staged interpreter probe identity is malformed") + if not isinstance(raw_paths, list) or not raw_paths: + raise CalibrationRunError("staged interpreter probe sys.path is malformed") + _exact_fields( + raw_python, + {"abi_flags", "cache_tag", "implementation", "version"}, + context="runtime probe python", + ) + _exact_fields( + raw_machine, + {"architecture", "byteorder", "machine", "pointer_bits", "system"}, + context="runtime probe machine", + ) + return RuntimeInterpreterProbe( + python_implementation=cast(str, raw_python["implementation"]), + python_version=cast(str, raw_python["version"]), + python_cache_tag=cast(str, raw_python["cache_tag"]), + python_abi_flags=cast(str, raw_python["abi_flags"]), + machine_system=cast(str, raw_machine["system"]), + machine_architecture=cast(str, raw_machine["architecture"]), + machine_name=cast(str, raw_machine["machine"]), + machine_byteorder=cast(str, raw_machine["byteorder"]), + machine_pointer_bits=_positive_int( + raw_machine["pointer_bits"], + context="runtime probe pointer_bits", + ), + base_sys_path=tuple( + _canonical_relative_path(item, context="runtime probe base sys.path") + for item in raw_paths + ), + ) + + +def prepare_calibration_runtime( + *, + source_python: Path, + requirements_file: Path, + output_root: Path, + package_root_name: str = DEFAULT_PACKAGE_RUNTIME_ROOT_NAME, +) -> dict[str, object]: + """Stage independent base bytes and only exact wheel-RECORD package bytes.""" + + name = _runtime_root_name(package_root_name, context="prepared package root name") + if name == BASE_RUNTIME_ROOT_NAME: + raise CalibrationRunError("prepared package root name is reserved") + destination = Path(os.path.abspath(output_root)) + if destination.exists(): + raise FileExistsError(f"refusing to overwrite prepared runtime: {destination}") + destination.parent.mkdir(parents=True, exist_ok=True) + source_base, source_packages, import_path = _source_runtime_layout(source_python) + requirements = _parse_runtime_requirements(requirements_file) + distributions = _selected_source_distributions( + source_packages, + import_path, + requirements, + ) + + prefix = f".{destination.name}.staging-" + staging = Path(tempfile.mkdtemp(prefix=prefix, dir=destination.parent)) + owned_staging = True + try: + base_root = staging / BASE_RUNTIME_ROOT_NAME + package_root = staging / name + base_root.mkdir() + package_root.mkdir() + _copy_base_runtime(source_base, base_root) + _copy_record_only_packages(source_packages, package_root, distributions) + source_executable_name = Path(source_python).name + staged_interpreter = base_root / source_executable_name + if not staged_interpreter.is_file(): + raise CalibrationRunError("staged base runtime omitted the selected interpreter") + probe = _probe_staged_interpreter(staged_interpreter, base_root) + package_roots = {name: package_root} + package_import_paths = {name: import_path} + payload = capture_calibration_runtime_manifest( + base_runtime_root=base_root, + package_roots=package_roots, + package_import_paths=package_import_paths, + interpreter_path=staged_interpreter, + runtime_probe=probe, + ) + manifest_path = staging / PREPARED_RUNTIME_MANIFEST_FILENAME + _atomic_publish_new(manifest_path, payload) + if manifest_path.read_bytes() != payload: + raise CalibrationRunError("prepared runtime manifest changed after publication") + repeated = capture_calibration_runtime_manifest( + base_runtime_root=base_root, + package_roots=package_roots, + package_import_paths=package_import_paths, + interpreter_path=staged_interpreter, + runtime_probe=probe, + ) + if repeated != payload: + raise CalibrationRunError("prepared runtime trees changed before publication") + _atomic_publish_new( + staging / PREPARED_RUNTIME_COMPLETE_FILENAME, + b"recurquant-experiment013-runtime-prepared-v1\n", + ) + if destination.exists(): + raise FileExistsError(f"refusing to overwrite prepared runtime: {destination}") + staging.rename(destination) + owned_staging = False + finally: + if owned_staging: + try: + staging.relative_to(destination.parent) + except ValueError as exc: + raise RuntimeError("owned runtime staging directory escaped its parent") from exc + if not staging.name.startswith(prefix): + raise RuntimeError("owned runtime staging directory name drifted") + shutil.rmtree(staging, ignore_errors=False) + return { + "base_runtime_root": str(destination / BASE_RUNTIME_ROOT_NAME), + "excluded_distributions": sorted(STAGING_EXCLUDED_DISTRIBUTIONS), + "manifest_file_sha256": sha256_bytes(payload), + "package_import_path": import_path, + "package_root": str(destination / name), + "package_root_name": name, + "prepared_runtime_root": str(destination), + "status": "prepared_record_only_runtime", + } + + +def _verify_empty_pycache_prefix(path: Path) -> Path: + root = _absolute_runtime_root(path, context="pycache prefix") + try: + entries = list(os.scandir(root)) + except OSError as exc: + raise CalibrationRunError("cannot enumerate pycache prefix") from exc + if entries: + raise CalibrationRunError("pycache prefix is not empty") + return root + + +def _verify_sealed_launch_state( + pycache_prefix: Path, + *, + manifest: CalibrationRuntimeManifest, + roots: Mapping[str, Path], +) -> Path: + if ( + sys.flags.isolated != 1 + or sys.flags.ignore_environment != 1 + or sys.flags.no_user_site != 1 + or sys.flags.no_site != 1 + or sys.flags.dont_write_bytecode != 1 + or sys.flags.utf8_mode != 1 + or sys.flags.safe_path is not True + ): + raise CalibrationRunError("sealed Python startup flags drifted") + if "site" in sys.modules or "_virtualenv" in sys.modules: + raise CalibrationRunError("site or virtualenv startup hook is loaded") + if sys.pycache_prefix is None: + raise CalibrationRunError("sealed Python startup has no pycache prefix") + expected = Path(os.path.abspath(pycache_prefix)) + actual = Path(os.path.abspath(sys.pycache_prefix)) + if actual != expected: + raise CalibrationRunError("sealed Python pycache prefix drifted") + if set(sys._xoptions) != {"pycache_prefix", "utf8"} or ( + Path(os.path.abspath(str(sys._xoptions["pycache_prefix"]))) != expected + or sys._xoptions["utf8"] is not True + ): + raise CalibrationRunError("sealed Python -X options drifted") + base_root = roots[BASE_RUNTIME_ROOT_NAME] + if ( + Path(sys.prefix).resolve(strict=True) != base_root + or Path(sys.base_prefix).resolve(strict=True) != base_root + ): + raise CalibrationRunError("sealed Python prefix differs from the staged base runtime") + expected_sys_path = [ + str(base_root / PurePosixPath(relative)) for relative in manifest.base_sys_path + ] + expected_sys_path.extend( + str(roots[item.name] / PurePosixPath(item.import_path)) + for item in manifest.package_roots + ) + if [os.path.abspath(item) for item in sys.path] != [ + os.path.abspath(item) for item in expected_sys_path + ]: + raise CalibrationRunError("sealed Python sys.path differs from authenticated roots") + return _verify_empty_pycache_prefix(expected) + + +def _authenticate_sealed_runtime_context( + runtime_manifest_bytes: bytes, + *, + base_runtime_root: Path, + package_roots: Mapping[str, Path], + package_import_paths: Mapping[str, str], + interpreter_path: Path, + pycache_prefix: Path, +) -> tuple[CalibrationRuntimeManifest, SealedRuntimeContext, AuthenticatedRuntime]: + """Reauthenticate explicit launcher inputs without copying them to globals.""" + + manifest = parse_calibration_runtime_manifest(runtime_manifest_bytes) + roots = _runtime_root_map(base_runtime_root, package_roots) + declared_names = tuple(item.name for item in manifest.package_roots) + actual_names = tuple(name for name in roots if name != BASE_RUNTIME_ROOT_NAME) + if actual_names != declared_names: + raise CalibrationRunError( + "bootstrap package roots differ from the frozen runtime manifest" + ) + normalized_import_paths = _normalized_package_import_paths( + {item.name: roots[item.name] for item in manifest.package_roots}, + package_import_paths, + ) + frozen_import_paths = {item.name: item.import_path for item in manifest.package_roots} + if normalized_import_paths != frozen_import_paths: + raise CalibrationRunError( + "bootstrap package import paths differ from the frozen runtime manifest" + ) + verified_pycache = _verify_sealed_launch_state( + pycache_prefix, + manifest=manifest, + roots=roots, + ) + authenticated = authenticate_calibration_runtime( + manifest, + base_runtime_root=base_runtime_root, + package_roots=package_roots, + interpreter_path=interpreter_path, + ) + context = SealedRuntimeContext( + manifest_file_sha256=manifest.file_sha256, + base_runtime_root=roots[BASE_RUNTIME_ROOT_NAME], + package_roots={item.name: roots[item.name] for item in manifest.package_roots}, + package_import_paths=normalized_import_paths, + pycache_prefix=verified_pycache, + ) + return manifest, context, authenticated + + +def _load_identity_resolver(repository_root: Path) -> Any: + module = _AUTHENTICATED_IDENTITY_RESOLVER + if module is None: + raise CalibrationRunError("identity resolver was not bootstrap-authenticated") + expected = (repository_root / IDENTITY_RESOLVER_SOURCE_PATH).resolve(strict=True) + actual = Path(cast(str, getattr(module, "__file__", ""))).resolve(strict=True) + if actual != expected: + raise CalibrationRunError("authenticated identity resolver path drifted") + return module + + +def _identity_view(data: bytes, repository_root: Path) -> FrozenCalibrationIdentity: + resolver = _load_identity_resolver(repository_root) + decoded = resolver.deserialize_frozen_calibration_identity_artifact(data) + root = _strict_json_bytes(data, context="frozen calibration identity") + evidence = root.get("evidence") + if not isinstance(evidence, dict): # independently decoded above; defensive only + raise ValueError("frozen identity evidence is missing") + model_contracts = cast(dict[str, object], evidence["model_contracts"]) + primary = cast(dict[str, object], model_contracts["primary"]) + tokenizer = cast(dict[str, object], evidence["tokenizer"]) + execution_bindings = getattr(decoded, "execution_bindings", None) + if not isinstance(execution_bindings, Mapping): + raise CalibrationRunError( + "frozen identity does not contain schema-v4 execution_bindings; " + "runtime/model access remains unauthorized" + ) + _exact_fields( + execution_bindings, + { + "calibration_runtime_manifest_file_sha256", + "model_file_manifest_file_sha256", + "parquet_materialization_manifest_file_sha256", + "repository_source_manifest_file_sha256", + }, + context="frozen identity execution bindings", + ) + if evidence.get("execution_bindings") != dict(execution_bindings): + raise CalibrationRunError("decoded execution bindings differ from identity evidence") + return FrozenCalibrationIdentity( + file_sha256=decoded.file_sha256, + canonical_evidence_sha256=decoded.canonical_evidence_sha256, + records=tuple(dict(record) for record in decoded.records), + assignment=tuple(dict(item) for item in decoded.assignment), + assignment_sha256=decoded.assignment_sha256, + tokenizer_manifest_sha256=decoded.tokenizer_manifest_sha256, + identity_input_manifest_sha256=_sha256( + evidence["source_manifest_sha256"], + context="identity input manifest SHA-256", + ), + repository_source_manifest_file_sha256=_sha256( + execution_bindings["repository_source_manifest_file_sha256"], + context="identity repository source manifest file SHA-256", + ), + runtime_manifest_file_sha256=_sha256( + execution_bindings["calibration_runtime_manifest_file_sha256"], + context="identity runtime manifest file SHA-256", + ), + model_file_manifest_file_sha256=_sha256( + execution_bindings["model_file_manifest_file_sha256"], + context="identity model manifest file SHA-256", + ), + parquet_materialization_manifest_file_sha256=_sha256( + execution_bindings["parquet_materialization_manifest_file_sha256"], + context="identity parquet materialization manifest file SHA-256", + ), + model_id=cast(str, primary["id"]), + model_revision=_git_revision(primary["revision"], context="identity model revision"), + transformers_version=cast(str, tokenizer["transformers_version"]), + artifact_bytes=data, + ) + + +def verify_repository_source_manifest( + expected: Mapping[str, object], + repository_root: Path, +) -> tuple[dict[str, object], str]: + """Use the frozen source API to reauthenticate code at point of use.""" + + module = _AUTHENTICATED_SOURCE_VERIFIER + if module is None: + raise CalibrationRunError("repository source verifier was not bootstrap-authenticated") + normalized_expected = module.validate_experiment013_source_manifest(expected) + verified = module.verify_experiment013_source_manifest( + normalized_expected, + repo_root=repository_root, + ) + if verified != normalized_expected: + raise CalibrationRunError("repository source verification returned a different manifest") + normalized = dict(verified) + claimed = normalized.pop("canonical_manifest_sha256", None) + if claimed is None: + raise CalibrationRunError("repository source manifest is missing its canonical hash") + digest = module.canonical_experiment013_source_manifest_sha256(normalized) + if claimed != digest: + raise CalibrationRunError("repository source manifest self-hash drifted") + return dict(verified), digest + + +def validate_adapter_contract(adapter: Any, *, calibration_api: ModuleType) -> None: + """Validate the reviewed adapter structurally against the authenticated API.""" + + if not isinstance(adapter, calibration_api.CalibrationAdapter): + raise TypeError("reviewed adapter does not implement CalibrationAdapter") + + +def _verify_repository_commit(repository_root: Path, expected: str) -> str: + revision = _git_revision(expected, context="expected source commit") + process = subprocess.run( + ["git", "-C", str(repository_root), "rev-parse", "HEAD"], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + if process.returncode != 0: + raise CalibrationRunError("cannot resolve repository HEAD for policy provenance") + actual = process.stdout.strip() + if actual != revision: + raise CalibrationRunError( + f"repository HEAD differs from expected source commit: {actual!r} != {revision!r}" + ) + return revision + + +def _record_int(record: Mapping[str, object], name: str) -> int: + value = record.get(name) + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise CalibrationRunError(f"identity record {name} must be a non-negative integer") + return value + + +def _token_ids_sha256(token_ids: Sequence[int], *, allow_empty: bool = False) -> str: + values: list[int] = [] + for index, token_id in enumerate(token_ids): + if isinstance(token_id, bool) or not isinstance(token_id, int) or token_id < 0: + raise CalibrationRunError(f"token_ids[{index}] must be a non-negative integer") + values.append(token_id) + if not values and not allow_empty: + raise CalibrationRunError("materialized token sequence cannot be empty") + return sha256_bytes(canonical_json_bytes(values)) + + +def validate_materialized_sequence( + record: Mapping[str, object], + materialized: Any, + *, + calibration_api: ModuleType, +) -> tuple[int, ...]: + """Reauthenticate all sequence commitments without retaining source text.""" + + if not isinstance(materialized, calibration_api.AuthenticatedSequence): + raise TypeError("adapter must return AuthenticatedSequence") + token_ids = tuple(materialized.token_ids) + expected_length = _record_int(record, "sequence_length") + if len(token_ids) != expected_length: + raise CalibrationRunError("materialized sequence length differs from frozen identity") + if _token_ids_sha256(token_ids) != record.get("sequence_token_ids_sha256"): + raise CalibrationRunError("materialized token IDs differ from frozen identity") + span = record.get("token_span") + if not isinstance(span, Mapping): + raise CalibrationRunError("identity token_span is missing") + prefill_stop = span.get("prefill_stop") + scored_stop = span.get("scored_stop") + if ( + isinstance(prefill_stop, bool) + or not isinstance(prefill_stop, int) + or not 0 < prefill_stop <= len(token_ids) + or scored_stop != len(token_ids) + ): + raise CalibrationRunError("identity token span is invalid for materialized sequence") + if _token_ids_sha256(token_ids[:prefill_stop], allow_empty=True) != record.get( + "prompt_token_ids_sha256" + ): + raise CalibrationRunError("materialized prompt token IDs differ from frozen identity") + if _token_ids_sha256(token_ids[prefill_stop:], allow_empty=True) != record.get( + "target_token_ids_sha256" + ): + raise CalibrationRunError("materialized target token IDs differ from frozen identity") + exact = { + "source_content_sha256": materialized.source_content_sha256, + "formatted_content_sha256": materialized.formatted_content_sha256, + "generator_receipt_sha256": materialized.generator_receipt_sha256, + "tokenizer_manifest_sha256": materialized.tokenizer_manifest_sha256, + } + for name, actual in exact.items(): + if actual != record.get(name): + raise CalibrationRunError(f"materialized {name} differs from frozen identity") + return token_ids + + +def frozen_anchor_positions(token_count: int) -> tuple[int, ...]: + tokens = _positive_int(token_count, context="token_count") + if tokens < 16: + return tuple(range(tokens)) + positions = tuple((index + 1) * tokens // 16 - 1 for index in range(16)) + if len(set(positions)) != len(positions): + raise RuntimeError("frozen anchor equation produced duplicate positions") + return positions + + +def compute_anchor_distortions( + state: Any, + geometry: Geometry, +) -> tuple[Any, Any, Any]: + """Return per-row RHT Q4/Q6/Q8 MSE, releasing layer workspaces eagerly.""" + + torch = _torch_runtime() + from recurquant.quantization import QuantizationSpec, quantize_dequantize + from recurquant.rht import right_rht_encode + + expected = ( + geometry.layers, + geometry.heads, + geometry.key_rows, + geometry.value_width, + ) + if not isinstance(state, torch.Tensor) or tuple(state.shape) != expected: + raise CalibrationRunError(f"anchor state must have shape {expected}") + if not state.is_floating_point() or not torch.isfinite(state).all().item(): + raise CalibrationRunError("anchor state must be finite floating point") + specifications = tuple( + QuantizationSpec( + bits=bits, + group_size=geometry.value_width, + scale_bits=16, + flatten_last_dims=1, + rounding="nearest", + seed=RHT_SEED, + ) + for bits in (4, 6, 8) + ) + per_bit: list[list[Any]] = [[], [], []] + with torch.no_grad(): + for local_index, layer_index in enumerate(geometry.layer_indices): + encoded = right_rht_encode( + state[local_index].unsqueeze(0), + layer_index=layer_index, + expected_heads=geometry.heads, + output_dtype=torch.float32, + ) + for destination, specification in zip(per_bit, specifications, strict=True): + restored = quantize_dequantize(encoded, specification).tensor + mse = (restored - encoded).square().mean(dim=-1).squeeze(0) + destination.append(mse.detach().to(device="cpu", dtype=torch.float64)) + return cast( + tuple[Any, Any, Any], + tuple(torch.stack(rows, dim=0).contiguous() for rows in per_bit), + ) + + +def capture_sequence_causally( + adapter: Any, + model: object, + record: Mapping[str, object], + token_ids: tuple[int, ...], + *, + geometry: Geometry, + calibration_api: ModuleType, + require_cuda: bool, + distortion_function: DistortionFunction = compute_anchor_distortions, +) -> CapturedSequence: + """Process exactly one token per adapter call and retain only anchor tensors.""" + + torch = _torch_runtime() + anchors = frozen_anchor_positions(len(token_ids)) + anchor_set = set(anchors) + query_ema: Any | None = None + energies: list[Any] = [] + q4_rows: list[Any] = [] + q6_rows: list[Any] = [] + q8_rows: list[Any] = [] + adapter.begin_sequence(model, record) + completed = False + try: + for position, token_id in enumerate(token_ids): + capture_state = position in anchor_set + observation = adapter.step_token( + model, + token_id=token_id, + position=position, + capture_state=capture_state, + ) + if not isinstance(observation, calibration_api.StepObservation): + raise TypeError("adapter.step_token must return StepObservation") + if ( + observation.position != position + or observation.token_id != token_id + or observation.layer_indices != geometry.layer_indices + or observation.successful_kernel_calls_per_layer != (1,) * geometry.layers + ): + raise CalibrationRunError("adapter did not prove one successful causal kernel call") + query = observation.recurrence_query + expected_query_shape = (geometry.layers, geometry.heads, geometry.key_rows) + if ( + not isinstance(query, torch.Tensor) + or tuple(query.shape) != expected_query_shape + or not query.is_floating_point() + or not torch.isfinite(query).all().item() + ): + raise CalibrationRunError( + f"recurrence query must be finite floating point {expected_query_shape}" + ) + if require_cuda and query.device.type != "cuda": + raise CalibrationRunError("official recurrence queries must be actual CUDA tensors") + query32 = query.detach().to(torch.float32) + squared = query32.square() + energy = squared / (squared.sum(dim=-1, keepdim=True) + QUERY_ENERGY_EPSILON) + if query_ema is None: + query_ema = torch.full_like(energy, 1.0 / geometry.key_rows) + query_ema = QUERY_EMA_DECAY * query_ema + (1.0 - QUERY_EMA_DECAY) * energy + if not torch.isfinite(query_ema).all().item() or (query_ema < 0).any().item(): + raise CalibrationRunError("normalized-query-energy EMA became invalid") + if capture_state: + if observation.recurrent_state is None: + raise CalibrationRunError("adapter omitted recurrent state at a frozen anchor") + state = observation.recurrent_state + if not isinstance(state, torch.Tensor): + raise CalibrationRunError("adapter recurrent state must be a tensor") + if state.device != query.device: + raise CalibrationRunError( + "anchor query and recurrent state use different devices" + ) + if require_cuda and state.device.type != "cuda": + raise CalibrationRunError( + "official recurrent states must be actual CUDA tensors" + ) + if state.dtype != torch.float32: + raise CalibrationRunError("reference recurrent state must be FP32") + d4, d6, d8 = distortion_function(state, geometry) + expected_rows = (geometry.layers, geometry.heads, geometry.key_rows) + for name, tensor in ( + ("D4", d4), + ("D6", d6), + ("D8", d8), + ): + if ( + not isinstance(tensor, torch.Tensor) + or tuple(tensor.shape) != expected_rows + or tensor.device.type != "cpu" + or tensor.dtype != torch.float64 + or not torch.isfinite(tensor).all().item() + or (tensor < 0).any().item() + ): + raise CalibrationRunError( + f"{name} distortion must be finite non-negative CPU FP64 " + f"{expected_rows}" + ) + energies.append(query_ema.detach().to(device="cpu", dtype=torch.float64)) + q4_rows.append(d4) + q6_rows.append(d6) + q8_rows.append(d8) + elif observation.recurrent_state is not None: + raise CalibrationRunError("adapter retained/exposed full state outside an anchor") + completed = True + finally: + adapter.end_sequence(model, record) + if not completed or len(energies) != len(anchors): + raise CalibrationRunError("causal sequence capture did not complete every frozen anchor") + return CapturedSequence( + anchor_positions=anchors, + query_energy=torch.stack(energies).contiguous(), + q4_mse=torch.stack(q4_rows).contiguous(), + q6_mse=torch.stack(q6_rows).contiguous(), + q8_mse=torch.stack(q8_rows).contiguous(), + ) + + +def _stability_record(value: object) -> dict[str, object]: + checks = getattr(value, "checks", ()) + shifts = getattr(value, "layer_mean_bitwidth_shifts", ()) + spearman = getattr(value, "spearman_average_ties", None) + jaccard = getattr(value, "q8_jaccard", None) + passed = getattr(value, "passed", None) + if not isinstance(passed, bool): + raise TypeError("stability result must expose a boolean passed field") + return { + "checks": [{"name": str(name), "passed": bool(ok)} for name, ok in checks], + "layer_mean_bitwidth_shifts": [ + {"layer_index": int(layer), "shift_hex": float(shift).hex()} for layer, shift in shifts + ], + "passed": passed, + "q8_jaccard_hex": None if jaccard is None else float(jaccard).hex(), + "spearman_average_ties_hex": None if spearman is None else float(spearman).hex(), + } + + +class Experiment013Backend: + """Production wrapper around the frozen resolver, math, codecs, and gates.""" + + def __init__(self, repository_root: Path = REPOSITORY_ROOT) -> None: + self.repository_root = repository_root + self._geometry: Geometry | None = None + + @property + def geometry(self) -> Geometry: + if self._geometry is None: + from recurquant.static_q468 import FROZEN_QWEN35_STATIC_Q468_GEOMETRY + + frozen = FROZEN_QWEN35_STATIC_Q468_GEOMETRY + self._geometry = Geometry( + layer_indices=frozen.layer_indices, + heads=frozen.heads, + key_rows=frozen.key_rows, + value_width=frozen.value_width, + ) + return self._geometry + + def decode_identity(self, data: bytes) -> FrozenCalibrationIdentity: + return _identity_view(data, self.repository_root) + + def reduce_sequence( + self, + record: Mapping[str, object], + token_ids: tuple[int, ...], + captured: CapturedSequence, + ) -> object: + from recurquant.static_q468_calibration import ( + AnchorDistortionBatch, + reduce_frozen_anchor_distortions, + ) + + batch = AnchorDistortionBatch( + family=cast(Any, record["family"]), + config=cast(str, record["config"]), + ruler_category=cast(Any, record["ruler_category"]), + canonical_id=cast(str, record["canonical_id"]), + seed=cast(int | None, record["seed"]), + configured_length=cast(int | None, record["configured_length"]), + token_count=len(token_ids), + anchor_positions=captured.anchor_positions, + query_energy=captured.query_energy, + q4_mse=captured.q4_mse, + q6_mse=captured.q6_mse, + q8_mse=captured.q8_mse, + sequence_token_ids=token_ids, + identity_record=record, + ) + return reduce_frozen_anchor_distortions(batch) + + def finalize( + self, + scores: Sequence[object], + *, + identity: FrozenCalibrationIdentity, + source_commit: str, + ) -> FinalizationResult: + from recurquant.static_q468 import ( + FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + FROZEN_STATIC_Q48_PROMOTIONS, + FROZEN_STATIC_Q468_ABLATION_STEPS, + FROZEN_STATIC_Q468_PRIMARY_STEPS, + STATIC_Q48_COMPARATOR_METHOD, + STATIC_Q468_ABLATION_METHOD, + STATIC_Q468_PRIMARY_METHOD, + build_static_rht_q48_policy, + build_static_rht_q468_policy, + deserialize_static_rht_q48_policy, + deserialize_static_rht_q468_policy, + serialize_static_rht_q48_policy, + serialize_static_rht_q468_policy, + ) + from recurquant.static_q468_calibration import ( + aggregate_calibration_scores, + build_frozen_calibration_score_artifact, + build_frozen_split_half_stability_artifact, + deserialize_calibration_score_artifact, + deserialize_frozen_split_half_stability_artifact, + fit_split_half_policy, + ) + + resolver = _load_identity_resolver(self.repository_root) + typed_scores = cast(list[Any], list(scores)) + aggregate = aggregate_calibration_scores(typed_scores) + geometry = FROZEN_QWEN35_STATIC_Q468_GEOMETRY + fit = fit_split_half_policy( + typed_scores, + layer_indices=geometry.layer_indices, + rows_per_layer=geometry.rows_per_layer, + marginal_steps=FROZEN_STATIC_Q468_PRIMARY_STEPS, + ) + expected_assignment = [dict(item) for item in identity.assignment] + actual_assignment = [item.canonical_dict() for item in fit.split.assignments] + if ( + fit.split.assignment_sha256 != identity.assignment_sha256 + or actual_assignment != expected_assignment + ): + raise CalibrationRunError( + "calibration split differs from the frozen resolver assignment" + ) + stability = _stability_record(fit.stability) + if not fit.stability.passed: + return FinalizationResult(passed=False, stability=stability, artifacts=None) + + score_bytes = build_frozen_calibration_score_artifact( + aggregate, + calibration_identity_sha256=identity.file_sha256, + ) + decoded_score = deserialize_calibration_score_artifact(score_bytes) + split_bytes = build_frozen_split_half_stability_artifact( + fit.half_a_aggregate, + fit.half_b_aggregate, + identity_file_sha256=identity.file_sha256, + canonical_identity_sha256=identity.canonical_evidence_sha256, + resolver_assignment_sha256=identity.assignment_sha256, + full_sequence_score_manifest_sha256=aggregate.sequence_score_manifest_sha256, + full_calibration_scores_sha256=decoded_score.calibration_scores_sha256, + ) + deserialize_frozen_split_half_stability_artifact( + split_bytes, + expected_identity_file_sha256=identity.file_sha256, + expected_canonical_identity_sha256=identity.canonical_evidence_sha256, + expected_resolver_assignment_sha256=identity.assignment_sha256, + ) + policy_common = { + "geometry": geometry, + "calibration_manifest_sha256": aggregate.sequence_score_manifest_sha256, + "identity_artifact_sha256": identity.file_sha256, + "tokenizer_manifest_sha256": identity.tokenizer_manifest_sha256, + "source_commit": source_commit, + "calibration_scores_sha256": decoded_score.calibration_scores_sha256, + } + k27030 = build_static_rht_q468_policy( + aggregate.d4, + aggregate.d6, + aggregate.d8, + marginal_steps=FROZEN_STATIC_Q468_ABLATION_STEPS, + method_id=STATIC_Q468_ABLATION_METHOD, + **policy_common, + ) + k29334 = build_static_rht_q468_policy( + aggregate.d4, + aggregate.d6, + aggregate.d8, + marginal_steps=FROZEN_STATIC_Q468_PRIMARY_STEPS, + method_id=STATIC_Q468_PRIMARY_METHOD, + **policy_common, + ) + q48 = build_static_rht_q48_policy( + aggregate.d4, + aggregate.d8, + geometry=geometry, + promoted_rows=FROZEN_STATIC_Q48_PROMOTIONS, + calibration_manifest_sha256=aggregate.sequence_score_manifest_sha256, + identity_artifact_sha256=identity.file_sha256, + tokenizer_manifest_sha256=identity.tokenizer_manifest_sha256, + source_commit=source_commit, + method_id=STATIC_Q48_COMPARATOR_METHOD, + ) + k27030_bytes = serialize_static_rht_q468_policy(k27030) + k29334_bytes = serialize_static_rht_q468_policy(k29334) + q48_bytes = serialize_static_rht_q48_policy(q48) + deserialize_static_rht_q468_policy(k27030_bytes) + deserialize_static_rht_q468_policy(k29334_bytes) + deserialize_static_rht_q48_policy(q48_bytes) + binding_bytes = resolver.build_stage_a_calibration_binding_artifact( + frozen_identity_artifact=identity.artifact_bytes, + calibration_score_artifact=score_bytes, + split_half_stability_artifact=split_bytes, + static_k27030_policy_artifact=k27030_bytes, + static_k29334_policy_artifact=k29334_bytes, + ) + resolver.deserialize_stage_a_calibration_binding_artifact(binding_bytes) + return FinalizationResult( + passed=True, + stability=stability, + artifacts=CalibrationArtifacts( + score=score_bytes, + split_half=split_bytes, + static_k27030=k27030_bytes, + static_k29334=k29334_bytes, + static_q48=q48_bytes, + stage_a_binding=binding_bytes, + stability=stability, + calibration_scores_sha256=decoded_score.calibration_scores_sha256, + sequence_score_manifest_sha256=aggregate.sequence_score_manifest_sha256, + ), + ) + + +def _model_contract_matches( + identity: FrozenCalibrationIdentity, + manifest: ModelFileManifest, +) -> None: + expected = ( + identity.model_id, + identity.model_revision, + identity.transformers_version, + ) + actual = (manifest.model_id, manifest.revision, manifest.transformers_version) + if actual != expected: + raise CalibrationRunError( + "model file manifest differs from frozen identity model/tokenizer runtime contract" + ) + + +def _runtime_environment( + adapter_metadata: Mapping[str, object], + *, + elapsed_seconds: float, + authenticated_runtime: AuthenticatedRuntime, +) -> dict[str, object]: + torch = _torch_runtime() + result: dict[str, object] = { + "adapter": dict(adapter_metadata), + "elapsed_seconds_hex": elapsed_seconds.hex(), + "platform": platform.platform(), + "python": platform.python_version(), + "authenticated_distribution_count": authenticated_runtime.distribution_count, + "authenticated_file_count": authenticated_runtime.file_count, + "packages": dict(authenticated_runtime.distributions), + "runtime_manifest_file_sha256": authenticated_runtime.manifest_file_sha256, + "torch": torch.__version__, + "cuda_available": torch.cuda.is_available(), + "cuda_runtime": torch.version.cuda, + } + if torch.cuda.is_available(): + device = torch.cuda.current_device() + torch.cuda.synchronize(device) + result["gpu"] = { + "capability": list(torch.cuda.get_device_capability(device)), + "device_index": device, + "name": torch.cuda.get_device_name(device), + "peak_allocated_bytes": torch.cuda.max_memory_allocated(device), + "peak_reserved_bytes": torch.cuda.max_memory_reserved(device), + } + else: + result["gpu"] = None + return result + + +def _report_bytes( + *, + status: str, + identity: FrozenCalibrationIdentity, + source_commit: str, + source_manifest_sha256: str, + source_manifest_file_sha256: str, + model_files: Any, + sequence_count: int, + token_count: int, + anchor_count: int, + stability: Mapping[str, object], + artifacts: Mapping[str, bytes], + runtime: Mapping[str, object], +) -> bytes: + evidence = { + "artifacts": {name: sha256_bytes(payload) for name, payload in sorted(artifacts.items())}, + "calibration": { + "anchor_count": anchor_count, + "sequence_count": sequence_count, + "token_count": token_count, + }, + "identity": { + "canonical_evidence_sha256": identity.canonical_evidence_sha256, + "file_sha256": identity.file_sha256, + "identity_input_manifest_sha256": identity.identity_input_manifest_sha256, + "tokenizer_manifest_sha256": identity.tokenizer_manifest_sha256, + "execution_bindings": { + "calibration_runtime_manifest_file_sha256": (identity.runtime_manifest_file_sha256), + "model_file_manifest_file_sha256": (identity.model_file_manifest_file_sha256), + "parquet_materialization_manifest_file_sha256": ( + identity.parquet_materialization_manifest_file_sha256 + ), + "repository_source_manifest_file_sha256": ( + identity.repository_source_manifest_file_sha256 + ), + }, + }, + "model_files": { + "file_count": len(model_files.files), + "hub_tree_manifest_sha256": model_files.hub_tree_manifest_sha256, + "manifest_file_sha256": model_files.manifest_file_sha256, + "model_id": model_files.model_id, + "revision": model_files.revision, + "transformers_version": model_files.transformers_version, + }, + "query_energy_ema": { + "decay_hex": QUERY_EMA_DECAY.hex(), + "epsilon_hex": QUERY_ENERGY_EPSILON.hex(), + "prior": "uniform_1_over_key_rows", + }, + "repository": { + "source_commit": source_commit, + "source_manifest_file_sha256": source_manifest_file_sha256, + "source_manifest_sha256": source_manifest_sha256, + }, + "runner_revision": RUNNER_REVISION, + "runtime": dict(runtime), + "stability": dict(stability), + "status": status, + } + document = { + "artifact_kind": RUN_REPORT_KIND, + "canonical_evidence_sha256": sha256_bytes(canonical_json_bytes(evidence)), + "evidence": evidence, + "schema_version": RUN_REPORT_SCHEMA, + } + return canonical_json_bytes(document) + + +def _atomic_publish_new(path: Path, payload: bytes) -> None: + if not isinstance(payload, bytes): + raise TypeError("artifact payload must be bytes") + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + try: + os.link(temporary, path) + except FileExistsError: + raise FileExistsError(f"refusing to overwrite existing artifact: {path}") from None + finally: + temporary.unlink(missing_ok=True) + + +def _publish_output_directory(output_dir: Path, payloads: Mapping[str, bytes]) -> None: + resolved = Path(os.path.abspath(output_dir)) + parent = resolved.parent + parent.mkdir(parents=True, exist_ok=True) + if resolved.exists(): + raise FileExistsError(f"refusing to overwrite existing calibration output: {resolved}") + prefix = f".{resolved.name}.staging-" + staging = Path(tempfile.mkdtemp(prefix=prefix, dir=parent)) + owned_staging = True + # The Stage-A binding and completion marker are deliberately last. The + # public directory appears only after every dependency is durable. + ordered = [name for name in sorted(payloads) if name not in {REPORT_FILENAME, BINDING_FILENAME}] + if REPORT_FILENAME in payloads: + ordered.append(REPORT_FILENAME) + if BINDING_FILENAME in payloads: + ordered.append(BINDING_FILENAME) + try: + for name in ordered: + _atomic_publish_new(staging / name, payloads[name]) + _atomic_publish_new( + staging / COMPLETE_FILENAME, + b"recurquant-experiment013-calibration-complete-v1\n", + ) + if resolved.exists(): + raise FileExistsError(f"refusing to overwrite existing calibration output: {resolved}") + staging.rename(resolved) + owned_staging = False + try: + descriptor = os.open(parent, os.O_RDONLY) + except OSError: + descriptor = None + if descriptor is not None: + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + finally: + if owned_staging: + try: + staging.relative_to(parent) + except ValueError as exc: # defensive: never recursively remove outside parent + raise RuntimeError("owned staging directory escaped its parent") from exc + if not staging.name.startswith(prefix): + raise RuntimeError("owned staging directory name drifted") + shutil.rmtree(staging, ignore_errors=False) + + +def run_calibration( + config: CalibrationRunConfig, + adapter: Any, + *, + services: RunnerServices, +) -> dict[str, object]: + """Execute the authenticated calibration and publish one no-overwrite result set.""" + + started = time.perf_counter() + # First executable boundary: a strict promoted identity decode. No source + # adapter, model path, repository command, or output path is touched first. + identity = services.backend.decode_identity(config.frozen_identity_bytes) + if config.output_dir.resolve().exists(): + raise FileExistsError( + f"refusing to overwrite existing calibration output: {config.output_dir.resolve()}" + ) + + source_commit = _verify_repository_commit( + config.repository_root, + config.expected_source_commit, + ) + source_manifest_file_sha256 = sha256_bytes(config.repository_source_manifest_bytes) + if source_manifest_file_sha256 != identity.repository_source_manifest_file_sha256: + raise CalibrationRunError( + "repository source manifest bytes differ from the frozen identity binding" + ) + source_manifest_input = _strict_json_bytes( + config.repository_source_manifest_bytes, + context="repository source manifest", + ) + source_manifest, source_manifest_sha256 = services.verify_repository_source( + source_manifest_input, + config.repository_root, + ) + if source_manifest.get("source_commit") != source_commit: + raise CalibrationRunError( + "reported source commit must equal the authenticated source-manifest commit" + ) + services.validate_adapter(adapter) + + expected_runtime_manifest_sha256 = _sha256( + config.expected_runtime_manifest_sha256, + context="expected calibration runtime manifest SHA-256", + ) + runtime_manifest_file_sha256 = sha256_bytes(config.runtime_manifest_bytes) + if ( + runtime_manifest_file_sha256 != expected_runtime_manifest_sha256 + or runtime_manifest_file_sha256 != identity.runtime_manifest_file_sha256 + ): + raise CalibrationRunError( + "calibration runtime manifest bytes differ from the frozen identity/config binding" + ) + runtime_manifest = parse_calibration_runtime_manifest(config.runtime_manifest_bytes) + authenticated_runtime = services.authenticate_runtime(runtime_manifest) + if authenticated_runtime.manifest_file_sha256 != runtime_manifest_file_sha256: + raise CalibrationRunError("runtime authenticator returned a different manifest identity") + runtime_versions = dict(authenticated_runtime.distributions) + if runtime_versions.get("transformers") != identity.transformers_version: + raise CalibrationRunError( + "authenticated Transformers version differs from the frozen identity contract" + ) + + expected_model_manifest_sha256 = _sha256( + config.expected_model_file_manifest_sha256, + context="expected model file manifest SHA-256", + ) + model_manifest_file_sha256 = sha256_bytes(config.model_file_manifest_bytes) + if ( + model_manifest_file_sha256 != expected_model_manifest_sha256 + or model_manifest_file_sha256 != identity.model_file_manifest_file_sha256 + ): + raise CalibrationRunError( + "model file manifest bytes differ from the frozen identity/config binding" + ) + model_manifest = parse_model_file_manifest(config.model_file_manifest_bytes) + _model_contract_matches(identity, model_manifest) + + expected_parquet_manifest_sha256 = _sha256( + config.expected_parquet_materialization_manifest_sha256, + context="expected parquet materialization manifest SHA-256", + ) + parquet_manifest_file_sha256 = sha256_bytes( + config.parquet_materialization_manifest_bytes + ) + if ( + parquet_manifest_file_sha256 != expected_parquet_manifest_sha256 + or parquet_manifest_file_sha256 + != identity.parquet_materialization_manifest_file_sha256 + ): + raise CalibrationRunError( + "parquet materialization manifest bytes differ from the frozen identity/config " + "binding" + ) + + runtime_before_data = services.authenticate_runtime(runtime_manifest) + if runtime_before_data != authenticated_runtime: + raise CalibrationRunError("calibration runtime identity changed before data access") + + materialized: list[tuple[dict[str, object], tuple[int, ...]]] = [] + for record in identity.records: + candidate = adapter.materialize_sequence(record) + materialized.append( + ( + record, + validate_materialized_sequence( + record, + candidate, + calibration_api=services.calibration_api, + ), + ) + ) + + # Reverify source after data/tokenizer adapter use and immediately before + # opening the local model files. + _verified_again, second_source_sha256 = services.verify_repository_source( + source_manifest_input, + config.repository_root, + ) + if second_source_sha256 != source_manifest_sha256: + raise CalibrationRunError("repository source changed during sequence materialization") + runtime_before_model = services.authenticate_runtime(runtime_manifest) + if runtime_before_model != authenticated_runtime: + raise CalibrationRunError("calibration runtime identity changed before model access") + + authenticated_model = services.authenticate_model_files(config.model_root, model_manifest) + torch = _torch_runtime() + if config.require_cuda and not torch.cuda.is_available(): + raise CalibrationRunError("official Experiment 013 calibration requires CUDA") + if torch.cuda.is_available(): + torch.cuda.empty_cache() + torch.cuda.reset_peak_memory_stats() + + model: object | None = None + adapter_runtime_metadata: Mapping[str, object] | None = None + scores: list[object] = [] + total_tokens = 0 + total_anchors = 0 + try: + # The adapter may call AutoModel only inside this method. The exact local + # file set has already been hashed, revision checked, and source verified. + model = adapter.load_model(authenticated_model) + authenticated_after_load = services.authenticate_model_files( + config.model_root, model_manifest + ) + if authenticated_after_load != authenticated_model: + raise CalibrationRunError("local model identity changed while loading weights") + for record, token_ids in materialized: + captured = capture_sequence_causally( + adapter, + model, + record, + token_ids, + geometry=services.backend.geometry, + calibration_api=services.calibration_api, + require_cuda=config.require_cuda, + distortion_function=services.distortion_function, + ) + scores.append(services.backend.reduce_sequence(record, token_ids, captured)) + total_tokens += len(token_ids) + total_anchors += len(captured.anchor_positions) + result = services.backend.finalize( + scores, + identity=identity, + source_commit=source_commit, + ) + # Snapshot run metadata while the authenticated model/device/observer + # are still live. Cleanup follows immediately and every external + # identity is then reauthenticated before publication. + adapter_runtime_metadata = dict(adapter.runtime_metadata()) + finally: + if model is not None: + adapter.close_model(model) + + if adapter_runtime_metadata is None: + raise RuntimeError("successful calibration omitted live adapter runtime metadata") + + authenticated_after_run = services.authenticate_model_files(config.model_root, model_manifest) + if authenticated_after_run != authenticated_model: + raise CalibrationRunError("local model identity changed during calibration") + + # Adapter callbacks finish before the final source/runtime/model checks. + runtime = _runtime_environment( + adapter_runtime_metadata, + elapsed_seconds=time.perf_counter() - started, + authenticated_runtime=authenticated_runtime, + ) + _verified_final, final_source_sha256 = services.verify_repository_source( + source_manifest_input, + config.repository_root, + ) + if final_source_sha256 != source_manifest_sha256: + raise CalibrationRunError("repository source changed during calibration") + final_runtime = services.authenticate_runtime(runtime_manifest) + if final_runtime != authenticated_runtime: + raise CalibrationRunError("calibration runtime identity changed during calibration") + final_model = services.authenticate_model_files(config.model_root, model_manifest) + if final_model != authenticated_model: + raise CalibrationRunError("local model identity changed before publication") + + if not result.passed: + report = _report_bytes( + status="stability_failed", + identity=identity, + source_commit=source_commit, + source_manifest_sha256=source_manifest_sha256, + source_manifest_file_sha256=source_manifest_file_sha256, + model_files=authenticated_model, + sequence_count=len(scores), + token_count=total_tokens, + anchor_count=total_anchors, + stability=result.stability, + artifacts={}, + runtime=runtime, + ) + _publish_output_directory(config.output_dir, {REPORT_FILENAME: report}) + report_path = config.output_dir / REPORT_FILENAME + raise CalibrationStabilityFailure( + f"split-half stability gate failed; failure report: {report_path}" + ) + if result.artifacts is None: + raise RuntimeError("passing finalization omitted calibration artifacts") + artifacts = result.artifacts + payloads = { + SCORE_FILENAME: artifacts.score, + SPLIT_FILENAME: artifacts.split_half, + K27030_FILENAME: artifacts.static_k27030, + K29334_FILENAME: artifacts.static_k29334, + Q48_FILENAME: artifacts.static_q48, + BINDING_FILENAME: artifacts.stage_a_binding, + } + report = _report_bytes( + status="passed", + identity=identity, + source_commit=source_commit, + source_manifest_sha256=source_manifest_sha256, + source_manifest_file_sha256=source_manifest_file_sha256, + model_files=authenticated_model, + sequence_count=len(scores), + token_count=total_tokens, + anchor_count=total_anchors, + stability=result.stability, + artifacts=payloads, + runtime=runtime, + ) + payloads[REPORT_FILENAME] = report + _publish_output_directory(config.output_dir, payloads) + return { + "artifact_sha256": {name: sha256_bytes(payload) for name, payload in payloads.items()}, + "output_dir": str(config.output_dir.resolve()), + "sequence_count": len(scores), + "status": "passed", + } + + +def default_services( + repository_root: Path = REPOSITORY_ROOT, + *, + base_runtime_root: Path, + calibration_api: ModuleType | None = None, + interpreter_path: Path, + package_roots: Mapping[str, Path], +) -> RunnerServices: + api = _AUTHENTICATED_CALIBRATION_API if calibration_api is None else calibration_api + if api is None: + raise CalibrationRunError("calibration API was not bootstrap-authenticated") + backend = Experiment013Backend(repository_root) + return RunnerServices( + backend=backend, + calibration_api=api, + verify_repository_source=verify_repository_source_manifest, + validate_adapter=lambda adapter: validate_adapter_contract( + adapter, + calibration_api=api, + ), + distortion_function=compute_anchor_distortions, + authenticate_model_files=lambda root, manifest: authenticate_local_model_files( + root, + manifest, + calibration_api=api, + ), + authenticate_runtime=lambda manifest: authenticate_calibration_runtime( + manifest, + base_runtime_root=base_runtime_root, + package_roots=package_roots, + interpreter_path=interpreter_path, + ), + ) + + +def _install_authenticated_recurquant_namespace(repository_root: Path) -> ModuleType: + if "recurquant" in sys.modules: + raise CalibrationRunError("refusing a preloaded recurquant package before adapter import") + package_root = Path(os.path.abspath(repository_root)) / "src" / "recurquant" + if not package_root.is_dir() or _is_link_or_reparse(package_root): + raise CalibrationRunError("authenticated recurquant package root is unavailable") + package = ModuleType("recurquant") + package.__package__ = "recurquant" + package.__path__ = [str(package_root)] # type: ignore[attr-defined] + package.__spec__ = importlib.machinery.ModuleSpec( + "recurquant", + loader=None, + is_package=True, + ) + package.__spec__.submodule_search_locations = [str(package_root)] + sys.modules["recurquant"] = package + return package + + +def _load_adapter( + specification: str, + *, + repository_root: Path, + source_entry: Mapping[str, object], + calibration_api: ModuleType, + context: Any, +) -> Any: + if specification != CANONICAL_ADAPTER_SPEC: + raise CalibrationRunError(f"official calibration requires exactly {CANONICAL_ADAPTER_SPEC}") + if CANONICAL_ADAPTER_MODULE in sys.modules: + raise CalibrationRunError("refusing a preloaded reviewed adapter module") + _install_authenticated_recurquant_namespace(repository_root) + module = _load_exact_source_module( + CANONICAL_ADAPTER_MODULE, + CANONICAL_ADAPTER_PATH, + repository_root=repository_root, + entry=source_entry, + ) + factory = getattr(module, "create_adapter", None) + if not callable(factory): + raise TypeError("reviewed adapter create_adapter factory is not callable") + adapter = factory(context) + validate_adapter_contract(adapter, calibration_api=calibration_api) + adapter_path = _assert_no_link_components( + Path(os.path.abspath(repository_root)), + PurePosixPath(CANONICAL_ADAPTER_PATH), + ) + digest, _size = _stream_file_sha256(adapter_path) + if digest != source_entry.get("raw_sha256"): + raise CalibrationRunError("reviewed adapter source changed during construction") + return adapter + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--frozen-identity", required=True, type=Path) + parser.add_argument("--repository-source-manifest", required=True, type=Path) + parser.add_argument("--model-file-manifest", required=True, type=Path) + parser.add_argument("--expected-model-file-manifest-sha256", required=True) + parser.add_argument("--parquet-materialization-manifest", required=True, type=Path) + parser.add_argument( + "--expected-parquet-materialization-manifest-sha256", + required=True, + ) + parser.add_argument("--runtime-manifest", required=True, type=Path) + parser.add_argument("--expected-runtime-manifest-sha256", required=True) + parser.add_argument("--model-root", required=True, type=Path) + parser.add_argument("--cache-root", required=True, type=Path) + parser.add_argument("--ruler-root", required=True, type=Path) + parser.add_argument("--repository-root", default=REPOSITORY_ROOT, type=Path) + parser.add_argument("--source-commit", required=True) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument( + "--adapter", + choices=[CANONICAL_ADAPTER_SPEC], + default=CANONICAL_ADAPTER_SPEC, + help="The single reviewed live adapter (generic adapters are test-injection only).", + ) + return parser + + +def _capture_manifest_mode(arguments: Sequence[str]) -> int | None: + if not arguments or arguments[0] not in { + "capture-source-manifest", + "capture-runtime-manifest", + "capture-model-manifest", + "prepare-runtime", + }: + return None + command = arguments[0] + parser = argparse.ArgumentParser(prog=f"{Path(__file__).name} {command}") + if command == "prepare-runtime": + parser.add_argument("--source-python", required=True, type=Path) + parser.add_argument("--requirements", required=True, type=Path) + parser.add_argument("--output-root", required=True, type=Path) + parser.add_argument( + "--package-root-name", + default=DEFAULT_PACKAGE_RUNTIME_ROOT_NAME, + ) + else: + parser.add_argument("--output", required=True, type=Path) + if command == "capture-source-manifest": + parser.add_argument("--repository-root", required=True, type=Path) + if command == "capture-runtime-manifest": + parser.add_argument("--base-runtime-root", required=True, type=Path) + parser.add_argument("--staged-interpreter", required=True, type=Path) + parser.add_argument("--package-root", required=True, action="append") + parser.add_argument("--package-import-path", required=True, action="append") + if command == "capture-model-manifest": + parser.add_argument("--model-id", required=True) + parser.add_argument("--revision", required=True) + parser.add_argument("--transformers-version", required=True) + args = parser.parse_args(arguments[1:]) + if command == "prepare-runtime": + details = prepare_calibration_runtime( + source_python=args.source_python, + requirements_file=args.requirements, + output_root=args.output_root, + package_root_name=args.package_root_name, + ) + print(json.dumps(details, sort_keys=True)) + return 0 + status = "captured_metadata_only" + details: dict[str, object] = {} + if command == "capture-source-manifest": + source_module = _load_source_capture_module(args.repository_root) + try: + captured = source_module.capture_experiment013_source_manifest(args.repository_root) + normalized = source_module.validate_experiment013_source_manifest(captured) + payload = source_module.canonical_experiment013_source_manifest_bytes(normalized) + output = _assert_source_manifest_output_location(args.repository_root, args.output) + before_publish = source_module.verify_experiment013_source_manifest( + normalized, + args.repository_root, + ) + if before_publish != normalized: + raise CalibrationRunError("source verifier changed the captured manifest") + _atomic_publish_new(output, payload) + if output.read_bytes() != payload: + raise CalibrationRunError("published source manifest bytes changed on disk") + after_publish = source_module.verify_experiment013_source_manifest( + normalized, + args.repository_root, + ) + if after_publish != normalized: + raise CalibrationRunError("source identity changed after manifest publication") + details = { + "canonical_manifest_sha256": normalized["canonical_manifest_sha256"], + "source_commit": normalized["source_commit"], + } + status = "captured_verified_source_metadata" + finally: + sys.modules.pop(SOURCE_CAPTURE_MODULE, None) + elif command == "capture-runtime-manifest": + package_roots = _parse_named_path_arguments( + args.package_root, + context="package root", + ) + package_import_paths = _parse_named_import_path_arguments( + args.package_import_path, + context="package import path", + ) + payload = capture_calibration_runtime_manifest( + base_runtime_root=args.base_runtime_root, + package_roots=package_roots, + package_import_paths=package_import_paths, + interpreter_path=args.staged_interpreter, + ) + else: + payload = capture_model_file_manifest_from_hub( + args.model_id, + args.revision, + transformers_version=args.transformers_version, + ) + if command != "capture-source-manifest": + output = args.output.resolve() + _atomic_publish_new(output, payload) + print( + json.dumps( + { + **details, + "file_sha256": sha256_bytes(payload), + "output": str(output), + "status": status, + }, + sort_keys=True, + ) + ) + return 0 + + +def _official_main( + argv: Sequence[str], + *, + runtime_context: SealedRuntimeContext, + interpreter_path: Path, +) -> int: + global _AUTHENTICATED_CALIBRATION_API + global _AUTHENTICATED_IDENTITY_RESOLVER + global _AUTHENTICATED_SOURCE_VERIFIER + + arguments = list(argv) + if any( + module is not None + for module in ( + _AUTHENTICATED_CALIBRATION_API, + _AUTHENTICATED_IDENTITY_RESOLVER, + _AUTHENTICATED_SOURCE_VERIFIER, + ) + ): + raise CalibrationRunError("authenticated runner modules were already loaded") + args = _parser().parse_args(arguments) + identity_bytes = args.frozen_identity.read_bytes() + bindings = _bootstrap_identity_bindings(identity_bytes) + source_manifest_bytes = args.repository_source_manifest.read_bytes() + runtime_manifest_bytes = args.runtime_manifest.read_bytes() + model_manifest_bytes = args.model_file_manifest.read_bytes() + parquet_manifest_bytes = args.parquet_materialization_manifest.read_bytes() + exact_manifest_hashes = { + "repository source": ( + sha256_bytes(source_manifest_bytes), + bindings.repository_source_manifest_file_sha256, + ), + "runtime": (sha256_bytes(runtime_manifest_bytes), bindings.runtime_manifest_file_sha256), + "model": (sha256_bytes(model_manifest_bytes), bindings.model_file_manifest_file_sha256), + "parquet materialization": ( + sha256_bytes(parquet_manifest_bytes), + bindings.parquet_materialization_manifest_file_sha256, + ), + } + for name, (actual, expected) in exact_manifest_hashes.items(): + if actual != expected: + raise CalibrationRunError( + f"{name} manifest bytes differ from the frozen identity bootstrap binding" + ) + if ( + _sha256( + args.expected_runtime_manifest_sha256, + context="expected runtime manifest SHA-256", + ) + != bindings.runtime_manifest_file_sha256 + ): + raise CalibrationRunError("CLI runtime manifest SHA-256 differs from frozen identity") + if ( + _sha256( + args.expected_model_file_manifest_sha256, + context="expected model manifest SHA-256", + ) + != bindings.model_file_manifest_file_sha256 + ): + raise CalibrationRunError("CLI model manifest SHA-256 differs from frozen identity") + if ( + _sha256( + args.expected_parquet_materialization_manifest_sha256, + context="expected parquet materialization manifest SHA-256", + ) + != bindings.parquet_materialization_manifest_file_sha256 + ): + raise CalibrationRunError( + "CLI parquet materialization manifest SHA-256 differs from frozen identity" + ) + + bootstrap_source = _bootstrap_source_manifest( + source_manifest_bytes, + repository_root=args.repository_root, + require_adapter=True, + ) + requested_commit = _git_revision(args.source_commit, context="requested source commit") + if requested_commit != bootstrap_source.source_commit: + raise CalibrationRunError("CLI source commit differs from source-manifest commit") + _verify_repository_commit(args.repository_root, requested_commit) + + _AUTHENTICATED_CALIBRATION_API = _load_exact_source_module( + CALIBRATION_API_MODULE, + CALIBRATION_API_PATH, + repository_root=args.repository_root, + entry=bootstrap_source.entries[CALIBRATION_API_PATH], + ) + _AUTHENTICATED_IDENTITY_RESOLVER = _load_exact_source_module( + IDENTITY_RESOLVER_MODULE, + IDENTITY_RESOLVER_SOURCE_PATH, + repository_root=args.repository_root, + entry=bootstrap_source.entries[IDENTITY_RESOLVER_SOURCE_PATH], + ) + _AUTHENTICATED_SOURCE_VERIFIER = _load_exact_source_module( + "recurquant.experiment013_source", + SOURCE_VERIFIER_PATH, + repository_root=args.repository_root, + entry=bootstrap_source.entries[SOURCE_VERIFIER_PATH], + ) + services = default_services( + args.repository_root, + base_runtime_root=runtime_context.base_runtime_root, + calibration_api=_AUTHENTICATED_CALIBRATION_API, + interpreter_path=interpreter_path, + package_roots=runtime_context.package_roots, + ) + identity = services.backend.decode_identity(identity_bytes) + if ( + identity.repository_source_manifest_file_sha256 + != bindings.repository_source_manifest_file_sha256 + or identity.runtime_manifest_file_sha256 != bindings.runtime_manifest_file_sha256 + or identity.model_file_manifest_file_sha256 != bindings.model_file_manifest_file_sha256 + or identity.parquet_materialization_manifest_file_sha256 + != bindings.parquet_materialization_manifest_file_sha256 + ): + raise CalibrationRunError("full identity decode differs from bootstrap execution bindings") + verified_source, _source_sha256 = services.verify_repository_source( + bootstrap_source.manifest, + args.repository_root, + ) + if verified_source.get("source_commit") != requested_commit: + raise CalibrationRunError("verified source-manifest commit differs from current HEAD") + runtime_manifest = parse_calibration_runtime_manifest(runtime_manifest_bytes) + authenticated_runtime = services.authenticate_runtime(runtime_manifest) + if authenticated_runtime.manifest_file_sha256 != bindings.runtime_manifest_file_sha256: + raise CalibrationRunError("runtime authenticator returned a different manifest identity") + model_manifest = parse_model_file_manifest(model_manifest_bytes) + _model_contract_matches(identity, model_manifest) + context = _AUTHENTICATED_CALIBRATION_API.AdapterConstructionContext( + repository_root=Path(args.repository_root), + model_root=Path(args.model_root), + cache_root=Path(args.cache_root), + ruler_root=Path(args.ruler_root), + execution_binding_artifacts={ + "repository_source_manifest_bytes": bytes(source_manifest_bytes), + "calibration_runtime_manifest_bytes": bytes(runtime_manifest_bytes), + "model_file_manifest_bytes": bytes(model_manifest_bytes), + "parquet_materialization_manifest_bytes": bytes(parquet_manifest_bytes), + }, + runtime_authentication_context={ + "base_runtime_root": runtime_context.base_runtime_root, + "staged_interpreter": Path(interpreter_path), + "package_runtime_roots": dict(runtime_context.package_roots), + "package_import_paths": dict(runtime_context.package_import_paths), + }, + ) + adapter = _load_adapter( + args.adapter, + repository_root=args.repository_root, + source_entry=bootstrap_source.entries[CANONICAL_ADAPTER_PATH], + calibration_api=_AUTHENTICATED_CALIBRATION_API, + context=context, + ) + config = CalibrationRunConfig( + frozen_identity_bytes=identity_bytes, + repository_source_manifest_bytes=source_manifest_bytes, + model_file_manifest_bytes=model_manifest_bytes, + parquet_materialization_manifest_bytes=parquet_manifest_bytes, + runtime_manifest_bytes=runtime_manifest_bytes, + model_root=args.model_root, + repository_root=args.repository_root, + expected_source_commit=args.source_commit, + expected_model_file_manifest_sha256=args.expected_model_file_manifest_sha256, + expected_parquet_materialization_manifest_sha256=( + args.expected_parquet_materialization_manifest_sha256 + ), + expected_runtime_manifest_sha256=args.expected_runtime_manifest_sha256, + output_dir=args.output_dir, + require_cuda=True, + ) + result = run_calibration( + config, + adapter, + services=services, + ) + print(json.dumps(result, sort_keys=True)) + return 0 + + +def sealed_main( + argv: Sequence[str], + *, + base_runtime_root: Path, + package_roots: Mapping[str, Path], + package_import_paths: Mapping[str, str], + interpreter_path: Path, + pycache_prefix: Path, +) -> int: + """Run only after the stdlib bootstrap supplies explicit authenticated roots.""" + + arguments = list(argv) + args = _parser().parse_args(arguments) + runtime_manifest_bytes = args.runtime_manifest.read_bytes() + manifest, runtime_context, _authenticated = _authenticate_sealed_runtime_context( + runtime_manifest_bytes, + base_runtime_root=base_runtime_root, + package_roots=package_roots, + package_import_paths=package_import_paths, + interpreter_path=interpreter_path, + pycache_prefix=pycache_prefix, + ) + if manifest.file_sha256 != _sha256( + args.expected_runtime_manifest_sha256, + context="expected runtime manifest SHA-256", + ): + raise CalibrationRunError("sealed runtime manifest differs from the CLI binding") + result = _official_main( + arguments, + runtime_context=runtime_context, + interpreter_path=Path(interpreter_path), + ) + _verify_sealed_launch_state( + runtime_context.pycache_prefix, + manifest=manifest, + roots={ + BASE_RUNTIME_ROOT_NAME: runtime_context.base_runtime_root, + **dict(runtime_context.package_roots), + }, + ) + authenticate_calibration_runtime( + manifest, + base_runtime_root=runtime_context.base_runtime_root, + package_roots=runtime_context.package_roots, + interpreter_path=interpreter_path, + ) + return result + + +def main(argv: Sequence[str] | None = None) -> int: + """Expose metadata preparation only; official runs require the sealed launcher.""" + + arguments = list(sys.argv[1:] if argv is None else argv) + capture_result = _capture_manifest_mode(arguments) + if capture_result is not None: + return capture_result + raise CalibrationRunError( + "official calibration must be started with launch_static_q468_calibration.py" + ) + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/src/recurquant/experiment013_calibration_api.py b/src/recurquant/experiment013_calibration_api.py new file mode 100644 index 0000000..b18eb2d --- /dev/null +++ b/src/recurquant/experiment013_calibration_api.py @@ -0,0 +1,233 @@ +"""Stdlib-only live-adapter contract for Experiment 013 calibration. + +This module intentionally imports no tensor, model, dataset, or Hub library. +The runner authenticates these exact source bytes before importing the module. +Adapter factories receive paths as inert :class:`~pathlib.Path` values and must +perform no I/O during construction. Dataset/tokenizer access starts only in +``materialize_sequence``; model-file access starts only in ``load_model``. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from types import MappingProxyType +from typing import Protocol, runtime_checkable + +EXECUTION_BINDING_ARTIFACT_KEYS = frozenset( + { + "calibration_runtime_manifest_bytes", + "model_file_manifest_bytes", + "parquet_materialization_manifest_bytes", + "repository_source_manifest_bytes", + } +) +RUNTIME_AUTHENTICATION_CONTEXT_KEYS = frozenset( + { + "base_runtime_root", + "package_import_paths", + "package_runtime_roots", + "staged_interpreter", + } +) + + +def _absolute_inert_path(value: object, *, name: str) -> Path: + if not isinstance(value, Path) or not value.is_absolute() or ".." in value.parts: + raise ValueError(f"{name} must be an absolute normalized Path") + return Path(value) + + +def _canonical_package_name(value: object) -> str: + if ( + not isinstance(value, str) + or not value + or value[0] not in "abcdefghijklmnopqrstuvwxyz0123456789" + or value[-1] not in "abcdefghijklmnopqrstuvwxyz0123456789" + or any(character not in "abcdefghijklmnopqrstuvwxyz0123456789-" for character in value) + ): + raise ValueError("runtime package-root names must be canonical lowercase names") + return value + + +def _canonical_import_path(value: object, *, name: str) -> str: + if not isinstance(value, str) or not value or "\\" in value: + raise ValueError(f"runtime import path for {name} is not canonical") + parsed = PurePosixPath(value) + if ( + parsed.is_absolute() + or str(parsed) != value + or any(part in {"", ".", ".."} for part in parsed.parts) + ): + raise ValueError(f"runtime import path for {name} is not canonical") + return value + + +def _normalize_runtime_authentication_context( + value: Mapping[str, object], +) -> Mapping[str, object]: + if not isinstance(value, Mapping) or set(value) != RUNTIME_AUTHENTICATION_CONTEXT_KEYS: + raise ValueError("runtime_authentication_context keys differ from the frozen API") + base_root = _absolute_inert_path(value["base_runtime_root"], name="base_runtime_root") + interpreter = _absolute_inert_path( + value["staged_interpreter"], + name="staged_interpreter", + ) + try: + interpreter.relative_to(base_root) + except ValueError as error: + raise ValueError("staged_interpreter must be inside base_runtime_root") from error + raw_roots = value["package_runtime_roots"] + raw_imports = value["package_import_paths"] + if not isinstance(raw_roots, Mapping) or not raw_roots: + raise ValueError("package_runtime_roots must be a non-empty mapping") + if not isinstance(raw_imports, Mapping) or set(raw_imports) != set(raw_roots): + raise ValueError("package runtime-root and import-path names must match exactly") + normalized_names = tuple(_canonical_package_name(raw_name) for raw_name in raw_roots) + package_roots: dict[str, Path] = {} + import_paths: dict[str, str] = {} + for name in sorted(normalized_names): + package_roots[name] = _absolute_inert_path( + raw_roots[name], + name=f"package_runtime_roots[{name!r}]", + ) + import_paths[name] = _canonical_import_path(raw_imports[name], name=name) + return MappingProxyType( + { + "base_runtime_root": base_root, + "package_import_paths": MappingProxyType(import_paths), + "package_runtime_roots": MappingProxyType(package_roots), + "staged_interpreter": interpreter, + } + ) + + +@dataclass(frozen=True, slots=True) +class AdapterConstructionContext: + """Authenticated locations supplied to the reviewed adapter factory. + + Construction is a pure wiring step. The factory must not stat, resolve, + open, import from, or otherwise inspect any of these locations. + """ + + repository_root: Path + model_root: Path + cache_root: Path + ruler_root: Path + runtime_authentication_context: Mapping[str, object] + execution_binding_artifacts: Mapping[str, bytes] + + def __post_init__(self) -> None: + object.__setattr__( + self, + "runtime_authentication_context", + _normalize_runtime_authentication_context(self.runtime_authentication_context), + ) + if set(self.execution_binding_artifacts) != EXECUTION_BINDING_ARTIFACT_KEYS: + raise ValueError("execution_binding_artifacts keys differ from the frozen API") + normalized: dict[str, bytes] = {} + for key in sorted(EXECUTION_BINDING_ARTIFACT_KEYS): + value = self.execution_binding_artifacts[key] + if not isinstance(value, bytes): + raise TypeError(f"execution_binding_artifacts[{key!r}] must be bytes") + normalized[key] = bytes(value) + object.__setattr__(self, "execution_binding_artifacts", MappingProxyType(normalized)) + + +@dataclass(frozen=True, slots=True) +class AuthenticatedSequence: + """Exact token sequence and content commitments returned by an adapter.""" + + token_ids: tuple[int, ...] + source_content_sha256: str + formatted_content_sha256: str + generator_receipt_sha256: str | None + tokenizer_manifest_sha256: str + + +@dataclass(frozen=True, slots=True) +class StepObservation: + """One reviewed one-token recurrent-kernel observation. + + Tensor values use ``object`` so importing the contract cannot import a + tensor runtime. The authenticated runner performs the exact tensor checks. + """ + + position: int + token_id: int + layer_indices: tuple[int, ...] + recurrence_query: object + recurrent_state: object | None + successful_kernel_calls_per_layer: tuple[int, ...] + + +@dataclass(frozen=True, slots=True) +class ModelFileIdentity: + """One immutable file in the authenticated local model snapshot.""" + + name: str + size_bytes: int + sha256: str | None + git_blob_oid: str + lfs_sha256: str | None + lfs_size_bytes: int | None + + +@dataclass(frozen=True, slots=True) +class AuthenticatedModelFiles: + """Point-of-use model-file contract passed to ``load_model``.""" + + model_root: Path + model_id: str + revision: str + transformers_version: str + files: tuple[ModelFileIdentity, ...] + hub_tree_manifest_sha256: str + manifest_file_sha256: str + + +@runtime_checkable +class CalibrationAdapter(Protocol): + """Reviewed bridge from authenticated data/model files to causal tensors.""" + + def materialize_sequence(self, record: Mapping[str, object]) -> AuthenticatedSequence: ... + + def load_model(self, authenticated: AuthenticatedModelFiles) -> object: ... + + def begin_sequence(self, model: object, record: Mapping[str, object]) -> None: ... + + def step_token( + self, + model: object, + *, + token_id: int, + position: int, + capture_state: bool, + ) -> StepObservation: ... + + def end_sequence(self, model: object, record: Mapping[str, object]) -> None: ... + + def close_model(self, model: object) -> None: ... + + def runtime_metadata(self) -> Mapping[str, object]: ... + + +@runtime_checkable +class CalibrationAdapterFactory(Protocol): + """Pure constructor for the single reviewed live adapter.""" + + def __call__(self, context: AdapterConstructionContext, /) -> CalibrationAdapter: ... + + +__all__ = [ + "AdapterConstructionContext", + "AuthenticatedModelFiles", + "AuthenticatedSequence", + "CalibrationAdapter", + "CalibrationAdapterFactory", + "EXECUTION_BINDING_ARTIFACT_KEYS", + "ModelFileIdentity", + "RUNTIME_AUTHENTICATION_CONTEXT_KEYS", + "StepObservation", +] diff --git a/src/recurquant/experiment013_parquet.py b/src/recurquant/experiment013_parquet.py new file mode 100644 index 0000000..fe82955 --- /dev/null +++ b/src/recurquant/experiment013_parquet.py @@ -0,0 +1,1197 @@ +"""Immutable Parquet access for Experiment 013 calibration rows. + +The checked-in materialization manifest is both schema-validated and bound by +its canonical file digest. Live reads authenticate the immutable Hub source +and conversion commits plus every selected Parquet object's Git/LFS metadata +before and after one projected row-group read. + +Network and Parquet operations sit behind small protocols so the identity and +offset contracts can be tested without network access or row data. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Protocol + +EXPERIMENT013_PARQUET_MANIFEST_SCHEMA = ( + "recurquant.experiment013.parquet-materializations.v1" +) +EXPERIMENT013_PARQUET_MANIFEST_SHA256 = ( + "ee5628e50e5d3516fd79077542d355fd915455ac0e53128d372f4177ad63d39c" +) +EXPERIMENT013_PARQUET_MANIFEST_SIZE_BYTES = 3918 +EXPERIMENT013_PARQUET_MANIFEST_PATH = ( + Path(__file__).resolve().parents[2] + / "research" + / "experiment013-parquet-materializations.json" +) + +_TOP_LEVEL_FIELDS = frozenset({"schema", "datasets"}) +_DATASET_FIELDS = frozenset( + { + "conversion_revision", + "dataset_id", + "failed", + "files", + "partial", + "pending", + "selected_splits", + "source_revision", + } +) +_FILE_FIELDS = frozenset( + { + "config", + "git_blob_oid", + "immutable_path", + "lfs_sha256", + "lfs_size_bytes", + "logical_split", + "size_bytes", + } +) +_SHA1_RE = re.compile(r"[0-9a-f]{40}") +_SHA256_RE = re.compile(r"[0-9a-f]{64}") + + +class Experiment013ParquetError(RuntimeError): + """Raised when immutable Experiment 013 Parquet access fails closed.""" + + +class Experiment013ParquetOffsetError(IndexError): + """Raised when a split-relative global row offset is outside its bounds.""" + + +@dataclass(frozen=True, slots=True) +class Experiment013ParquetFile: + """One exact Parquet materialization object.""" + + config: str + logical_split: str + immutable_path: str + size_bytes: int + git_blob_oid: str + lfs_sha256: str + lfs_size_bytes: int + + +@dataclass(frozen=True, slots=True) +class Experiment013ParquetDataset: + """Frozen source/conversion identity for one calibration dataset.""" + + key: str + dataset_id: str + source_revision: str + conversion_revision: str + selected_splits: tuple[str, ...] + partial: bool + files: tuple[Experiment013ParquetFile, ...] + + +@dataclass(frozen=True, slots=True) +class Experiment013ParquetManifest: + """Validated immutable materialization inventory.""" + + schema: str + datasets: tuple[Experiment013ParquetDataset, ...] + + def dataset(self, key: str) -> Experiment013ParquetDataset: + for dataset in self.datasets: + if dataset.key == key: + return dataset + raise Experiment013ParquetError(f"unknown Experiment 013 dataset key: {key!r}") + + +@dataclass(frozen=True, slots=True) +class HubFileMetadata: + """Point-of-use Hub metadata for one immutable LFS object.""" + + path: str + commit_hash: str + size_bytes: int + git_blob_oid: str + lfs_sha256: str + lfs_size_bytes: int + etag: str + + +@dataclass(frozen=True, slots=True) +class HubDatasetMetadata: + """One ordered metadata snapshot at an exact conversion commit.""" + + commit_hash: str + files: tuple[HubFileMetadata, ...] + + +class HubMetadataBackend(Protocol): + """Injectable metadata-only Hugging Face Hub boundary.""" + + def resolve_dataset_revision(self, *, repo_id: str, revision: str) -> str: + """Resolve an exact dataset revision without opening row content.""" + + def snapshot_parquet_files( + self, + *, + repo_id: str, + revision: str, + paths: tuple[str, ...], + ) -> HubDatasetMetadata: + """Return ordered Hub/LFS metadata for the requested immutable paths.""" + + +@dataclass(frozen=True, slots=True) +class ParquetFileLayout: + """Footer-only layout needed to locate a row.""" + + row_group_rows: tuple[int, ...] + columns: tuple[str, ...] + + +class ParquetBackend(Protocol): + """Injectable Parquet footer and projected row-group boundary.""" + + def inspect(self, uri: str) -> ParquetFileLayout: + """Read only Parquet footer/schema metadata.""" + + def read_row( + self, + uri: str, + *, + row_group_index: int, + row_index_in_group: int, + columns: tuple[str, ...], + ) -> Mapping[str, object]: + """Read one projected row from exactly one row group.""" + + def read_row_group_projection( + self, + uri: str, + *, + row_group_index: int, + columns: tuple[str, ...], + ) -> Sequence[Mapping[str, object]]: + """Read requested columns from exactly one ordered row group.""" + + +@dataclass(frozen=True, slots=True) +class Experiment013ParquetRowLocation: + """Deterministic split-relative global-offset resolution.""" + + dataset_key: str + dataset_id: str + logical_split: str + global_offset: int + split_row_count: int + manifest_file_index: int + split_file_index: int + immutable_path: str + immutable_uri: str + file_row_index: int + row_group_index: int + row_index_in_group: int + row_group_row_count: int + + +@dataclass(frozen=True, slots=True) +class Experiment013ParquetRow: + """One authenticated, projected Experiment 013 row.""" + + location: Experiment013ParquetRowLocation + columns: tuple[str, ...] + values: Mapping[str, object] + + +@dataclass(frozen=True, slots=True) +class Experiment013ParquetProjectionRow: + """One immutable ID-only projection row in split-global order.""" + + global_offset: int + values: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class Experiment013ParquetProjection: + """Complete deterministic projection and its canonical commitment.""" + + dataset_key: str + dataset_id: str + logical_split: str + columns: tuple[str, ...] + rows: tuple[Experiment013ParquetProjectionRow, ...] + canonical_projection_sha256: str + + +_EXPECTED_MANIFEST = Experiment013ParquetManifest( + schema=EXPERIMENT013_PARQUET_MANIFEST_SCHEMA, + datasets=( + Experiment013ParquetDataset( + key="humaneval_plus", + dataset_id="evalplus/humanevalplus", + source_revision="d32357cf319e50e9c8d8dab5ea876c72b0fd321b", + conversion_revision="1cf4467306a94e0828b355ff1f32e9222d2d588a", + selected_splits=("test",), + partial=False, + files=( + Experiment013ParquetFile( + config="default", + logical_split="test", + immutable_path="default/test/0000.parquet", + size_bytes=2_902_210, + git_blob_oid="9877db06683d4245bc39aed18ee7cbad013ba5fa", + lfs_sha256=( + "4436f5c03d77c17e0cbc57543b90665b5c1266f55a43992a5ed7922cd34a7558" + ), + lfs_size_bytes=2_902_210, + ), + ), + ), + Experiment013ParquetDataset( + key="pg19", + dataset_id="emozilla/pg19", + source_revision="c021754c8e01c5b1cc83a1f549c1f97fbbb756b8", + conversion_revision="b3624dc44b60cb01e74876e8869234d2660812cf", + selected_splits=("train", "validation"), + partial=True, + files=( + Experiment013ParquetFile( + config="default", + logical_split="train", + immutable_path="default/partial-train/0000.parquet", + size_bytes=603_127_902, + git_blob_oid="00245b214ff9806a04f32debff0fd2e7b0737997", + lfs_sha256=( + "ea701af2e8a11bb8601150a47affff658452d687494dbed52a82d3b1fcf48811" + ), + lfs_size_bytes=603_127_902, + ), + Experiment013ParquetFile( + config="default", + logical_split="train", + immutable_path="default/partial-train/0001.parquet", + size_bytes=526_793_959, + git_blob_oid="1169b6deb8c1cd46a46f0ae752b68806c9b5cca9", + lfs_sha256=( + "5c1c025f46b4a6b52b56167efeb89a2b9378f9ea8a50cdf5ddcbca8c4e17db1f" + ), + lfs_size_bytes=526_793_959, + ), + Experiment013ParquetFile( + config="default", + logical_split="train", + immutable_path="default/partial-train/0002.parquet", + size_bytes=576_668_259, + git_blob_oid="b9413777553240574488fa36b74f4bd286c06719", + lfs_sha256=( + "80cc198a2ef5239bf22a496eb10e6afd6fba075c4f6dd3d26dae7ed82c3bb1ad" + ), + lfs_size_bytes=576_668_259, + ), + Experiment013ParquetFile( + config="default", + logical_split="train", + immutable_path="default/partial-train/0003.parquet", + size_bytes=583_939_098, + git_blob_oid="08f92b1ad15eb9f90268fa7cf9823523f1fb056a", + lfs_sha256=( + "326718129b7d13a9f45ae8e6e68ae90d95c15bf40fa457a053716832e4d07c1c" + ), + lfs_size_bytes=583_939_098, + ), + Experiment013ParquetFile( + config="default", + logical_split="train", + immutable_path="default/partial-train/0004.parquet", + size_bytes=588_756_614, + git_blob_oid="9cb48d05cf6568879582eb5bd894a7d1b34aee7b", + lfs_sha256=( + "c4dff8b2cd993d1bb6bded41eb0eef56dff5449753ea13c79e730b7a9e1f6907" + ), + lfs_size_bytes=588_756_614, + ), + Experiment013ParquetFile( + config="default", + logical_split="train", + immutable_path="default/partial-train/0005.parquet", + size_bytes=321_273_724, + git_blob_oid="6a1b9b38d31ca025cbf06192a3dfd66067eb7571", + lfs_sha256=( + "9ab4d07d379720a9b18e7e3a060a948e2338b7aa338e9534a12d52fbc4fd8e2e" + ), + lfs_size_bytes=321_273_724, + ), + Experiment013ParquetFile( + config="default", + logical_split="validation", + immutable_path="default/partial-validation/0000.parquet", + size_bytes=10_803_864, + git_blob_oid="3e86263f595fae38387a938ec882417649c2bbd4", + lfs_sha256=( + "81680529564d4ead1c0e3859509a62d86c7126c32afc95dce6bd98e729e491ef" + ), + lfs_size_bytes=10_803_864, + ), + ), + ), + ), +) + +# Bulk projection exists only to rank the complete frozen populations without +# decoding dataset content. Keep this allow-list separate from the generic +# one-row reader, which deliberately projects the selected row's content. +_FROZEN_ID_PROJECTION_COLUMNS = { + ("humaneval_plus", "test"): ("task_id",), + ("pg19", "train"): ("url",), + ("pg19", "validation"): ("url",), +} + + +def _exact_fields(value: Mapping[str, object], expected: frozenset[str], *, name: str) -> None: + if any(not isinstance(field, str) for field in value): + raise Experiment013ParquetError(f"{name} field names must be strings") + actual = frozenset(value) + if actual != expected: + missing = sorted(expected - actual) + extra = sorted(actual - expected) + raise Experiment013ParquetError( + f"{name} fields drifted (missing={missing}, extra={extra})" + ) + + +def _expect_string(value: object, expected: str, *, name: str) -> str: + if not isinstance(value, str) or value != expected: + raise Experiment013ParquetError(f"{name} drifted from its frozen value") + return value + + +def _expect_commit(value: object, expected: str, *, name: str) -> str: + if not isinstance(value, str) or _SHA1_RE.fullmatch(value) is None: + raise Experiment013ParquetError(f"{name} must be an immutable lowercase commit SHA") + return _expect_string(value, expected, name=name) + + +def _expect_sha1(value: object, expected: str, *, name: str) -> str: + if not isinstance(value, str) or _SHA1_RE.fullmatch(value) is None: + raise Experiment013ParquetError(f"{name} must be a lowercase SHA-1 object ID") + return _expect_string(value, expected, name=name) + + +def _expect_sha256(value: object, expected: str, *, name: str) -> str: + if not isinstance(value, str) or _SHA256_RE.fullmatch(value) is None: + raise Experiment013ParquetError(f"{name} must be a lowercase SHA-256 digest") + return _expect_string(value, expected, name=name) + + +def _expect_size(value: object, expected: int, *, name: str) -> int: + if type(value) is not int or value != expected: # bool is intentionally rejected + raise Experiment013ParquetError(f"{name} drifted from its frozen byte size") + return value + + +def _expect_path(value: object, expected: str, *, name: str) -> str: + if not isinstance(value, str) or not value or value != value.strip(): + raise Experiment013ParquetError(f"{name} must be a canonical immutable path") + path = PurePosixPath(value) + if ( + "\\" in value + or "\0" in value + or path.is_absolute() + or any(part in {"", ".", ".."} for part in path.parts) + or path.as_posix() != value + ): + raise Experiment013ParquetError(f"{name} must be a canonical immutable POSIX path") + return _expect_string(value, expected, name=name) + + +def validate_experiment013_parquet_manifest( + manifest: Mapping[str, object], +) -> Experiment013ParquetManifest: + """Strictly validate every field and ordered value in the frozen manifest.""" + + if not isinstance(manifest, Mapping): + raise Experiment013ParquetError("Parquet materialization manifest must be a mapping") + _exact_fields(manifest, _TOP_LEVEL_FIELDS, name="Parquet materialization manifest") + _expect_string( + manifest["schema"], + EXPERIMENT013_PARQUET_MANIFEST_SCHEMA, + name="Parquet materialization schema", + ) + raw_datasets = manifest["datasets"] + if not isinstance(raw_datasets, Mapping): + raise Experiment013ParquetError("datasets must be a mapping") + expected_keys = tuple(dataset.key for dataset in _EXPECTED_MANIFEST.datasets) + if tuple(raw_datasets) != expected_keys: + raise Experiment013ParquetError("dataset keys or their canonical order drifted") + + for expected_dataset in _EXPECTED_MANIFEST.datasets: + raw_dataset = raw_datasets[expected_dataset.key] + name = f"datasets.{expected_dataset.key}" + if not isinstance(raw_dataset, Mapping): + raise Experiment013ParquetError(f"{name} must be a mapping") + _exact_fields(raw_dataset, _DATASET_FIELDS, name=name) + _expect_string( + raw_dataset["dataset_id"], + expected_dataset.dataset_id, + name=f"{name}.dataset_id", + ) + _expect_commit( + raw_dataset["source_revision"], + expected_dataset.source_revision, + name=f"{name}.source_revision", + ) + _expect_commit( + raw_dataset["conversion_revision"], + expected_dataset.conversion_revision, + name=f"{name}.conversion_revision", + ) + if type(raw_dataset["selected_splits"]) is not list or tuple( + raw_dataset["selected_splits"] # type: ignore[arg-type] + ) != expected_dataset.selected_splits: + raise Experiment013ParquetError(f"{name}.selected_splits or order drifted") + if ( + type(raw_dataset["partial"]) is not bool + or raw_dataset["partial"] is not expected_dataset.partial + ): + raise Experiment013ParquetError(f"{name}.partial drifted") + for field in ("failed", "pending"): + if type(raw_dataset[field]) is not list or raw_dataset[field]: + raise Experiment013ParquetError(f"{name}.{field} must remain an empty list") + + raw_files = raw_dataset["files"] + if type(raw_files) is not list or len(raw_files) != len(expected_dataset.files): + raise Experiment013ParquetError(f"{name}.files inventory size drifted") + for index, expected_file in enumerate(expected_dataset.files): + raw_file = raw_files[index] + file_name = f"{name}.files[{index}]" + if not isinstance(raw_file, Mapping): + raise Experiment013ParquetError(f"{file_name} must be a mapping") + _exact_fields(raw_file, _FILE_FIELDS, name=file_name) + _expect_string(raw_file["config"], expected_file.config, name=f"{file_name}.config") + _expect_string( + raw_file["logical_split"], + expected_file.logical_split, + name=f"{file_name}.logical_split", + ) + _expect_path( + raw_file["immutable_path"], + expected_file.immutable_path, + name=f"{file_name}.immutable_path", + ) + _expect_size( + raw_file["size_bytes"], expected_file.size_bytes, name=f"{file_name}.size_bytes" + ) + _expect_sha1( + raw_file["git_blob_oid"], + expected_file.git_blob_oid, + name=f"{file_name}.git_blob_oid", + ) + _expect_sha256( + raw_file["lfs_sha256"], + expected_file.lfs_sha256, + name=f"{file_name}.lfs_sha256", + ) + _expect_size( + raw_file["lfs_size_bytes"], + expected_file.lfs_size_bytes, + name=f"{file_name}.lfs_size_bytes", + ) + return _EXPECTED_MANIFEST + + +def _manifest_as_dict(manifest: Experiment013ParquetManifest) -> dict[str, object]: + datasets: dict[str, object] = {} + for dataset in manifest.datasets: + datasets[dataset.key] = { + "conversion_revision": dataset.conversion_revision, + "dataset_id": dataset.dataset_id, + "failed": [], + "files": [ + { + "config": file.config, + "git_blob_oid": file.git_blob_oid, + "immutable_path": file.immutable_path, + "lfs_sha256": file.lfs_sha256, + "lfs_size_bytes": file.lfs_size_bytes, + "logical_split": file.logical_split, + "size_bytes": file.size_bytes, + } + for file in dataset.files + ], + "partial": dataset.partial, + "pending": [], + "selected_splits": list(dataset.selected_splits), + "source_revision": dataset.source_revision, + } + return {"datasets": datasets, "schema": manifest.schema} + + +def canonical_experiment013_parquet_manifest_bytes( + manifest: Mapping[str, object], +) -> bytes: + """Validate and encode the exact canonical checked-in manifest bytes.""" + + normalized = validate_experiment013_parquet_manifest(manifest) + return ( + json.dumps(_manifest_as_dict(normalized), indent=2, sort_keys=True, allow_nan=False) + "\n" + ).encode("utf-8") + + +def _reject_json_number(value: str) -> object: + raise Experiment013ParquetError(f"non-integer JSON number is forbidden: {value}") + + +def _reject_json_constant(value: str) -> object: + raise Experiment013ParquetError(f"non-finite JSON constant is forbidden: {value}") + + +def _unique_json_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise Experiment013ParquetError(f"duplicate JSON field is forbidden: {key}") + result[key] = value + return result + + +def load_experiment013_parquet_manifest( + path: str | Path = EXPERIMENT013_PARQUET_MANIFEST_PATH, +) -> Experiment013ParquetManifest: + """Load the byte-bound canonical manifest, rejecting any local drift.""" + + manifest_path = Path(path) + if manifest_path.is_symlink(): + raise Experiment013ParquetError("Parquet materialization manifest must not be a symlink") + try: + raw = manifest_path.read_bytes() + except OSError as error: + raise Experiment013ParquetError("Parquet materialization manifest is unreadable") from error + if len(raw) != EXPERIMENT013_PARQUET_MANIFEST_SIZE_BYTES: + raise Experiment013ParquetError("Parquet materialization manifest byte size drifted") + digest = hashlib.sha256(raw).hexdigest() + if digest != EXPERIMENT013_PARQUET_MANIFEST_SHA256: + raise Experiment013ParquetError("Parquet materialization manifest SHA-256 drifted") + try: + text = raw.decode("utf-8") + parsed = json.loads( + text, + object_pairs_hook=_unique_json_object, + parse_float=_reject_json_number, + parse_constant=_reject_json_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise Experiment013ParquetError( + "Parquet materialization manifest is not strict UTF-8 JSON" + ) from error + if not isinstance(parsed, Mapping): + raise Experiment013ParquetError("Parquet materialization manifest must be a JSON object") + normalized = validate_experiment013_parquet_manifest(parsed) + if canonical_experiment013_parquet_manifest_bytes(parsed) != raw: + raise Experiment013ParquetError("Parquet materialization manifest is not canonical JSON") + return normalized + + +def _object_field(value: object, name: str) -> object: + if isinstance(value, Mapping): + return value.get(name) + return getattr(value, name, None) + + +class HuggingFaceHubMetadataBackend: + """Metadata-only implementation using official Hub APIs and exact revisions.""" + + def __init__(self, *, token: str | bool | None = None) -> None: + self._token = token + + def resolve_dataset_revision(self, *, repo_id: str, revision: str) -> str: + try: + from huggingface_hub import HfApi + + info = HfApi(token=self._token).dataset_info( + repo_id=repo_id, + revision=revision, + files_metadata=False, + ) + resolved = info.sha + except Exception as error: + raise Experiment013ParquetError( + f"Hub could not resolve immutable dataset revision for {repo_id}" + ) from error + if not isinstance(resolved, str): + raise Experiment013ParquetError("Hub returned no resolved dataset commit") + return resolved + + def snapshot_parquet_files( + self, + *, + repo_id: str, + revision: str, + paths: tuple[str, ...], + ) -> HubDatasetMetadata: + try: + from huggingface_hub import HfApi, get_hf_file_metadata, hf_hub_url + + info = HfApi(token=self._token).dataset_info( + repo_id=repo_id, + revision=revision, + files_metadata=True, + ) + siblings = { + sibling.rfilename: sibling + for sibling in (info.siblings or ()) + if isinstance(sibling.rfilename, str) + } + files: list[HubFileMetadata] = [] + for path in paths: + sibling = siblings.get(path) + if sibling is None: + raise Experiment013ParquetError(f"Hub metadata omitted immutable path: {path}") + lfs = sibling.lfs + if lfs is None: + raise Experiment013ParquetError( + f"Hub path is not an authenticated LFS object: {path}" + ) + url = hf_hub_url( + repo_id=repo_id, + filename=path, + repo_type="dataset", + revision=revision, + ) + head = get_hf_file_metadata(url, token=self._token) + files.append( + HubFileMetadata( + path=path, + commit_hash=head.commit_hash, + size_bytes=head.size, + git_blob_oid=sibling.blob_id, + lfs_sha256=_object_field(lfs, "sha256"), # type: ignore[arg-type] + lfs_size_bytes=_object_field(lfs, "size"), # type: ignore[arg-type] + etag=head.etag, + ) + ) + commit_hash = info.sha + except Experiment013ParquetError: + raise + except Exception as error: + raise Experiment013ParquetError( + f"Hub could not authenticate immutable Parquet metadata for {repo_id}" + ) from error + if not isinstance(commit_hash, str): + raise Experiment013ParquetError("Hub returned no conversion commit") + return HubDatasetMetadata(commit_hash=commit_hash, files=tuple(files)) + + +class PyArrowParquetBackend: + """Read immutable ``hf://`` Parquet files through footer/range access.""" + + @staticmethod + def _dependencies() -> tuple[object, object]: + try: + import fsspec + import pyarrow.parquet as parquet + except ImportError as error: + raise Experiment013ParquetError( + "PyArrow Parquet access requires fsspec and pyarrow" + ) from error + return fsspec, parquet + + def inspect(self, uri: str) -> ParquetFileLayout: + fsspec, parquet = self._dependencies() + try: + with fsspec.open(uri, mode="rb") as stream: # type: ignore[attr-defined] + parquet_file = parquet.ParquetFile(stream) # type: ignore[attr-defined] + metadata = parquet_file.metadata + row_group_rows = tuple( + metadata.row_group(index).num_rows + for index in range(metadata.num_row_groups) + ) + columns = tuple(parquet_file.schema_arrow.names) + except Exception as error: + raise Experiment013ParquetError( + f"could not inspect immutable Parquet footer: {uri}" + ) from error + return ParquetFileLayout(row_group_rows=row_group_rows, columns=columns) + + def read_row( + self, + uri: str, + *, + row_group_index: int, + row_index_in_group: int, + columns: tuple[str, ...], + ) -> Mapping[str, object]: + fsspec, parquet = self._dependencies() + try: + with fsspec.open(uri, mode="rb") as stream: # type: ignore[attr-defined] + parquet_file = parquet.ParquetFile(stream) # type: ignore[attr-defined] + table = parquet_file.read_row_group(row_group_index, columns=list(columns)) + if row_index_in_group >= table.num_rows: + raise Experiment013ParquetError("Parquet row-group layout changed during read") + rows = table.slice(row_index_in_group, 1).to_pylist() + except Experiment013ParquetError: + raise + except Exception as error: + raise Experiment013ParquetError( + f"could not read immutable Parquet row: {uri}" + ) from error + if len(rows) != 1 or not isinstance(rows[0], Mapping): + raise Experiment013ParquetError("Parquet backend did not return exactly one row") + return rows[0] + + def read_row_group_projection( + self, + uri: str, + *, + row_group_index: int, + columns: tuple[str, ...], + ) -> Sequence[Mapping[str, object]]: + fsspec, parquet = self._dependencies() + try: + with fsspec.open(uri, mode="rb") as stream: # type: ignore[attr-defined] + parquet_file = parquet.ParquetFile(stream) # type: ignore[attr-defined] + rows = parquet_file.read_row_group( + row_group_index, + columns=list(columns), + ).to_pylist() + except Exception as error: + raise Experiment013ParquetError( + f"could not read immutable Parquet projection row group: {uri}" + ) from error + return rows + + +def _immutable_uri( + dataset: Experiment013ParquetDataset, + file: Experiment013ParquetFile, +) -> str: + return ( + f"hf://datasets/{dataset.dataset_id}@{dataset.conversion_revision}/" + f"{file.immutable_path}" + ) + + +def _selected_files( + dataset: Experiment013ParquetDataset, + logical_split: str, +) -> tuple[tuple[int, Experiment013ParquetFile], ...]: + if not isinstance(logical_split, str) or logical_split not in dataset.selected_splits: + raise Experiment013ParquetError( + f"split {logical_split!r} is not frozen for dataset {dataset.key!r}" + ) + files = tuple( + (index, file) + for index, file in enumerate(dataset.files) + if file.logical_split == logical_split + ) + if not files: + raise Experiment013ParquetError("frozen split has no Parquet files") + return files + + +def _authenticate_hub_snapshot( + hub: HubMetadataBackend, + dataset: Experiment013ParquetDataset, + files: tuple[tuple[int, Experiment013ParquetFile], ...], + *, + phase: str, +) -> HubDatasetMetadata: + try: + source_commit = hub.resolve_dataset_revision( + repo_id=dataset.dataset_id, + revision=dataset.source_revision, + ) + snapshot = hub.snapshot_parquet_files( + repo_id=dataset.dataset_id, + revision=dataset.conversion_revision, + paths=tuple(file.immutable_path for _, file in files), + ) + except Experiment013ParquetError: + raise + except Exception as error: + raise Experiment013ParquetError( + f"Hub metadata backend failed {phase} row access" + ) from error + if source_commit != dataset.source_revision: + raise Experiment013ParquetError(f"source commit drifted {phase} row access") + if not isinstance(snapshot, HubDatasetMetadata): + raise Experiment013ParquetError(f"Hub metadata snapshot is malformed {phase} row access") + if snapshot.commit_hash != dataset.conversion_revision: + raise Experiment013ParquetError(f"conversion commit drifted {phase} row access") + if len(snapshot.files) != len(files): + raise Experiment013ParquetError(f"Hub file inventory drifted {phase} row access") + for index, (observed, (_, expected)) in enumerate(zip(snapshot.files, files, strict=True)): + name = f"Hub files[{index}] {phase} row access" + if not isinstance(observed, HubFileMetadata): + raise Experiment013ParquetError(f"{name} is malformed") + _expect_path(observed.path, expected.immutable_path, name=f"{name}.path") + _expect_commit( + observed.commit_hash, + dataset.conversion_revision, + name=f"{name}.commit_hash", + ) + _expect_size(observed.size_bytes, expected.size_bytes, name=f"{name}.size_bytes") + _expect_sha1( + observed.git_blob_oid, + expected.git_blob_oid, + name=f"{name}.git_blob_oid", + ) + _expect_sha256( + observed.lfs_sha256, + expected.lfs_sha256, + name=f"{name}.lfs_sha256", + ) + _expect_size( + observed.lfs_size_bytes, + expected.lfs_size_bytes, + name=f"{name}.lfs_size_bytes", + ) + _expect_sha256(observed.etag, expected.lfs_sha256, name=f"{name}.etag") + return snapshot + + +def _validated_layout(layout: object, *, uri: str) -> ParquetFileLayout: + if not isinstance(layout, ParquetFileLayout): + raise Experiment013ParquetError(f"Parquet backend returned a malformed layout: {uri}") + if not layout.row_group_rows: + raise Experiment013ParquetError(f"Parquet file has no row groups: {uri}") + if any(type(count) is not int or count <= 0 for count in layout.row_group_rows): + raise Experiment013ParquetError(f"Parquet row-group counts are invalid: {uri}") + if ( + not layout.columns + or any(not isinstance(column, str) or not column for column in layout.columns) + or len(set(layout.columns)) != len(layout.columns) + ): + raise Experiment013ParquetError(f"Parquet schema columns are invalid: {uri}") + return layout + + +def _inspect_selected_files( + parquet: ParquetBackend, + dataset: Experiment013ParquetDataset, + files: tuple[tuple[int, Experiment013ParquetFile], ...], +) -> tuple[tuple[int, Experiment013ParquetFile, str, ParquetFileLayout], ...]: + inspected: list[tuple[int, Experiment013ParquetFile, str, ParquetFileLayout]] = [] + for manifest_index, file in files: + uri = _immutable_uri(dataset, file) + try: + raw_layout = parquet.inspect(uri) + except Experiment013ParquetError: + raise + except Exception as error: + raise Experiment013ParquetError( + "Parquet backend failed while reading a footer" + ) from error + inspected.append((manifest_index, file, uri, _validated_layout(raw_layout, uri=uri))) + return tuple(inspected) + + +def _locate_offset( + dataset: Experiment013ParquetDataset, + logical_split: str, + global_offset: int, + inspected: tuple[tuple[int, Experiment013ParquetFile, str, ParquetFileLayout], ...], +) -> Experiment013ParquetRowLocation: + if type(global_offset) is not int or global_offset < 0: + raise Experiment013ParquetOffsetError("global row offset must be a non-negative integer") + split_row_count = sum( + sum(layout.row_group_rows) for _, _, _, layout in inspected + ) + if global_offset >= split_row_count: + raise Experiment013ParquetOffsetError( + f"global row offset {global_offset} is outside split row count {split_row_count}" + ) + remaining = global_offset + for split_file_index, (manifest_index, file, uri, layout) in enumerate(inspected): + file_row_count = sum(layout.row_group_rows) + if remaining >= file_row_count: + remaining -= file_row_count + continue + file_row_index = remaining + for row_group_index, row_group_row_count in enumerate(layout.row_group_rows): + if remaining < row_group_row_count: + return Experiment013ParquetRowLocation( + dataset_key=dataset.key, + dataset_id=dataset.dataset_id, + logical_split=logical_split, + global_offset=global_offset, + split_row_count=split_row_count, + manifest_file_index=manifest_index, + split_file_index=split_file_index, + immutable_path=file.immutable_path, + immutable_uri=uri, + file_row_index=file_row_index, + row_group_index=row_group_index, + row_index_in_group=remaining, + row_group_row_count=row_group_row_count, + ) + remaining -= row_group_row_count + raise Experiment013ParquetError("validated Parquet offset could not be resolved") + + +def _normalize_projection( + columns: Sequence[str] | None, + layout: ParquetFileLayout, +) -> tuple[str, ...]: + if columns is None: + return layout.columns + if isinstance(columns, (str, bytes, bytearray)) or not isinstance(columns, Sequence): + raise Experiment013ParquetError("columns must be a sequence of unique column names") + normalized = tuple(columns) + if ( + not normalized + or any(not isinstance(column, str) or not column for column in normalized) + or len(set(normalized)) != len(normalized) + ): + raise Experiment013ParquetError("columns must be a non-empty unique string sequence") + missing = [column for column in normalized if column not in layout.columns] + if missing: + raise Experiment013ParquetError( + f"projected columns are absent from Parquet schema: {missing}" + ) + return normalized + + +def _projection_sha256( + *, + dataset_key: str, + logical_split: str, + columns: tuple[str, ...], + rows: tuple[Experiment013ParquetProjectionRow, ...], +) -> str: + payload = { + "columns": list(columns), + "dataset_key": dataset_key, + "logical_split": logical_split, + "rows": [ + {"global_offset": row.global_offset, "values": list(row.values)} for row in rows + ], + } + encoded = ( + json.dumps(payload, sort_keys=True, separators=(",", ":"), allow_nan=False) + "\n" + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def project_experiment013_parquet_columns( + dataset_key: str, + logical_split: str, + *, + columns: Sequence[str], + expected_count: int | None = None, + hub_backend: HubMetadataBackend | None = None, + parquet_backend: ParquetBackend | None = None, + manifest_path: str | Path = EXPERIMENT013_PARQUET_MANIFEST_PATH, +) -> Experiment013ParquetProjection: + """Authenticate and read a complete ordered string-column projection. + + Hub metadata and all relevant Parquet footers are authenticated once + before any projected values are decoded and once after the final row + group. Only ``columns`` are passed to the backend. Values are restricted to + strings because this surface exists solely for frozen canonical IDs. + """ + + if isinstance(columns, (str, bytes, bytearray)) or not isinstance(columns, Sequence): + raise Experiment013ParquetError("ID projection columns must be a sequence") + requested_columns = tuple(columns) + frozen_columns = ( + _FROZEN_ID_PROJECTION_COLUMNS.get((dataset_key, logical_split)) + if isinstance(dataset_key, str) and isinstance(logical_split, str) + else None + ) + if requested_columns != frozen_columns: + raise Experiment013ParquetError( + "bulk Parquet projection is restricted to the frozen canonical-ID column" + ) + if expected_count is not None and ( + type(expected_count) is not int or expected_count < 0 + ): + raise Experiment013ParquetError("expected projection count must be a non-negative integer") + hub = hub_backend if hub_backend is not None else HuggingFaceHubMetadataBackend() + parquet = parquet_backend if parquet_backend is not None else PyArrowParquetBackend() + manifest = load_experiment013_parquet_manifest(manifest_path) + dataset = manifest.dataset(dataset_key) + files = _selected_files(dataset, logical_split) + before = _authenticate_hub_snapshot(hub, dataset, files, phase="before") + inspected = _inspect_selected_files(parquet, dataset, files) + projection: tuple[str, ...] | None = None + split_row_count = sum(sum(layout.row_group_rows) for _, _, _, layout in inspected) + if expected_count is not None and split_row_count != expected_count: + raise Experiment013ParquetError( + "Parquet footer row count differs from the frozen projection population" + ) + rows: list[Experiment013ParquetProjectionRow] = [] + for _manifest_index, _file, uri, layout in inspected: + normalized = _normalize_projection(requested_columns, layout) + if projection is None: + projection = normalized + elif normalized != projection: + raise Experiment013ParquetError("projected Parquet schemas differ across files") + for row_group_index, row_group_count in enumerate(layout.row_group_rows): + try: + raw_rows = parquet.read_row_group_projection( + uri, + row_group_index=row_group_index, + columns=projection, + ) + except Experiment013ParquetError: + raise + except Exception as error: + raise Experiment013ParquetError( + "Parquet backend failed while reading an ID projection" + ) from error + if ( + isinstance(raw_rows, (str, bytes, bytearray)) + or not isinstance(raw_rows, Sequence) + or len(raw_rows) != row_group_count + ): + raise Experiment013ParquetError( + "Parquet projection backend row count differs from authenticated footer" + ) + for raw_row in raw_rows: + if not isinstance(raw_row, Mapping) or set(raw_row) != set(projection): + raise Experiment013ParquetError( + "Parquet projection backend returned columns outside the projection" + ) + values: list[str] = [] + for column in projection: + value = raw_row[column] + if not isinstance(value, str) or not value: + raise Experiment013ParquetError( + "Parquet canonical-ID projection values must be non-empty strings" + ) + values.append(value) + rows.append( + Experiment013ParquetProjectionRow( + global_offset=len(rows), + values=tuple(values), + ) + ) + if projection is None: + raise Experiment013ParquetError("Parquet projection contains no files") + frozen_rows = tuple(rows) + if len(frozen_rows) != split_row_count: + raise Experiment013ParquetError("Parquet projection population drifted") + after = _authenticate_hub_snapshot(hub, dataset, files, phase="after") + if after != before: + raise Experiment013ParquetError("Hub metadata changed during Parquet projection") + return Experiment013ParquetProjection( + dataset_key=dataset.key, + dataset_id=dataset.dataset_id, + logical_split=logical_split, + columns=projection, + rows=frozen_rows, + canonical_projection_sha256=_projection_sha256( + dataset_key=dataset.key, + logical_split=logical_split, + columns=projection, + rows=frozen_rows, + ), + ) + + +def locate_experiment013_parquet_row( + dataset_key: str, + logical_split: str, + global_offset: int, + *, + hub_backend: HubMetadataBackend | None = None, + parquet_backend: ParquetBackend | None = None, + manifest_path: str | Path = EXPERIMENT013_PARQUET_MANIFEST_PATH, +) -> Experiment013ParquetRowLocation: + """Authenticate and map one split-relative global offset without reading rows.""" + + hub = hub_backend if hub_backend is not None else HuggingFaceHubMetadataBackend() + parquet = parquet_backend if parquet_backend is not None else PyArrowParquetBackend() + manifest = load_experiment013_parquet_manifest(manifest_path) + dataset = manifest.dataset(dataset_key) + files = _selected_files(dataset, logical_split) + before = _authenticate_hub_snapshot(hub, dataset, files, phase="before") + inspected = _inspect_selected_files(parquet, dataset, files) + location = _locate_offset(dataset, logical_split, global_offset, inspected) + after = _authenticate_hub_snapshot(hub, dataset, files, phase="after") + if after != before: + raise Experiment013ParquetError("Hub metadata changed during Parquet offset resolution") + return location + + +def read_experiment013_parquet_row( + dataset_key: str, + logical_split: str, + global_offset: int, + *, + columns: Sequence[str] | None = None, + hub_backend: HubMetadataBackend | None = None, + parquet_backend: ParquetBackend | None = None, + manifest_path: str | Path = EXPERIMENT013_PARQUET_MANIFEST_PATH, +) -> Experiment013ParquetRow: + """Read exactly one projected row from an immutable conversion commit. + + The split-relative offset is resolved from ordered Parquet footer row-group + counts. Only the containing row group and requested columns are decoded; + the returned mapping contains exactly one row. + """ + + hub = hub_backend if hub_backend is not None else HuggingFaceHubMetadataBackend() + parquet = parquet_backend if parquet_backend is not None else PyArrowParquetBackend() + manifest = load_experiment013_parquet_manifest(manifest_path) + dataset = manifest.dataset(dataset_key) + files = _selected_files(dataset, logical_split) + before = _authenticate_hub_snapshot(hub, dataset, files, phase="before") + inspected = _inspect_selected_files(parquet, dataset, files) + location = _locate_offset(dataset, logical_split, global_offset, inspected) + target_layout = inspected[location.split_file_index][3] + projection = _normalize_projection(columns, target_layout) + try: + raw_values = parquet.read_row( + location.immutable_uri, + row_group_index=location.row_group_index, + row_index_in_group=location.row_index_in_group, + columns=projection, + ) + except Experiment013ParquetError: + raise + except Exception as error: + raise Experiment013ParquetError("Parquet backend failed while reading one row") from error + after = _authenticate_hub_snapshot(hub, dataset, files, phase="after") + if after != before: + raise Experiment013ParquetError("Hub metadata changed during Parquet row access") + if not isinstance(raw_values, Mapping): + raise Experiment013ParquetError("Parquet backend returned a non-mapping row") + if set(raw_values) != set(projection): + raise Experiment013ParquetError("Parquet backend returned columns outside the projection") + values = {column: raw_values[column] for column in projection} + return Experiment013ParquetRow(location=location, columns=projection, values=values) + + +__all__ = [ + "EXPERIMENT013_PARQUET_MANIFEST_PATH", + "EXPERIMENT013_PARQUET_MANIFEST_SCHEMA", + "EXPERIMENT013_PARQUET_MANIFEST_SHA256", + "EXPERIMENT013_PARQUET_MANIFEST_SIZE_BYTES", + "Experiment013ParquetDataset", + "Experiment013ParquetError", + "Experiment013ParquetFile", + "Experiment013ParquetManifest", + "Experiment013ParquetOffsetError", + "Experiment013ParquetProjection", + "Experiment013ParquetProjectionRow", + "Experiment013ParquetRow", + "Experiment013ParquetRowLocation", + "HubDatasetMetadata", + "HubFileMetadata", + "HubMetadataBackend", + "HuggingFaceHubMetadataBackend", + "ParquetBackend", + "ParquetFileLayout", + "PyArrowParquetBackend", + "canonical_experiment013_parquet_manifest_bytes", + "load_experiment013_parquet_manifest", + "locate_experiment013_parquet_row", + "project_experiment013_parquet_columns", + "read_experiment013_parquet_row", + "validate_experiment013_parquet_manifest", +] diff --git a/src/recurquant/experiment013_qwen35_adapter.py b/src/recurquant/experiment013_qwen35_adapter.py new file mode 100644 index 0000000..ee5dc5b --- /dev/null +++ b/src/recurquant/experiment013_qwen35_adapter.py @@ -0,0 +1,1094 @@ +"""Reviewed live Qwen3.5 adapter for Experiment 013 calibration. + +The factory in this module is deliberately inert. Dataset/tokenizer access is +deferred until the first sequence materialization, and model-file access is +deferred until :meth:`Experiment013Qwen35Adapter.load_model`. The adapter uses +the canonical capture module for all formatting and observes the exact +post-convolution query and returned recurrent state at the pinned Transformers +Gated DeltaNet kernel boundary. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import stat +import sys +from collections.abc import Mapping, Sequence +from contextvars import ContextVar, Token +from dataclasses import dataclass +from pathlib import Path +from types import ModuleType +from typing import Any, Final + +import torch + +from .experiment013_calibration_api import ( + AdapterConstructionContext, + AuthenticatedModelFiles, + AuthenticatedSequence, + StepObservation, +) + +ADAPTER_REVISION: Final = "experiment-013-qwen35-live-adapter-v1" +ADAPTER_SOURCE_PATH: Final = "src/recurquant/experiment013_qwen35_adapter.py" +CAPTURE_SOURCE_PATH: Final = "scripts/capture_static_q468_identity_input.py" +CAPTURE_MODULE_NAME: Final = "_recurquant_experiment013_capture_for_live_adapter" +SOURCE_MANIFEST_SCHEMA: Final = "recurquant.experiment013.source-manifest.v1" + +MODEL_ID: Final = "Qwen/Qwen3.5-0.8B-Base" +MODEL_REVISION: Final = "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68" +TRANSFORMERS_VERSION: Final = "5.14.1" +MODEL_DTYPE_NAME: Final = "bfloat16" + +RECURRENT_LAYER_INDICES: Final = ( + 0, + 1, + 2, + 4, + 5, + 6, + 8, + 9, + 10, + 12, + 13, + 14, + 16, + 17, + 18, + 20, + 21, + 22, +) +LAYER_TYPES: Final = tuple( + "full_attention" if (index + 1) % 4 == 0 else "linear_attention" for index in range(24) +) +QUERY_SHAPE: Final = (1, 1, 16, 128) +STATE_SHAPE: Final = (1, 16, 128, 128) +_SHA256_RE: Final = re.compile(r"[0-9a-f]{64}") +_WEIGHT_FILE_RE: Final = re.compile( + r"(?:model(?:-[0-9]+-of-[0-9]+)?|model\.safetensors-[0-9]+-of-[0-9]+)\.safetensors" +) +_OBSERVER_ATTRIBUTE: Final = "_recurquant_experiment013_qwen35_observer" +_FILE_ATTRIBUTE_REPARSE_POINT: Final = 0x0400 + +_GATED_DELTA_GEOMETRY: Final = { + "hidden_size": 1024, + "num_v_heads": 16, + "num_k_heads": 16, + "head_k_dim": 128, + "head_v_dim": 128, + "key_dim": 2048, + "value_dim": 2048, + "conv_kernel_size": 4, + "conv_dim": 6144, +} + +_LOADING_DIAGNOSTIC_TYPES: Final = { + "missing_keys": set, + "unexpected_keys": set, + "mismatched_keys": set, + "error_msgs": list, +} + +_CAPTURE_BINDING_KEYS: Final = { + "repository_source_manifest_file_sha256": "repository_source_manifest_bytes", + "calibration_runtime_manifest_file_sha256": "calibration_runtime_manifest_bytes", + "model_file_manifest_file_sha256": "model_file_manifest_bytes", + "parquet_materialization_manifest_file_sha256": ("parquet_materialization_manifest_bytes"), +} + + +class Experiment013AdapterError(RuntimeError): + """Raised when the reviewed adapter cannot prove its live observation.""" + + +def _require_sha256(value: object, *, context: str) -> str: + if not isinstance(value, str) or _SHA256_RE.fullmatch(value) is None: + raise Experiment013AdapterError(f"{context} must be a lowercase SHA-256") + return value + + +def _require_non_negative_int(value: object, *, context: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise Experiment013AdapterError(f"{context} must be a non-negative integer") + return value + + +def _argument( + args: tuple[object, ...], + kwargs: Mapping[str, object], + name: str, + position: int, +) -> object | None: + if name in kwargs: + return kwargs[name] + return args[position] if len(args) > position else None + + +def _capture_manifest_sha256(source_manifest_bytes: bytes) -> str: + if not isinstance(source_manifest_bytes, bytes): + raise Experiment013AdapterError("repository source manifest must be exact bytes") + try: + manifest = json.loads(source_manifest_bytes.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise Experiment013AdapterError("repository source manifest is not UTF-8 JSON") from error + if not isinstance(manifest, dict) or manifest.get("schema") != SOURCE_MANIFEST_SCHEMA: + raise Experiment013AdapterError("repository source manifest schema drifted") + entries = manifest.get("paths") + if not isinstance(entries, list): + raise Experiment013AdapterError("repository source manifest paths are not a list") + matches = [ + entry + for entry in entries + if isinstance(entry, dict) and entry.get("path") == CAPTURE_SOURCE_PATH + ] + if len(matches) != 1: + raise Experiment013AdapterError( + "repository source manifest must bind exactly one capture source" + ) + return _require_sha256( + matches[0].get("raw_sha256"), + context="repository source manifest capture source", + ) + + +def _assert_no_link_or_reparse(repository_root: Path, source_path: Path) -> None: + root = Path(os.path.abspath(repository_root)) + path = Path(os.path.abspath(source_path)) + try: + path.relative_to(root) + except ValueError as error: + raise Experiment013AdapterError( + "capture source resolves outside repository_root" + ) from error + candidates = [root] + current = root + for part in path.relative_to(root).parts: + current /= part + candidates.append(current) + for candidate in candidates: + try: + status = candidate.lstat() + except OSError as error: + raise Experiment013AdapterError("capture source path is unavailable") from error + if stat.S_ISLNK(status.st_mode) or ( + getattr(status, "st_file_attributes", 0) & _FILE_ATTRIBUTE_REPARSE_POINT + ): + raise Experiment013AdapterError("capture source path traverses a link or reparse point") + + +def _read_authenticated_capture_source( + repository_root: Path, + source_path: Path, + expected_sha256: str, +) -> bytes: + _assert_no_link_or_reparse(repository_root, source_path) + try: + before = source_path.stat(follow_symlinks=False) + payload = source_path.read_bytes() + after = source_path.stat(follow_symlinks=False) + except OSError as error: + raise Experiment013AdapterError("authenticated capture source is unreadable") from error + _assert_no_link_or_reparse(repository_root, source_path) + if not stat.S_ISREG(before.st_mode) or not stat.S_ISREG(after.st_mode): + raise Experiment013AdapterError("authenticated capture source is not a regular file") + identity_fields = ("st_dev", "st_ino", "st_size", "st_mtime_ns") + if any(getattr(before, field) != getattr(after, field) for field in identity_fields): + raise Experiment013AdapterError("capture source changed while it was read") + if hashlib.sha256(payload).hexdigest() != expected_sha256: + raise Experiment013AdapterError( + "capture source differs from the repository source manifest" + ) + return payload + + +@dataclass(frozen=True, slots=True) +class _CaptureModuleBinding: + module: ModuleType + repository_root: Path + source_path: Path + raw_sha256: str + + +def _verify_capture_binding(binding: _CaptureModuleBinding) -> None: + if not isinstance(binding, _CaptureModuleBinding): + raise Experiment013AdapterError("capture module lacks an authenticated source binding") + if binding.module.__name__ != CAPTURE_MODULE_NAME or getattr( + binding.module, "__file__", None + ) != str(binding.source_path): + raise Experiment013AdapterError("capture module identity differs from its source binding") + if CAPTURE_MODULE_NAME in sys.modules: + raise Experiment013AdapterError( + "capture module name was preloaded outside the exact loader" + ) + _read_authenticated_capture_source( + binding.repository_root, + binding.source_path, + binding.raw_sha256, + ) + + +def _load_capture_module( + repository_root: Path, + source_manifest_bytes: bytes, +) -> _CaptureModuleBinding: + """Execute only the manifest-bound capture bytes and recheck them afterwards.""" + + if CAPTURE_MODULE_NAME in sys.modules: + raise Experiment013AdapterError("capture module name is already loaded") + root = Path(os.path.abspath(repository_root)) + path = root / Path(CAPTURE_SOURCE_PATH) + expected_sha256 = _capture_manifest_sha256(source_manifest_bytes) + payload = _read_authenticated_capture_source(root, path, expected_sha256) + if CAPTURE_MODULE_NAME in sys.modules: + raise Experiment013AdapterError("capture module name is already loaded") + module = ModuleType(CAPTURE_MODULE_NAME) + module.__file__ = str(path) + module.__package__ = "" + module.__spec__ = None + sys.modules[CAPTURE_MODULE_NAME] = module + try: + code = compile(payload, str(path), "exec", dont_inherit=True) + exec(code, module.__dict__) + if sys.modules.get(CAPTURE_MODULE_NAME) is not module: + raise Experiment013AdapterError("capture module replaced its exact loader binding") + finally: + sys.modules.pop(CAPTURE_MODULE_NAME, None) + _read_authenticated_capture_source(root, path, expected_sha256) + return _CaptureModuleBinding( + module=module, + repository_root=root, + source_path=path, + raw_sha256=expected_sha256, + ) + + +@dataclass(frozen=True, slots=True) +class _TransformersRuntime: + version: str + qwen_config_class: type[Any] + qwen_model_class: type[Any] + qwen_gated_delta_net_class: type[Any] + dynamic_cache_class: type[Any] + torch_chunk_gated_delta_rule: Any + torch_recurrent_gated_delta_rule: Any + torch_causal_conv1d_update: Any + + +def _load_transformers_runtime() -> _TransformersRuntime: + import transformers + from transformers import DynamicCache, Qwen3_5ForCausalLM, Qwen3_5TextConfig + from transformers.models.qwen3_5.modeling_qwen3_5 import ( + Qwen3_5GatedDeltaNet, + torch_causal_conv1d_update, + torch_chunk_gated_delta_rule, + torch_recurrent_gated_delta_rule, + ) + + return _TransformersRuntime( + version=str(transformers.__version__), + qwen_config_class=Qwen3_5TextConfig, + qwen_model_class=Qwen3_5ForCausalLM, + qwen_gated_delta_net_class=Qwen3_5GatedDeltaNet, + dynamic_cache_class=DynamicCache, + torch_chunk_gated_delta_rule=torch_chunk_gated_delta_rule, + torch_recurrent_gated_delta_rule=torch_recurrent_gated_delta_rule, + torch_causal_conv1d_update=torch_causal_conv1d_update, + ) + + +def _validate_qwen_config(config: object) -> None: + exact = { + "model_type": "qwen3_5_text", + "hidden_size": 1024, + "num_hidden_layers": 24, + "linear_num_key_heads": 16, + "linear_num_value_heads": 16, + "linear_key_head_dim": 128, + "linear_value_head_dim": 128, + "linear_conv_kernel_dim": 4, + } + for name, expected in exact.items(): + if getattr(config, name, None) != expected: + raise Experiment013AdapterError( + f"authenticated Qwen3.5 config {name} differs from {expected!r}" + ) + layer_types = getattr(config, "layer_types", None) + if not isinstance(layer_types, (list, tuple)) or tuple(layer_types) != LAYER_TYPES: + raise Experiment013AdapterError("authenticated Qwen3.5 layer schedule drifted") + + +def _qwen_modules( + model: object, + runtime: _TransformersRuntime, +) -> tuple[tuple[int, object], ...]: + text_model = getattr(model, "model", None) + layers = getattr(text_model, "layers", None) + if layers is None or not hasattr(layers, "__len__") or not hasattr(layers, "__getitem__"): + raise Experiment013AdapterError("loaded Qwen3.5 model does not expose indexed layers") + if len(layers) != len(LAYER_TYPES): + raise Experiment013AdapterError("loaded Qwen3.5 model does not expose exactly 24 layers") + result: list[tuple[int, object]] = [] + for layer_index in RECURRENT_LAYER_INDICES: + module = getattr(layers[layer_index], "linear_attn", None) + if module is None or type(module) is not runtime.qwen_gated_delta_net_class: + raise Experiment013AdapterError( + f"Qwen3.5 layer {layer_index} is not the pinned Gated DeltaNet module" + ) + if getattr(module, "layer_idx", None) != layer_index: + raise Experiment013AdapterError("Qwen3.5 Gated DeltaNet layer index drifted") + for name, expected in _GATED_DELTA_GEOMETRY.items(): + if getattr(module, name, None) != expected: + raise Experiment013AdapterError( + f"Qwen3.5 Gated DeltaNet {name} geometry drifted at layer {layer_index}" + ) + if getattr(module, "layer_type", None) != "linear_attention": + raise Experiment013AdapterError("Qwen3.5 Gated DeltaNet layer type drifted") + result.append((layer_index, module)) + return tuple(result) + + +def _freeze_torch_fallbacks( + modules: Sequence[tuple[int, object]], runtime: _TransformersRuntime +) -> None: + for _layer_index, module in modules: + module.causal_conv1d_fn = None # type: ignore[attr-defined] + module.causal_conv1d_update = runtime.torch_causal_conv1d_update # type: ignore[attr-defined] + module.chunk_gated_delta_rule = runtime.torch_chunk_gated_delta_rule # type: ignore[attr-defined] + module.recurrent_gated_delta_rule = ( # type: ignore[attr-defined] + runtime.torch_recurrent_gated_delta_rule + ) + if ( + module.causal_conv1d_fn is not None # type: ignore[attr-defined] + or module.causal_conv1d_update # type: ignore[attr-defined] + is not runtime.torch_causal_conv1d_update + or module.chunk_gated_delta_rule # type: ignore[attr-defined] + is not runtime.torch_chunk_gated_delta_rule + or module.recurrent_gated_delta_rule # type: ignore[attr-defined] + is not runtime.torch_recurrent_gated_delta_rule + ): + raise Experiment013AdapterError("could not freeze the pure-Torch Qwen3.5 kernels") + + +def _model_devices(model: object) -> set[torch.device]: + parameters = getattr(model, "parameters", None) + buffers = getattr(model, "buffers", None) + if not callable(parameters) or not callable(buffers): + raise Experiment013AdapterError("loaded model does not expose parameters and buffers") + devices = {parameter.device for parameter in parameters()} + devices.update(buffer.device for buffer in buffers()) + return devices + + +@dataclass(slots=True) +class _KernelReceipt: + layer_index: int + kernel_name: str + query: torch.Tensor + final_state: torch.Tensor + + +@dataclass(slots=True) +class _StepCapture: + cache: object + position: int + receipts: dict[int, _KernelReceipt] + + @property + def expected_kernel(self) -> str: + return "chunk_gated_delta_rule" if self.position == 0 else "recurrent_gated_delta_rule" + + +class _Qwen35StepObserver: + """Token-scoped observer for the exact pinned Gated DeltaNet kernels.""" + + _POSITIONS: Final = { + "chunk_gated_delta_rule": { + "initial_state": 6, + "output_final_state": 7, + "use_qk_l2norm_in_kernel": 8, + }, + "recurrent_gated_delta_rule": { + "initial_state": 5, + "output_final_state": 6, + "use_qk_l2norm_in_kernel": 7, + }, + } + + def __init__( + self, + modules: Sequence[tuple[int, object]], + *, + query_device: torch.device, + _allow_test_non_cuda: bool = False, + ) -> None: + normalized = tuple(modules) + if tuple(index for index, _module in normalized) != RECURRENT_LAYER_INDICES: + raise Experiment013AdapterError("observer modules differ from frozen recurrent layers") + if not isinstance(query_device, torch.device): + raise TypeError("query_device must be torch.device") + if query_device.type != "cuda" and not _allow_test_non_cuda: + raise Experiment013AdapterError("Qwen3.5 query device must be CUDA") + self.modules = normalized + self.query_device = query_device + self._active: ContextVar[_StepCapture | None] = ContextVar( + f"recurquant_experiment013_qwen35_step_{id(self)}", + default=None, + ) + self._installed: list[tuple[object, str, object]] = [] + + @staticmethod + def _cache_state(cache: object, layer_index: int) -> torch.Tensor | None: + layers = getattr(cache, "layers", None) + if layers is None or not hasattr(layers, "__len__") or not hasattr(layers, "__getitem__"): + return None + if layer_index >= len(layers): + return None + recurrent_states = getattr(layers[layer_index], "recurrent_states", None) + if not isinstance(recurrent_states, Mapping): + return None + state = recurrent_states.get(0) + return state if isinstance(state, torch.Tensor) else None + + def _make_wrapper(self, layer_index: int, kernel_name: str, original: Any): + positions = self._POSITIONS[kernel_name] + + def wrapped(*args: object, **kwargs: object): + capture = self._active.get() + if capture is None: + raise Experiment013AdapterError( + "Qwen3.5 state kernel ran outside an authenticated one-token step" + ) + if kernel_name != capture.expected_kernel: + raise Experiment013AdapterError( + f"position {capture.position} called unexpected {kernel_name}" + ) + if layer_index in capture.receipts: + raise Experiment013AdapterError( + f"Qwen3.5 layer {layer_index} called more than one state kernel" + ) + query = _argument(args, kwargs, "query", 0) + if ( + not isinstance(query, torch.Tensor) + or tuple(query.shape) != QUERY_SHAPE + or query.dtype != torch.bfloat16 + or query.device != self.query_device + ): + raise Experiment013AdapterError( + f"Qwen3.5 layer {layer_index} query must be BF16 {QUERY_SHAPE} " + f"on {self.query_device}" + ) + initial_state = _argument( + args, + kwargs, + "initial_state", + positions["initial_state"], + ) + if capture.position == 0: + if initial_state is not None: + raise Experiment013AdapterError("the first token must have no recurrent state") + else: + cached = self._cache_state(capture.cache, layer_index) + if not isinstance(initial_state, torch.Tensor) or initial_state is not cached: + raise Experiment013AdapterError( + "cached one-token recurrence did not use the exact persistent state" + ) + if ( + _argument( + args, + kwargs, + "output_final_state", + positions["output_final_state"], + ) + is not True + or _argument( + args, + kwargs, + "use_qk_l2norm_in_kernel", + positions["use_qk_l2norm_in_kernel"], + ) + is not True + ): + raise Experiment013AdapterError( + "Qwen3.5 state kernel omitted the pinned final-state/L2-normalization flags" + ) + + output = original(*args, **kwargs) + if not isinstance(output, tuple) or len(output) != 2: + raise Experiment013AdapterError( + "Qwen3.5 state kernel did not return exactly (output, final_state)" + ) + final_state = output[1] + if ( + not isinstance(final_state, torch.Tensor) + or tuple(final_state.shape) != STATE_SHAPE + or final_state.dtype != torch.float32 + or final_state.device != query.device + ): + raise Experiment013AdapterError( + f"Qwen3.5 layer {layer_index} final state must be FP32 {STATE_SHAPE}" + ) + # The receipt becomes visible only after the selected kernel returned. + capture.receipts[layer_index] = _KernelReceipt( + layer_index=layer_index, + kernel_name=kernel_name, + query=query, + final_state=final_state, + ) + return output + + return wrapped + + def install(self) -> None: + if self._installed: + raise Experiment013AdapterError("Qwen3.5 step observer is already installed") + try: + for layer_index, module in self.modules: + if hasattr(module, _OBSERVER_ATTRIBUTE): + raise Experiment013AdapterError( + "another Experiment 013 observer is already installed" + ) + setattr(module, _OBSERVER_ATTRIBUTE, self) + self._installed.append((module, _OBSERVER_ATTRIBUTE, _Missing)) + for kernel_name in self._POSITIONS: + original = getattr(module, kernel_name, None) + if not callable(original): + raise Experiment013AdapterError( + f"Qwen3.5 module has no callable {kernel_name}" + ) + restore = ( + original if kernel_name in getattr(module, "__dict__", {}) else _Missing + ) + setattr( + module, + kernel_name, + self._make_wrapper(layer_index, kernel_name, original), + ) + self._installed.append((module, kernel_name, restore)) + except BaseException: + self.remove() + raise + + def remove(self) -> None: + while self._installed: + target, name, original = self._installed.pop() + if original is _Missing: + if name in getattr(target, "__dict__", {}): + delattr(target, name) + else: + setattr(target, name, original) + + def activate(self, capture: _StepCapture) -> Token[_StepCapture | None]: + if self._active.get() is not None: + raise Experiment013AdapterError("nested Qwen3.5 token observation is forbidden") + return self._active.set(capture) + + def deactivate(self, token: Token[_StepCapture | None]) -> None: + self._active.reset(token) + + @property + def is_idle(self) -> bool: + return self._active.get() is None + + +class _MissingType: + pass + + +_Missing = _MissingType() + + +@dataclass(slots=True) +class _SequenceState: + cache: object + identity_record_sha256: str + token_count: int + next_position: int = 0 + + +class Experiment013Qwen35Adapter: + """Canonical materialization and one-token Qwen3.5 observation adapter.""" + + def __init__(self, context: AdapterConstructionContext) -> None: + if not isinstance(context, AdapterConstructionContext): + raise TypeError("context must be AdapterConstructionContext") + # Do not resolve, stat, import, or open any path during construction. + self._repository_root = Path(context.repository_root) + self._model_root = Path(context.model_root) + self._cache_root = Path(context.cache_root) + self._ruler_root = Path(context.ruler_root) + self._execution_binding_artifacts: Mapping[str, bytes] | None = dict( + context.execution_binding_artifacts + ) + self._runtime_authentication_context: Mapping[str, object] | None = ( + context.runtime_authentication_context + ) + self._materialization_attempted = False + self._materialized_sequences: dict[str, AuthenticatedSequence] | None = None + self._capture_input_sha256: str | None = None + self._token_sequence_manifest_sha256: str | None = None + self._runtime: _TransformersRuntime | None = None + self._model: object | None = None + self._model_device: torch.device | None = None + self._observer: _Qwen35StepObserver | None = None + self._model_loading_diagnostic_counts: dict[str, int] | None = None + self._sequence: _SequenceState | None = None + self._sequence_failed = False + + def _prepare_materialization(self) -> None: + if self._materialization_attempted: + if self._materialized_sequences is None: + raise Experiment013AdapterError("calibration materialization previously failed") + return + self._materialization_attempted = True + bindings = self._execution_binding_artifacts + try: + if bindings is None: + raise Experiment013AdapterError("execution-binding artifacts were already released") + runtime_context = self._runtime_authentication_context + if runtime_context is None: + raise Experiment013AdapterError("runtime authentication context was released") + source_manifest_bytes = bytes(bindings["repository_source_manifest_bytes"]) + capture_binding = _load_capture_module( + self._repository_root, + source_manifest_bytes, + ) + capture = capture_binding.module + source_class = getattr(capture, "LiveCaptureSource", None) + materialize = getattr(capture, "materialize_calibration_identity_sequences", None) + if not callable(source_class) or not callable(materialize): + raise Experiment013AdapterError( + "authenticated capture module lacks the calibration materialization API" + ) + source = source_class( + cache_dir=self._cache_root, + ruler_receipt_dir=self._ruler_root, + ) + capture_bindings = { + capture_key: bytes(bindings[context_key]) + for capture_key, context_key in _CAPTURE_BINDING_KEYS.items() + } + _verify_capture_binding(capture_binding) + try: + materialization = materialize( + source=source, + execution_binding_artifacts=capture_bindings, + runtime_authentication_context=runtime_context, + ) + finally: + _verify_capture_binding(capture_binding) + sequences = getattr(materialization, "sequences", None) + if not isinstance(sequences, tuple) or len(sequences) != 160: + raise Experiment013AdapterError( + "canonical capture did not return exactly 160 calibration sequences" + ) + tokenizer_hash = _require_sha256( + getattr(materialization, "tokenizer_manifest_sha256", None), + context="materialization tokenizer manifest", + ) + prepared: dict[str, AuthenticatedSequence] = {} + for sequence in sequences: + record = getattr(sequence, "identity_record", None) + token_ids = getattr(sequence, "sequence_token_ids", None) + digest = _require_sha256( + getattr(sequence, "identity_record_sha256", None), + context="materialized identity record", + ) + if ( + not isinstance(record, Mapping) + or record.get("identity_record_sha256") != digest + ): + raise Experiment013AdapterError("materialized identity record digest drifted") + if not isinstance(token_ids, tuple): + raise Experiment013AdapterError( + "materialized sequence token IDs are not a tuple" + ) + if digest in prepared: + raise Experiment013AdapterError("materialization returned a duplicate identity") + sequence_tokenizer_hash = _require_sha256( + record.get("tokenizer_manifest_sha256"), + context="materialized sequence tokenizer manifest", + ) + if sequence_tokenizer_hash != tokenizer_hash: + raise Experiment013AdapterError("materialized tokenizer commitments differ") + prepared[digest] = AuthenticatedSequence( + token_ids=tuple(token_ids), + source_content_sha256=_require_sha256( + record.get("source_content_sha256"), + context="materialized source content", + ), + formatted_content_sha256=_require_sha256( + record.get("formatted_content_sha256"), + context="materialized formatted content", + ), + generator_receipt_sha256=( + None + if record.get("generator_receipt_sha256") is None + else _require_sha256( + record.get("generator_receipt_sha256"), + context="materialized generator receipt", + ) + ), + tokenizer_manifest_sha256=sequence_tokenizer_hash, + ) + self._capture_input_sha256 = _require_sha256( + getattr(materialization, "capture_input_sha256", None), + context="materialization capture input", + ) + self._token_sequence_manifest_sha256 = _require_sha256( + getattr(materialization, "token_sequence_manifest_sha256", None), + context="materialization token sequence manifest", + ) + self._materialized_sequences = prepared + finally: + # These potentially large artifact bytes are needed only for the + # canonical capture call. Do not retain capture/source objects. + self._execution_binding_artifacts = None + self._runtime_authentication_context = None + + def materialize_sequence(self, record: Mapping[str, object]) -> AuthenticatedSequence: + if not isinstance(record, Mapping): + raise TypeError("record must be a mapping") + self._prepare_materialization() + digest = _require_sha256(record.get("identity_record_sha256"), context="identity record") + assert self._materialized_sequences is not None + try: + result = self._materialized_sequences[digest] + except KeyError as error: + raise Experiment013AdapterError( + f"canonical materialization has no identity record {digest}" + ) from error + exact = { + "source_content_sha256": result.source_content_sha256, + "formatted_content_sha256": result.formatted_content_sha256, + "generator_receipt_sha256": result.generator_receipt_sha256, + "tokenizer_manifest_sha256": result.tokenizer_manifest_sha256, + "sequence_length": len(result.token_ids), + } + for name, expected in exact.items(): + if record.get(name) != expected: + raise Experiment013AdapterError( + f"frozen identity {name} differs from canonical materialization" + ) + return result + + def _validate_authenticated_model(self, authenticated: AuthenticatedModelFiles) -> Path: + if not isinstance(authenticated, AuthenticatedModelFiles): + raise TypeError("authenticated must be AuthenticatedModelFiles") + if ( + authenticated.model_id != MODEL_ID + or authenticated.revision != MODEL_REVISION + or authenticated.transformers_version != TRANSFORMERS_VERSION + ): + raise Experiment013AdapterError( + "authenticated model identity differs from the frozen model" + ) + expected_root = self._model_root.resolve(strict=True) + actual_root = authenticated.model_root.resolve(strict=True) + if actual_root != expected_root or not actual_root.is_dir(): + raise Experiment013AdapterError("authenticated model root differs from adapter context") + names = tuple(item.name for item in authenticated.files) + if ( + names != tuple(sorted(names)) + or len(names) != len(set(names)) + or "config.json" not in names + or "model.safetensors.index.json" not in names + or not any(_WEIGHT_FILE_RE.fullmatch(name) for name in names) + or any( + name not in {"config.json", "model.safetensors.index.json"} + and _WEIGHT_FILE_RE.fullmatch(name) is None + for name in names + ) + ): + raise Experiment013AdapterError( + "authenticated model file inventory is not the pinned profile" + ) + return actual_root + + def load_model(self, authenticated: AuthenticatedModelFiles) -> object: + if self._model is not None: + raise Experiment013AdapterError("the live model is already loaded") + model_root = self._validate_authenticated_model(authenticated) + runtime = _load_transformers_runtime() + if runtime.version != TRANSFORMERS_VERSION: + raise Experiment013AdapterError("Transformers runtime differs from 5.14.1") + if not torch.cuda.is_available(): + raise Experiment013AdapterError("official Experiment 013 model loading requires CUDA") + device = torch.device("cuda", torch.cuda.current_device()) + config = runtime.qwen_config_class.from_pretrained( + str(model_root), + local_files_only=True, + trust_remote_code=False, + ) + _validate_qwen_config(config) + loaded = runtime.qwen_model_class.from_pretrained( + str(model_root), + config=config, + dtype=torch.bfloat16, + attn_implementation="eager", + low_cpu_mem_usage=True, + use_safetensors=True, + weights_only=True, + local_files_only=True, + trust_remote_code=False, + output_loading_info=True, + ) + if type(loaded) is not tuple or len(loaded) != 2: + raise Experiment013AdapterError( + "Transformers did not return exactly (model, loading_info)" + ) + model, loading_info = loaded + if type(loading_info) is not dict or set(loading_info) != set(_LOADING_DIAGNOSTIC_TYPES): + raise Experiment013AdapterError("Transformers loading diagnostics schema drifted") + diagnostic_counts: dict[str, int] = {} + for name, expected_type in _LOADING_DIAGNOSTIC_TYPES.items(): + diagnostic = loading_info[name] + if type(diagnostic) is not expected_type: + raise Experiment013AdapterError(f"Transformers {name} diagnostic type drifted") + diagnostic_counts[name] = len(diagnostic) + if diagnostic: + raise Experiment013AdapterError( + f"Transformers reported non-empty {name} while loading authenticated weights" + ) + self._model_loading_diagnostic_counts = diagnostic_counts + model = model.to(device) + model.eval() + model.requires_grad_(False) + _validate_qwen_config(getattr(model, "config", None)) + if getattr(model.config, "_attn_implementation", None) != "eager": + raise Experiment013AdapterError("loaded Qwen3.5 model is not using eager attention") + if getattr(model, "training", True): + raise Experiment013AdapterError("loaded Qwen3.5 model remained in training mode") + devices = _model_devices(model) + parameters = tuple(model.parameters()) + if not parameters: + raise Experiment013AdapterError("loaded Qwen3.5 model exposes no parameters") + if ( + devices != {device} + or any(parameter.requires_grad for parameter in parameters) + or any( + parameter.is_floating_point() and parameter.dtype != torch.bfloat16 + for parameter in parameters + ) + ): + raise Experiment013AdapterError( + "loaded Qwen3.5 weights are not frozen BF16 on exactly one CUDA device" + ) + modules = _qwen_modules(model, runtime) + _freeze_torch_fallbacks(modules, runtime) + observer = _Qwen35StepObserver(modules, query_device=device) + observer.install() + self._runtime = runtime + self._model = model + self._model_device = device + self._observer = observer + return model + + def _require_loaded_model(self, model: object) -> None: + if model is not self._model or self._runtime is None or self._observer is None: + raise Experiment013AdapterError("model is not the adapter's authenticated live model") + + def begin_sequence(self, model: object, record: Mapping[str, object]) -> None: + self._require_loaded_model(model) + if self._sequence is not None: + raise Experiment013AdapterError("a calibration sequence is already active") + if not isinstance(record, Mapping): + raise TypeError("record must be a mapping") + digest = _require_sha256(record.get("identity_record_sha256"), context="identity record") + token_count = _require_non_negative_int( + record.get("sequence_length"), context="sequence length" + ) + if token_count == 0: + raise Experiment013AdapterError("calibration sequence cannot be empty") + assert self._runtime is not None + cache = self._runtime.dynamic_cache_class(config=model.config) + self._sequence = _SequenceState( + cache=cache, + identity_record_sha256=digest, + token_count=token_count, + ) + self._sequence_failed = False + + @staticmethod + def _cache_length(cache: object) -> int: + get_seq_length = getattr(cache, "get_seq_length", None) + if not callable(get_seq_length): + raise Experiment013AdapterError("DynamicCache does not expose get_seq_length") + length = get_seq_length() + if isinstance(length, bool) or not isinstance(length, int) or length < 0: + raise Experiment013AdapterError("DynamicCache returned an invalid sequence length") + return length + + def step_token( + self, + model: object, + *, + token_id: int, + position: int, + capture_state: bool, + ) -> StepObservation: + self._require_loaded_model(model) + sequence = self._sequence + observer = self._observer + device = self._model_device + if sequence is None or observer is None or device is None or self._sequence_failed: + raise Experiment013AdapterError("no healthy calibration sequence is active") + token = _require_non_negative_int(token_id, context="token_id") + current = _require_non_negative_int(position, context="position") + if not isinstance(capture_state, bool): + raise TypeError("capture_state must be bool") + if current != sequence.next_position or current >= sequence.token_count: + raise Experiment013AdapterError( + "adapter token position is not the next causal position" + ) + if getattr(model, "training", True): + raise Experiment013AdapterError( + "Qwen3.5 model entered training mode during calibration" + ) + if self._cache_length(sequence.cache) != current: + raise Experiment013AdapterError("DynamicCache length drifted before one-token forward") + + capture = _StepCapture(cache=sequence.cache, position=current, receipts={}) + context_token = observer.activate(capture) + try: + input_ids = torch.tensor([[token]], dtype=torch.long, device=device) + position_ids = torch.tensor([[current]], dtype=torch.long, device=device) + with torch.inference_mode(): + output = model.model( + input_ids=input_ids, + position_ids=position_ids, + past_key_values=sequence.cache, + use_cache=True, + ) + if getattr(output, "past_key_values", None) is not sequence.cache: + raise Experiment013AdapterError("Qwen3.5 returned a different DynamicCache object") + if self._cache_length(sequence.cache) != current + 1: + raise Experiment013AdapterError("DynamicCache did not advance by exactly one token") + if tuple(capture.receipts) != RECURRENT_LAYER_INDICES: + raise Experiment013AdapterError( + "one-token forward did not produce one ordered receipt for every " + "recurrent layer" + ) + queries: list[torch.Tensor] = [] + states: list[torch.Tensor] = [] + cache_matches: list[torch.Tensor] = [] + for layer_index in RECURRENT_LAYER_INDICES: + receipt = capture.receipts[layer_index] + cached = observer._cache_state(sequence.cache, layer_index) + if ( + cached is None + or tuple(cached.shape) != STATE_SHAPE + or cached.dtype != torch.float32 + or cached.device != receipt.final_state.device + ): + raise Experiment013AdapterError( + f"DynamicCache state geometry differs at layer {layer_index}" + ) + # Queue every exact comparison, then transfer only 18 scalar + # receipts. Calling torch.equal here would synchronize CUDA + # separately for every layer and every calibration token. + cache_matches.append(torch.eq(cached, receipt.final_state).all()) + queries.append(receipt.query[0, 0]) + if capture_state: + states.append(cached[0]) + match_values = torch.stack(cache_matches).detach().to(device="cpu").tolist() + mismatched = [ + layer_index + for layer_index, matches in zip(RECURRENT_LAYER_INDICES, match_values, strict=True) + if not matches + ] + if mismatched: + raise Experiment013AdapterError( + f"DynamicCache state differs from kernel output at layers {mismatched}" + ) + recurrence_query = torch.stack(queries, dim=0).contiguous() + recurrent_state = torch.stack(states, dim=0).contiguous() if capture_state else None + sequence.next_position += 1 + return StepObservation( + position=current, + token_id=token, + layer_indices=RECURRENT_LAYER_INDICES, + recurrence_query=recurrence_query, + recurrent_state=recurrent_state, + successful_kernel_calls_per_layer=(1,) * len(RECURRENT_LAYER_INDICES), + ) + except BaseException: + self._sequence_failed = True + raise + finally: + observer.deactivate(context_token) + + def end_sequence(self, model: object, record: Mapping[str, object]) -> None: + self._require_loaded_model(model) + del record + # Cleanup must not mask a runner-side failure with a second exception. + self._sequence = None + self._sequence_failed = False + + def close_model(self, model: object) -> None: + if self._model is None: + return + if model is not self._model: + raise Experiment013AdapterError("refusing to close a different model object") + self._sequence = None + self._sequence_failed = False + if self._observer is not None: + self._observer.remove() + self._observer = None + self._runtime = None + self._model_device = None + self._model = None + + def runtime_metadata(self) -> Mapping[str, object]: + return { + "adapter_revision": ADAPTER_REVISION, + "capture_input_sha256": self._capture_input_sha256, + "device": None if self._model_device is None else str(self._model_device), + "kernel_backend": "transformers_pure_torch_gated_delta_rule", + "materialization_attempted": self._materialization_attempted, + "materialized_sequence_count": ( + 0 if self._materialized_sequences is None else len(self._materialized_sequences) + ), + "model_dtype": MODEL_DTYPE_NAME, + "model_id": MODEL_ID, + "model_loaded": self._model is not None, + "model_loading_diagnostic_counts": ( + None + if self._model_loading_diagnostic_counts is None + else dict(self._model_loading_diagnostic_counts) + ), + "model_revision": MODEL_REVISION, + "query_shape": list(QUERY_SHAPE), + "recurrent_layer_indices": list(RECURRENT_LAYER_INDICES), + "state_shape": list(STATE_SHAPE), + "token_sequence_manifest_sha256": self._token_sequence_manifest_sha256, + "transformers_version": TRANSFORMERS_VERSION, + } + + @staticmethod + def source_binding() -> Mapping[str, object]: + """Return the adapter's own source bytes identity for independent checks.""" + + payload = Path(__file__).read_bytes() + return { + "path": ADAPTER_SOURCE_PATH, + "raw_sha256": hashlib.sha256(payload).hexdigest(), + } + + +def create_adapter(context: AdapterConstructionContext) -> Experiment013Qwen35Adapter: + """Construct the canonical adapter without performing I/O or importing data/model code.""" + + return Experiment013Qwen35Adapter(context) + + +__all__ = [ + "ADAPTER_REVISION", + "Experiment013AdapterError", + "Experiment013Qwen35Adapter", + "create_adapter", +] diff --git a/src/recurquant/experiment013_source.py b/src/recurquant/experiment013_source.py new file mode 100644 index 0000000..17f5867 --- /dev/null +++ b/src/recurquant/experiment013_source.py @@ -0,0 +1,848 @@ +"""Fail-closed local-source identity for Experiment 013. + +The manifest produced here is portable: it contains only repository-relative +paths and Git/content identities. Absolute worktree, Git-directory, index, +and object-store paths are authenticated locally but never serialized. + +The verifier and its tests are part of the frozen inventory. This does not +create a hash cycle: their committed bytes do not embed the resulting manifest +or its canonical hash. No expected source-manifest digest is hard-coded here. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import subprocess +import sys +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path, PurePosixPath + +EXPERIMENT013_SOURCE_MANIFEST_SCHEMA = "recurquant.experiment013.source-manifest.v1" +EXPERIMENT013_SOURCE_MANIFEST_PROFILE = "experiment-013-static-q468-frozen-source-v1" +EXPERIMENT013_REPOSITORY_BINDING_SCHEMA = "recurquant.experiment013.repository-binding.v1" + +# Keep this explicit. Discovery by glob would silently change the experiment +# when an unrelated file was added or removed. +EXPERIMENT013_SOURCE_PATHS: tuple[str, ...] = tuple( + sorted( + { + "pyproject.toml", + "requirements/experiment013-calibration.txt", + "requirements/experiment013-ruler.txt", + "research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md", + "research/experiment013-parquet-materializations.json", + "scripts/capture_static_q468_identity_input.py", + "scripts/generate_static_q468_ruler_receipts.py", + "scripts/launch_static_q468_calibration.py", + "scripts/resolve_static_q468_identity.py", + "scripts/run_static_q468_calibration.py", + "src/recurquant/evidence.py", + "src/recurquant/experiment013_calibration_api.py", + "src/recurquant/experiment013_parquet.py", + "src/recurquant/experiment013_qwen35_adapter.py", + "src/recurquant/experiment013_source.py", + "src/recurquant/metrics.py", + "src/recurquant/mixed_quantization.py", + "src/recurquant/multibit_policy.py", + "src/recurquant/multibit_quantization.py", + "src/recurquant/packed_cache.py", + "src/recurquant/quantization.py", + "src/recurquant/qwen35.py", + "src/recurquant/rht.py", + "src/recurquant/row_policy.py", + "src/recurquant/statelease.py", + "src/recurquant/statelease_baselines.py", + "src/recurquant/statelease_cache.py", + "src/recurquant/statelease_equal_byte_baselines.py", + "src/recurquant/statelease_equal_byte_cache.py", + "src/recurquant/statelease_observer.py", + "src/recurquant/static_q468.py", + "src/recurquant/static_q468_cache.py", + "src/recurquant/static_q468_calibration.py", + "tests/test_capture_static_q468_identity_input.py", + "tests/test_experiment013_calibration_api.py", + "tests/test_experiment013_parquet.py", + "tests/test_experiment013_qwen35_adapter.py", + "tests/test_experiment013_source.py", + "tests/test_generate_static_q468_ruler_receipts.py", + "tests/test_launch_static_q468_calibration.py", + "tests/test_resolve_static_q468_identity.py", + "tests/test_run_static_q468_calibration.py", + "tests/test_static_q468.py", + "tests/test_static_q468_cache.py", + "tests/test_static_q468_calibration.py", + } + ) +) + +_TOP_LEVEL_FIELDS = frozenset( + { + "schema", + "profile", + "object_format", + "source_commit", + "repository_binding", + "paths", + "canonical_manifest_sha256", + } +) +_PATH_FIELDS = frozenset( + { + "path", + "mode", + "git_blob_oid", + "index_blob_oid", + "worktree_blob_oid", + "raw_sha256", + } +) +_BINDING_FIELDS = frozenset( + { + "schema", + "worktree_layout", + "top_level_matches_requested_root", + "git_directory_bound", + "common_object_store_bound", + "index_bound_to_worktree_git_directory", + "linked_worktree_pointers_verified", + "hidden_index_flags_absent", + "tracked_staged_changes_absent", + "tracked_unstaged_changes_absent", + "untracked_nonignored_paths_absent", + "ignored_artifacts_permitted", + "unsafe_local_config_absent", + "shallow_history_absent", + "alternate_object_stores_absent", + "replacement_refs_absent", + "history_grafts_absent", + "inherited_git_environment_scrubbed", + "system_and_global_git_config_disabled", + } +) +_TRUE_BINDING_FIELDS = _BINDING_FIELDS - {"schema", "worktree_layout"} +_SHA1_RE = re.compile(r"[0-9a-f]{40}") +_SHA256_RE = re.compile(r"[0-9a-f]{64}") +_SAFE_GIT_MODES = frozenset({"100644", "100755"}) +_FORBIDDEN_LOCAL_CONFIG_KEYS = frozenset( + { + "core.alternaterefscommand", + "core.alternaterefsprefixes", + "core.attributesfile", + "core.checkstat", + "core.excludesfile", + "core.fsmonitor", + "core.hookspath", + "core.ignorestat", + "core.sparsecheckout", + "core.sparsecheckoutcone", + "core.splitindex", + "core.trustctime", + "core.untrackedcache", + "core.worktree", + "extensions.partialclone", + "extensions.worktreeconfig", + "index.sparse", + } +) + + +class Experiment013SourceError(RuntimeError): + """Raised when the Experiment 013 source identity cannot be authenticated.""" + + +@dataclass(frozen=True) +class _RepositoryIdentity: + root: Path + git_dir: Path + common_dir: Path + index_path: Path + object_dir: Path + public_binding: dict[str, object] + + +def _canonical_json_bytes(value: object) -> bytes: + try: + return (json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n").encode("utf-8") + except (TypeError, ValueError) as error: + raise Experiment013SourceError("source manifest is not canonical JSON data") from error + + +def canonical_experiment013_source_manifest_sha256( + payload_without_hash: Mapping[str, object], +) -> str: + """Hash a canonical manifest payload that does not contain its own hash.""" + + if "canonical_manifest_sha256" in payload_without_hash: + raise Experiment013SourceError( + "canonical payload must not contain canonical_manifest_sha256" + ) + return hashlib.sha256(_canonical_json_bytes(dict(payload_without_hash))).hexdigest() + + +def _exact_fields(value: Mapping[str, object], expected: frozenset[str], *, name: str) -> None: + if any(not isinstance(field, str) for field in value): + raise Experiment013SourceError(f"{name} field names must be strings") + actual = frozenset(value) + if actual != expected: + missing = sorted(expected - actual) + extra = sorted(actual - expected) + raise Experiment013SourceError(f"{name} fields drifted (missing={missing}, extra={extra})") + + +def _sha1(value: object, *, name: str) -> str: + if not isinstance(value, str) or _SHA1_RE.fullmatch(value) is None: + raise Experiment013SourceError(f"{name} must be a lowercase SHA-1 object ID") + return value + + +def _sha256(value: object, *, name: str) -> str: + if not isinstance(value, str) or _SHA256_RE.fullmatch(value) is None: + raise Experiment013SourceError(f"{name} must be a lowercase SHA-256 digest") + return value + + +def _canonical_relative_path(value: object, *, name: str) -> str: + if not isinstance(value, str) or not value or value != value.strip(): + raise Experiment013SourceError(f"{name} must be a non-empty canonical path") + if "\\" in value or "\0" in value or "\n" in value or "\r" in value: + raise Experiment013SourceError(f"{name} must use a single-line POSIX path") + path = PurePosixPath(value) + if ( + path.is_absolute() + or path.parts[0].endswith(":") + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise Experiment013SourceError(f"{name} must be repository-relative") + if path.as_posix() != value: + raise Experiment013SourceError(f"{name} is not a canonical POSIX path") + return value + + +def validate_experiment013_source_manifest( + manifest: Mapping[str, object], +) -> dict[str, object]: + """Strictly normalize a source manifest without consulting Git or the filesystem.""" + + if not isinstance(manifest, Mapping): + raise Experiment013SourceError("source manifest must be a mapping") + _exact_fields(manifest, _TOP_LEVEL_FIELDS, name="source manifest") + if manifest["schema"] != EXPERIMENT013_SOURCE_MANIFEST_SCHEMA: + raise Experiment013SourceError("source manifest schema drifted") + if manifest["profile"] != EXPERIMENT013_SOURCE_MANIFEST_PROFILE: + raise Experiment013SourceError("source manifest profile drifted") + if manifest["object_format"] != "sha1": + raise Experiment013SourceError("source manifest requires the frozen SHA-1 format") + source_commit = _sha1(manifest["source_commit"], name="source_commit") + + raw_binding = manifest["repository_binding"] + if not isinstance(raw_binding, Mapping): + raise Experiment013SourceError("repository_binding must be a mapping") + _exact_fields(raw_binding, _BINDING_FIELDS, name="repository_binding") + if raw_binding["schema"] != EXPERIMENT013_REPOSITORY_BINDING_SCHEMA: + raise Experiment013SourceError("repository binding schema drifted") + if raw_binding["worktree_layout"] not in {"primary", "linked"}: + raise Experiment013SourceError("repository worktree_layout is invalid") + for field in _TRUE_BINDING_FIELDS: + if raw_binding[field] is not True: + raise Experiment013SourceError(f"repository binding check is not true: {field}") + binding = {field: raw_binding[field] for field in sorted(_BINDING_FIELDS)} + + raw_paths = manifest["paths"] + if isinstance(raw_paths, (str, bytes, bytearray)) or not isinstance(raw_paths, Sequence): + raise Experiment013SourceError("source manifest paths must be a sequence") + if len(raw_paths) != len(EXPERIMENT013_SOURCE_PATHS): + raise Experiment013SourceError("source manifest path inventory size drifted") + entries: list[dict[str, str]] = [] + for index, raw_entry in enumerate(raw_paths): + if not isinstance(raw_entry, Mapping): + raise Experiment013SourceError(f"paths[{index}] must be a mapping") + _exact_fields(raw_entry, _PATH_FIELDS, name=f"paths[{index}]") + path = _canonical_relative_path(raw_entry["path"], name=f"paths[{index}].path") + if path != EXPERIMENT013_SOURCE_PATHS[index]: + raise Experiment013SourceError( + f"source path inventory drifted at index {index}: {path}" + ) + mode = raw_entry["mode"] + if not isinstance(mode, str) or mode not in _SAFE_GIT_MODES: + raise Experiment013SourceError(f"paths[{index}].mode is not a regular-file mode") + git_blob_oid = _sha1(raw_entry["git_blob_oid"], name=f"paths[{index}].git_blob_oid") + index_blob_oid = _sha1(raw_entry["index_blob_oid"], name=f"paths[{index}].index_blob_oid") + worktree_blob_oid = _sha1( + raw_entry["worktree_blob_oid"], name=f"paths[{index}].worktree_blob_oid" + ) + if len({git_blob_oid, index_blob_oid, worktree_blob_oid}) != 1: + raise Experiment013SourceError( + f"paths[{index}] Git/index/worktree blob identities differ" + ) + entries.append( + { + "path": path, + "mode": mode, + "git_blob_oid": git_blob_oid, + "index_blob_oid": index_blob_oid, + "worktree_blob_oid": worktree_blob_oid, + "raw_sha256": _sha256(raw_entry["raw_sha256"], name=f"paths[{index}].raw_sha256"), + } + ) + + normalized_without_hash: dict[str, object] = { + "schema": EXPERIMENT013_SOURCE_MANIFEST_SCHEMA, + "profile": EXPERIMENT013_SOURCE_MANIFEST_PROFILE, + "object_format": "sha1", + "source_commit": source_commit, + "repository_binding": binding, + "paths": entries, + } + recorded_hash = _sha256(manifest["canonical_manifest_sha256"], name="canonical_manifest_sha256") + computed_hash = canonical_experiment013_source_manifest_sha256(normalized_without_hash) + if recorded_hash != computed_hash: + raise Experiment013SourceError("canonical source manifest SHA-256 drifted") + return {**normalized_without_hash, "canonical_manifest_sha256": recorded_hash} + + +def _sanitized_git_environment() -> dict[str, str]: + environment = { + key: value for key, value in os.environ.items() if not key.upper().startswith("GIT_") + } + environment.update( + { + "GIT_NO_REPLACE_OBJECTS": "1", + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_COUNT": "3", + "GIT_CONFIG_KEY_0": "core.fsmonitor", + "GIT_CONFIG_VALUE_0": "false", + "GIT_CONFIG_KEY_1": "core.untrackedCache", + "GIT_CONFIG_VALUE_1": "false", + "GIT_CONFIG_KEY_2": "core.hooksPath", + "GIT_CONFIG_VALUE_2": os.devnull, + "LC_ALL": "C", + "LANG": "C", + } + ) + return environment + + +def _run_git( + root: Path, + *arguments: str, + input_bytes: bytes | None = None, +) -> subprocess.CompletedProcess[bytes]: + try: + return subprocess.run( + ["git", *arguments], + cwd=root, + check=False, + capture_output=True, + input=input_bytes, + env=_sanitized_git_environment(), + timeout=30, + ) + except (OSError, subprocess.TimeoutExpired) as error: + raise Experiment013SourceError(f"git {arguments[0]} could not be executed") from error + + +def _git_bytes( + root: Path, + *arguments: str, + input_bytes: bytes | None = None, +) -> bytes: + process = _run_git(root, *arguments, input_bytes=input_bytes) + if process.returncode != 0: + detail = process.stderr.decode("utf-8", errors="replace").strip() + raise Experiment013SourceError( + f"git {' '.join(arguments)} failed" + (f": {detail}" if detail else "") + ) + return process.stdout + + +def _git_text(root: Path, *arguments: str) -> str: + try: + return _git_bytes(root, *arguments).decode("utf-8").strip() + except UnicodeDecodeError as error: + raise Experiment013SourceError("Git returned non-UTF-8 identity data") from error + + +def _resolved_git_path(root: Path, value: str, *, must_exist: bool = True) -> Path: + path = Path(value) + if not path.is_absolute(): + path = root / path + try: + return path.resolve(strict=must_exist) + except OSError as error: + raise Experiment013SourceError("authenticated Git path is unavailable") from error + + +def _path_has_symlink_component(root: Path, relative: str) -> bool: + current = root + for part in PurePosixPath(relative).parts: + current = current / part + if current.is_symlink(): + return True + return False + + +def _assert_safe_local_config(root: Path) -> None: + raw = _git_bytes(root, "config", "--local", "--no-includes", "--null", "--list") + for record in (item for item in raw.split(b"\0") if item): + key_bytes, separator, value_bytes = record.partition(b"\n") + if not separator or not key_bytes: + raise Experiment013SourceError("local Git config contains a malformed entry") + try: + key = key_bytes.decode("utf-8").lower() + value = value_bytes.decode("utf-8") + except UnicodeDecodeError as error: + raise Experiment013SourceError("local Git config is not UTF-8") from error + forbidden = ( + key in _FORBIDDEN_LOCAL_CONFIG_KEYS + or key.startswith(("include.", "includeif.", "filter.")) + or (key.startswith("remote.") and key.endswith((".promisor", ".partialclonefilter"))) + ) + if forbidden: + raise Experiment013SourceError(f"unsafe local Git config key is present: {key}") + if key == "core.usereplacerefs" and value.lower() not in {"0", "false", "no", "off"}: + raise Experiment013SourceError("local Git config enables replacement objects") + + +def _assert_no_hidden_index_flags(root: Path) -> None: + raw = _git_bytes(root, "ls-files", "--cached", "-v", "-z") + records = [record for record in raw.split(b"\0") if record] + malformed = [record for record in records if len(record) < 3 or record[1:2] != b" "] + if malformed: + raise Experiment013SourceError("Git index visibility output is malformed") + unsafe = sorted({chr(record[0]) for record in records if record[:1] != b"H"}) + if unsafe: + raise Experiment013SourceError( + "Git index contains skip-worktree, assume-unchanged, or non-stage-zero entries " + f"(tags={unsafe})" + ) + + +def _assert_no_tracked_or_untracked_changes(root: Path) -> None: + checks = ( + ("tracked staged", ("diff", "--cached", "--no-ext-diff", "--quiet", "HEAD", "--")), + ("tracked unstaged", ("diff", "--no-ext-diff", "--quiet", "--")), + ) + for label, arguments in checks: + process = _run_git(root, *arguments) + if process.returncode == 1: + raise Experiment013SourceError(f"repository has {label} changes") + if process.returncode != 0: + raise Experiment013SourceError(f"cannot authenticate {label} changes") + status = _git_bytes(root, "status", "--porcelain=v1", "-z", "--untracked-files=all") + if status: + raise Experiment013SourceError( + "repository contains non-ignored untracked paths or tracked status drift" + ) + + +def _authenticate_repository(repo_root: str | Path) -> _RepositoryIdentity: + try: + root = Path(repo_root).resolve(strict=True) + except OSError as error: + raise Experiment013SourceError("repository root is unavailable") from error + if not root.is_dir(): + raise Experiment013SourceError("repository root must be a directory") + + top_level = _resolved_git_path(root, _git_text(root, "rev-parse", "--show-toplevel")) + git_dir = _resolved_git_path(root, _git_text(root, "rev-parse", "--absolute-git-dir")) + common_dir = _resolved_git_path(root, _git_text(root, "rev-parse", "--git-common-dir")) + object_dir = _resolved_git_path(root, _git_text(root, "rev-parse", "--git-path", "objects")) + index_path = _resolved_git_path(root, _git_text(root, "rev-parse", "--git-path", "index")) + if top_level != root: + raise Experiment013SourceError("Git top-level does not equal the requested repository root") + if _git_text(root, "rev-parse", "--is-inside-work-tree") != "true": + raise Experiment013SourceError("repository root is not inside a Git worktree") + if _git_text(root, "rev-parse", "--is-bare-repository") != "false": + raise Experiment013SourceError("bare Git repositories are forbidden") + if _git_text(root, "rev-parse", "--show-object-format") != "sha1": + raise Experiment013SourceError("Experiment 013 requires the SHA-1 Git object format") + if _git_text(root, "rev-parse", "--is-shallow-repository") != "false": + raise Experiment013SourceError("shallow Git history is forbidden") + if not git_dir.is_dir() or not common_dir.is_dir() or not index_path.is_file(): + raise Experiment013SourceError("Git directory or index identity is malformed") + if index_path != (git_dir / "index").resolve(strict=True): + raise Experiment013SourceError("Git index is not bound to the worktree Git directory") + if object_dir != (common_dir / "objects").resolve(strict=True) or not object_dir.is_dir(): + raise Experiment013SourceError("Git object directory is not the common object store") + + dot_git = root / ".git" + if dot_git.is_symlink(): + raise Experiment013SourceError("repository .git identity cannot be a symlink") + if dot_git.is_dir(): + if git_dir != dot_git.resolve(strict=True) or common_dir != git_dir: + raise Experiment013SourceError("primary-worktree Git directory binding drifted") + layout = "primary" + elif dot_git.is_file(): + try: + marker = dot_git.read_text(encoding="utf-8").strip() + except (OSError, UnicodeError) as error: + raise Experiment013SourceError("linked-worktree .git pointer is unreadable") from error + if not marker.startswith("gitdir: "): + raise Experiment013SourceError("linked-worktree .git pointer is malformed") + marker_path = Path(marker.removeprefix("gitdir: ")) + if not marker_path.is_absolute(): + marker_path = root / marker_path + try: + marker_git_dir = marker_path.resolve(strict=True) + except OSError as error: + raise Experiment013SourceError( + "linked-worktree Git directory is unavailable" + ) from error + if ( + marker_git_dir != git_dir + or git_dir.parent.name != "worktrees" + or git_dir.parent.parent != common_dir + ): + raise Experiment013SourceError("linked-worktree/common-directory binding drifted") + reverse_pointer = git_dir / "gitdir" + if not reverse_pointer.is_file() or reverse_pointer.is_symlink(): + raise Experiment013SourceError("linked-worktree reverse pointer is unavailable") + try: + reverse_path = Path(reverse_pointer.read_text(encoding="utf-8").strip()) + if not reverse_path.is_absolute(): + reverse_path = git_dir / reverse_path + reverse_path = reverse_path.resolve(strict=True) + except (OSError, UnicodeError) as error: + raise Experiment013SourceError( + "linked-worktree reverse pointer is malformed" + ) from error + if reverse_path != dot_git.resolve(strict=True): + raise Experiment013SourceError("linked-worktree reverse pointer drifted") + layout = "linked" + else: + raise Experiment013SourceError("repository has no canonical .git identity") + + for unsafe_path in ( + object_dir / "info" / "alternates", + object_dir / "info" / "http-alternates", + common_dir / "info" / "grafts", + git_dir / "info" / "grafts", + common_dir / "shallow", + ): + if unsafe_path.exists(): + raise Experiment013SourceError("Git alternates, grafts, or shallow metadata exists") + if _git_text(root, "for-each-ref", "--format=%(refname)", "refs/replace"): + raise Experiment013SourceError("Git replacement refs are forbidden") + _assert_safe_local_config(root) + _assert_no_hidden_index_flags(root) + _assert_no_tracked_or_untracked_changes(root) + + binding: dict[str, object] = { + "schema": EXPERIMENT013_REPOSITORY_BINDING_SCHEMA, + "worktree_layout": layout, + "top_level_matches_requested_root": True, + "git_directory_bound": True, + "common_object_store_bound": True, + "index_bound_to_worktree_git_directory": True, + "linked_worktree_pointers_verified": True, + "hidden_index_flags_absent": True, + "tracked_staged_changes_absent": True, + "tracked_unstaged_changes_absent": True, + "untracked_nonignored_paths_absent": True, + "ignored_artifacts_permitted": True, + "unsafe_local_config_absent": True, + "shallow_history_absent": True, + "alternate_object_stores_absent": True, + "replacement_refs_absent": True, + "history_grafts_absent": True, + "inherited_git_environment_scrubbed": True, + "system_and_global_git_config_disabled": True, + } + return _RepositoryIdentity(root, git_dir, common_dir, index_path, object_dir, binding) + + +def _tree_entries(root: Path, commit: str) -> dict[str, tuple[str, str]]: + raw = _git_bytes( + root, + "ls-tree", + "-r", + "--full-tree", + "-z", + commit, + "--", + *EXPERIMENT013_SOURCE_PATHS, + ) + entries: dict[str, tuple[str, str]] = {} + for record in (item for item in raw.split(b"\0") if item): + metadata, separator, path_bytes = record.partition(b"\t") + fields = metadata.split(b" ") + try: + path = path_bytes.decode("utf-8") + mode = fields[0].decode("ascii") + kind = fields[1].decode("ascii") + oid = fields[2].decode("ascii") + except (IndexError, UnicodeDecodeError) as error: + raise Experiment013SourceError("Git tree entry is malformed") from error + if separator != b"\t" or len(fields) != 3 or kind != "blob" or mode not in _SAFE_GIT_MODES: + raise Experiment013SourceError("source inventory contains a non-regular Git tree entry") + if path not in EXPERIMENT013_SOURCE_PATHS or path in entries: + raise Experiment013SourceError("Git tree returned an unexpected source path") + entries[path] = (mode, _sha1(oid, name=f"Git tree OID for {path}")) + if set(entries) != set(EXPERIMENT013_SOURCE_PATHS): + missing = sorted(set(EXPERIMENT013_SOURCE_PATHS) - set(entries)) + raise Experiment013SourceError(f"source inventory is not tracked by the commit: {missing}") + return entries + + +def _index_entries(root: Path) -> dict[str, tuple[str, str]]: + raw = _git_bytes(root, "ls-files", "--stage", "-z", "--", *EXPERIMENT013_SOURCE_PATHS) + entries: dict[str, tuple[str, str]] = {} + for record in (item for item in raw.split(b"\0") if item): + metadata, separator, path_bytes = record.partition(b"\t") + fields = metadata.split(b" ") + try: + path = path_bytes.decode("utf-8") + mode = fields[0].decode("ascii") + oid = fields[1].decode("ascii") + stage = fields[2].decode("ascii") + except (IndexError, UnicodeDecodeError) as error: + raise Experiment013SourceError("Git index entry is malformed") from error + if ( + separator != b"\t" + or len(fields) != 3 + or stage != "0" + or mode not in _SAFE_GIT_MODES + or path not in EXPERIMENT013_SOURCE_PATHS + or path in entries + ): + raise Experiment013SourceError("source inventory has a non-canonical index entry") + entries[path] = (mode, _sha1(oid, name=f"Git index OID for {path}")) + if set(entries) != set(EXPERIMENT013_SOURCE_PATHS): + missing = sorted(set(EXPERIMENT013_SOURCE_PATHS) - set(entries)) + raise Experiment013SourceError(f"source inventory is not in the Git index: {missing}") + return entries + + +def _worktree_oids(root: Path) -> dict[str, str]: + payload = "".join(f"{path}\n" for path in EXPERIMENT013_SOURCE_PATHS).encode("utf-8") + raw = _git_bytes(root, "hash-object", "--no-filters", "--stdin-paths", input_bytes=payload) + lines = raw.decode("ascii", errors="strict").splitlines() + if len(lines) != len(EXPERIMENT013_SOURCE_PATHS): + raise Experiment013SourceError("Git did not hash every source worktree path") + return { + path: _sha1(oid, name=f"worktree hash-object OID for {path}") + for path, oid in zip(EXPERIMENT013_SOURCE_PATHS, lines, strict=True) + } + + +def _raw_file_identities(root: Path) -> dict[str, tuple[str, str]]: + identities: dict[str, tuple[str, str]] = {} + for relative in EXPERIMENT013_SOURCE_PATHS: + path = root / PurePosixPath(relative) + if _path_has_symlink_component(root, relative) or not path.is_file(): + raise Experiment013SourceError(f"source path is not a regular local file: {relative}") + try: + raw = path.read_bytes() + except OSError as error: + raise Experiment013SourceError(f"source path is unreadable: {relative}") from error + git_header = f"blob {len(raw)}\0".encode("ascii") + identities[relative] = ( + hashlib.sha1(git_header + raw).hexdigest(), # noqa: S324 - Git's frozen object ID. + hashlib.sha256(raw).hexdigest(), + ) + return identities + + +def _source_entries(root: Path, commit: str) -> list[dict[str, str]]: + tree = _tree_entries(root, commit) + index = _index_entries(root) + worktree = _worktree_oids(root) + raw = _raw_file_identities(root) + entries: list[dict[str, str]] = [] + for relative in EXPERIMENT013_SOURCE_PATHS: + mode, git_oid = tree[relative] + index_mode, index_oid = index[relative] + worktree_oid = worktree[relative] + raw_git_oid, raw_sha256 = raw[relative] + if index_mode != mode or len({git_oid, index_oid, worktree_oid, raw_git_oid}) != 1: + raise Experiment013SourceError( + f"source bytes/blob/mode differ from commit and index: {relative}" + ) + entries.append( + { + "path": relative, + "mode": mode, + "git_blob_oid": git_oid, + "index_blob_oid": index_oid, + "worktree_blob_oid": worktree_oid, + "raw_sha256": raw_sha256, + } + ) + return entries + + +def _head(root: Path) -> str: + head = _sha1(_git_text(root, "rev-parse", "HEAD"), name="repository HEAD") + if _git_text(root, "cat-file", "-t", head) != "commit": + raise Experiment013SourceError("repository HEAD is not a commit") + return head + + +def capture_experiment013_source_manifest(repo_root: str | Path) -> dict[str, object]: + """Capture the exact committed Experiment 013 source identity. + + Non-ignored untracked files and all tracked staged/unstaged changes are + rejected. Ignored artifacts do not affect capture. + """ + + first_repository = _authenticate_repository(repo_root) + first_head = _head(first_repository.root) + first_entries = _source_entries(first_repository.root, first_head) + + second_repository = _authenticate_repository(first_repository.root) + second_head = _head(second_repository.root) + second_entries = _source_entries(second_repository.root, second_head) + if ( + first_head != second_head + or first_entries != second_entries + or first_repository.public_binding != second_repository.public_binding + ): + raise Experiment013SourceError("repository source identity changed during capture") + + payload: dict[str, object] = { + "schema": EXPERIMENT013_SOURCE_MANIFEST_SCHEMA, + "profile": EXPERIMENT013_SOURCE_MANIFEST_PROFILE, + "object_format": "sha1", + "source_commit": first_head, + "repository_binding": first_repository.public_binding, + "paths": first_entries, + } + payload["canonical_manifest_sha256"] = canonical_experiment013_source_manifest_sha256(payload) + return validate_experiment013_source_manifest(payload) + + +def _assert_ancestor(root: Path, ancestor: str, descendant: str) -> None: + object_type = _run_git(root, "cat-file", "-e", f"{ancestor}^{{commit}}") + if object_type.returncode != 0: + raise Experiment013SourceError("source commit is unavailable from the local object store") + process = _run_git(root, "merge-base", "--is-ancestor", ancestor, descendant) + if process.returncode == 1: + raise Experiment013SourceError("source commit is not an ancestor of current HEAD") + if process.returncode != 0: + raise Experiment013SourceError("cannot authenticate source-commit ancestry") + + +def verify_experiment013_source_manifest( + manifest: Mapping[str, object], + repo_root: str | Path, +) -> dict[str, object]: + """Verify a frozen manifest against the clean source used at point-of-use. + + The source commit may be an ancestor of current ``HEAD``. Every frozen + path must nevertheless have byte-identical commit, index, and worktree + identities, so descendant commits cannot alter experiment code silently. + """ + + normalized = validate_experiment013_source_manifest(manifest) + source_commit = str(normalized["source_commit"]) + + first_repository = _authenticate_repository(repo_root) + first_head = _head(first_repository.root) + _assert_ancestor(first_repository.root, source_commit, first_head) + first_entries = _source_entries(first_repository.root, source_commit) + if first_entries != normalized["paths"]: + raise Experiment013SourceError("live Experiment 013 source differs from its manifest") + + second_repository = _authenticate_repository(first_repository.root) + second_head = _head(second_repository.root) + _assert_ancestor(second_repository.root, source_commit, second_head) + second_entries = _source_entries(second_repository.root, source_commit) + if first_head != second_head or first_entries != second_entries: + raise Experiment013SourceError("repository source identity changed during verification") + if second_entries != normalized["paths"]: + raise Experiment013SourceError("live Experiment 013 source drifted during verification") + return normalized + + +def canonical_experiment013_source_manifest_bytes( + manifest: Mapping[str, object], +) -> bytes: + """Validate and serialize a source manifest in its canonical file form.""" + + return _canonical_json_bytes(validate_experiment013_source_manifest(manifest)) + + +def verify_loaded_experiment013_recurquant_modules( + manifest: Mapping[str, object], + repo_root: str | Path, + required_module_names: Sequence[str], +) -> dict[str, str]: + """Authenticate selected loaded ``recurquant`` modules against a manifest. + + This helper is intentionally explicit rather than scanning every eagerly + imported package module. Callers supply the modules that participate in a + point-of-use code path after verifying the manifest itself. + """ + + normalized = validate_experiment013_source_manifest(manifest) + try: + root = Path(repo_root).resolve(strict=True) + except OSError as error: + raise Experiment013SourceError("repository root is unavailable") from error + entries = {str(item["path"]): item for item in normalized["paths"]} # type: ignore[index] + observed: dict[str, str] = {} + for module_name in required_module_names: + if ( + not isinstance(module_name, str) + or not module_name.startswith("recurquant.") + or module_name.endswith(".") + ): + raise Experiment013SourceError("required module name is not canonical") + module = sys.modules.get(module_name) + if module is None: + raise Experiment013SourceError( + f"required RecurQuant module is not loaded: {module_name}" + ) + raw_file = getattr(module, "__file__", None) + if not isinstance(raw_file, (str, os.PathLike)): + raise Experiment013SourceError(f"loaded module has no source file: {module_name}") + declared = Path(raw_file) + if declared.is_symlink(): + raise Experiment013SourceError(f"loaded module source is a symlink: {module_name}") + try: + resolved = declared.resolve(strict=True) + relative = resolved.relative_to(root).as_posix() + except (OSError, ValueError) as error: + raise Experiment013SourceError( + f"loaded module is outside the authenticated repository: {module_name}" + ) from error + expected = f"src/{module_name.replace('.', '/')}.py" + if relative != expected or relative not in entries: + raise Experiment013SourceError( + f"loaded module is not declared at its canonical source path: {module_name}" + ) + if _path_has_symlink_component(root, relative): + raise Experiment013SourceError(f"loaded module path traverses a symlink: {module_name}") + try: + digest = hashlib.sha256(resolved.read_bytes()).hexdigest() + except OSError as error: + raise Experiment013SourceError( + f"loaded module source is unreadable: {module_name}" + ) from error + if digest != entries[relative]["raw_sha256"]: # type: ignore[index] + raise Experiment013SourceError(f"loaded module source bytes drifted: {module_name}") + observed[module_name] = relative + if len(observed) != len(required_module_names): + raise Experiment013SourceError("required module names contain duplicates") + return observed + + +__all__ = [ + "EXPERIMENT013_REPOSITORY_BINDING_SCHEMA", + "EXPERIMENT013_SOURCE_MANIFEST_PROFILE", + "EXPERIMENT013_SOURCE_MANIFEST_SCHEMA", + "EXPERIMENT013_SOURCE_PATHS", + "Experiment013SourceError", + "canonical_experiment013_source_manifest_bytes", + "canonical_experiment013_source_manifest_sha256", + "capture_experiment013_source_manifest", + "validate_experiment013_source_manifest", + "verify_experiment013_source_manifest", + "verify_loaded_experiment013_recurquant_modules", +] diff --git a/tests/test_capture_static_q468_identity_input.py b/tests/test_capture_static_q468_identity_input.py index 328057c..837d1ab 100644 --- a/tests/test_capture_static_q468_identity_input.py +++ b/tests/test_capture_static_q468_identity_input.py @@ -7,6 +7,7 @@ import json import subprocess import sys +import urllib.parse from pathlib import Path from types import SimpleNamespace from typing import Any @@ -41,6 +42,22 @@ SPEC.loader.exec_module(capture) resolver = capture.resolver FIXTURE_BINDING_ARTIFACT = b"verified-fixture-binding-artifact" +FIXTURE_EXECUTION_ARTIFACTS = { + "repository_source_manifest_file_sha256": b"fixture-source-manifest", + "calibration_runtime_manifest_file_sha256": b"fixture-runtime-manifest", + "model_file_manifest_file_sha256": b"fixture-model-manifest", + "parquet_materialization_manifest_file_sha256": ( + REPOSITORY_ROOT / "research" / "experiment013-parquet-materializations.json" + ).read_bytes(), +} +FIXTURE_RUNTIME_CONTEXT = { + "base_runtime_root": REPOSITORY_ROOT / "fixture-base-runtime", + "staged_interpreter": REPOSITORY_ROOT / "fixture-base-runtime" / "python.exe", + "package_runtime_roots": { + "fixture-packages": REPOSITORY_ROOT / "fixture-packages" + }, + "package_import_paths": {"fixture-packages": "Lib/site-packages"}, +} def test_capture_script_imports_in_direct_cli_process() -> None: @@ -82,6 +99,88 @@ def _fake_generator_files() -> dict[str, bytes]: } +def _fake_command(receipt: dict[str, Any]) -> dict[str, Any]: + launcher = capture.RULER_LAUNCHER_PATH.read_bytes() + return { + "launcher_revision": capture.RULER_LAUNCHER_REVISION, + "launcher_source_sha256": capture.sha256_bytes(launcher), + "ruler_revision": resolver.RULER_REVISION, + "config": receipt["config"], + "configured_length": receipt["configured_length"], + "seed": receipt["seed"], + "argv": ["", "fixture", receipt["filename"]], + "shell": False, + } + + +def _fake_ruler_content(config: str, seed: int) -> tuple[str, str, list[str]]: + count = capture.RULER_REQUIRED_OUTPUT_COUNTS.get(config, 2) + if config in capture.RULER_NIAH_CONFIGS: + outputs = [f"{1_000_000 + seed + index:07d}" for index in range(count)] + if config in {"niah_multiquery", "niah_multivalue"}: + prompt = ( + "Some special magic numbers are hidden within the following text. " + f"Memorize these values: {' '.join(outputs)}. " + "What are all the special magic numbers for fixture-key mentioned in the " + "provided text?" + ) + answer_prefix = ( + " The special magic numbers for fixture-key mentioned in the provided text are" + ) + else: + prompt = ( + "A special magic number is hidden within the following text. " + f"Memorize this value: {outputs[0]}. " + "What is the special magic number for fixture-key mentioned in the provided text?" + ) + answer_prefix = ( + " The special magic number for fixture-key mentioned in the provided text is" + ) + return prompt, answer_prefix, outputs + if config == "vt": + outputs = [chr(65 + index) * 5 for index in range(count)] + prompt = ( + "Memorize and track the chain(s) of variable assignment hidden in the following " + f"text. {' '.join(outputs)}. Question: Find all variables that are assigned the " + "value 12345 in the text above." + ) + prefix = ( + " Answer: According to the chain(s) of variable assignment in the text above, " + "5 variables are assigned the value 12345, they are: " + ) + return prompt, prefix, outputs + if config == "cwe": + outputs = [f"fixtureword{index}" for index in range(count)] + prompt = ( + "Below is a numbered list of words. In these words, some appear more often than " + f"others. {' '.join(outputs)}. Question: What are the 10 most common words in " + "the above list?" + ) + prefix = " Answer: The top 10 words that appear most often in the list are:" + return prompt, prefix, outputs + if config == "fwe": + outputs = [chr(97 + index) * 6 for index in range(count)] + prompt = ( + "Read the following coded text and track the frequency of each coded word. " + f"{' '.join(outputs)}. What are the three most frequently appeared words in " + "the above coded text?" + ) + prefix = ( + " Answer: According to the coded text above, the three most frequently appeared " + "words are:" + ) + return prompt, prefix, outputs + if config in {"qa_1", "qa_2"}: + outputs = [f"fixture-answer-{seed}-{index}" for index in range(count)] + prompt = ( + "Answer the question based on the given documents. Only give me the answer and do " + "not output any other words.\n\nThe following are given documents.\n\n" + "Fixture evidence.\n\nQuestion: Is the fixture valid?" + ) + return prompt, " Answer:", outputs + raise AssertionError(f"unhandled fake RULER config: {config}") + + @pytest.fixture(autouse=True) def _bind_fixture_generator_blobs(monkeypatch: pytest.MonkeyPatch) -> None: files = _fake_generator_files() @@ -90,6 +189,90 @@ def _bind_fixture_generator_blobs(monkeypatch: pytest.MonkeyPatch) -> None: "RULER_GENERATOR_GIT_BLOBS", {path: capture._git_blob_sha1(content) for path, content in files.items()}, ) + monkeypatch.setattr( + capture, + "RULER_EXPECTED_CORPORA", + {"fixture-corpus.json": (17, _hash("fixture-corpus"))}, + ) + monkeypatch.setattr( + capture, + "RULER_EXPECTED_PACKAGE_RESOURCES", + {"fixture/resource.txt": (19, _hash("fixture-resource"))}, + ) + tokenizer_files = { + "tokenizer.json": b"fixture-tokenizer", + "tokenizer_config.json": b"fixture-tokenizer-config", + } + monkeypatch.setattr( + capture, + "RULER_EXPECTED_TOKENIZER_ASSETS", + {name: (len(data), capture.sha256_bytes(data)) for name, data in tokenizer_files.items()}, + ) + monkeypatch.setattr( + capture, + "RULER_COMMAND_MANIFEST_SHA256_BY_FILENAME", + { + item["filename"]: capture.sha256_bytes( + capture.canonical_json_bytes(_fake_command(item)) + ) + for item in capture.required_ruler_receipts() + }, + ) + strict_execution_decoder = capture._validate_execution_binding_artifacts + strict_execution_authenticator = capture._authenticate_execution_binding_artifacts + + def decode_execution(artifacts: dict[str, bytes]) -> dict[str, str]: + if artifacts == FIXTURE_EXECUTION_ARTIFACTS: + return { + field: capture.sha256_bytes(data) + for field, data in sorted(FIXTURE_EXECUTION_ARTIFACTS.items()) + } + return strict_execution_decoder(artifacts) + + monkeypatch.setattr(capture, "_validate_execution_binding_artifacts", decode_execution) + + def authenticate_execution( + artifacts: dict[str, bytes], + *, + runtime_context: Any, + previous: Any = None, + ) -> Any: + if artifacts == FIXTURE_EXECUTION_ARTIFACTS: + normalized_context = capture._normalize_runtime_authentication_context( + runtime_context + if isinstance(runtime_context, dict) + else { + "base_runtime_root": runtime_context.base_runtime_root, + "staged_interpreter": runtime_context.staged_interpreter, + "package_runtime_roots": runtime_context.package_runtime_roots, + "package_import_paths": runtime_context.package_import_paths, + } + ) + if previous is not None: + assert normalized_context == previous.runtime_context + return previous + return capture._AuthenticatedExecutionBindings( + bindings={ + field: capture.sha256_bytes(data) + for field, data in sorted(FIXTURE_EXECUTION_ARTIFACTS.items()) + }, + source_manifest={}, + runtime_manifest=object(), + model_manifest=object(), + runner=object(), + runtime_context=normalized_context, + ) + return strict_execution_authenticator( + artifacts, + runtime_context=runtime_context, + previous=previous, + ) + + monkeypatch.setattr( + capture, + "_authenticate_execution_binding_artifacts", + authenticate_execution, + ) strict_binding_decoder = resolver.deserialize_stage_a_calibration_binding_artifact def decode_binding(data: bytes) -> object: @@ -118,6 +301,7 @@ def __init__(self) -> None: self.extra_tokenizer_files: dict[str, bytes] = {} self.generator_files = _fake_generator_files() self.receipt_mutator: Any = None + self.manifest_mutator: Any = None def source_heads(self) -> dict[str, str]: self.accesses.append("source_heads") @@ -141,6 +325,21 @@ def tokenizer_material(self) -> Any: model_weights_loaded=False, ) + def execution_binding_artifacts(self) -> dict[str, bytes]: + return dict(FIXTURE_EXECUTION_ARTIFACTS) + + def runtime_authentication_context(self) -> dict[str, object]: + return { + "base_runtime_root": FIXTURE_RUNTIME_CONTEXT["base_runtime_root"], + "staged_interpreter": FIXTURE_RUNTIME_CONTEXT["staged_interpreter"], + "package_runtime_roots": dict( + FIXTURE_RUNTIME_CONTEXT["package_runtime_roots"] # type: ignore[arg-type] + ), + "package_import_paths": dict( + FIXTURE_RUNTIME_CONTEXT["package_import_paths"] # type: ignore[arg-type] + ), + } + def mbpp_train_rows(self) -> tuple[dict[str, Any], ...]: self.accesses.append("mbpp_train_rows") return tuple( @@ -182,8 +381,7 @@ def ruler_receipt( self, *, category: str, config: str, configured_length: int, seed: int ) -> dict[str, Any]: self.accesses.append(f"ruler_receipt:{category}:{config}:{configured_length}:{seed}") - prompt = f"RULER {category} {config} {configured_length} {seed}." - output_count = capture.RULER_REQUIRED_OUTPUT_COUNTS.get(config, 2) + prompt, answer_prefix, outputs = _fake_ruler_content(config, seed) receipt: dict[str, Any] = { "schema": capture.RULER_RECEIPT_SCHEMA, "source_id": resolver.RULER_SOURCE_ID, @@ -193,22 +391,234 @@ def ruler_receipt( "configured_length": configured_length, "seed": seed, "sample_index": 0, - "generator_reported_length": len(prompt) + 32, + "generator_reported_length": len( + FakeTokenizer().encode(prompt + answer_prefix, add_special_tokens=False) + ) + + capture.RULER_GENERATOR_TOKENS[config], "input": prompt, - "answer_prefix": " Answer:", - "outputs": [f"result-{config}-{seed}-{index}" for index in range(output_count)], - "auxiliary_files": [ - { - "name": f"fixture/{config}.txt", - "sha256": _hash(f"aux-{config}"), - "size_bytes": 100 + len(config), - } - ], + "answer_prefix": answer_prefix, + "outputs": outputs, + "auxiliary_files": [], } if self.receipt_mutator is not None: self.receipt_mutator(receipt) + generator_manifest = capture._ruler_generator_manifest(self.generator_files) + runtime_manifest = self._runtime_manifest() + static_inputs = capture._expected_ruler_static_inputs( + source_manifest=generator_manifest, + runtime_manifest=runtime_manifest, + ) + identity = next( + item + for item in capture.required_ruler_receipts() + if item["category"] == category + and item["config"] == config + and item["configured_length"] == configured_length + and item["seed"] == seed + ) + command = _fake_command(identity) + raw_data = self._raw_row(receipt) + receipt["auxiliary_files"] = sorted( + [ + *static_inputs, + { + "name": "generator/command-manifest.json", + "sha256": capture.sha256_bytes(capture.canonical_json_bytes(command)), + "size_bytes": len(capture.canonical_json_bytes(command)), + }, + { + "name": "generator/raw-validation.jsonl", + "sha256": capture.sha256_bytes(raw_data), + "size_bytes": len(raw_data), + }, + ], + key=lambda item: item["name"], + ) return receipt + @staticmethod + def _runtime_manifest() -> dict[str, Any]: + packages = capture._ruler_runtime_packages() + inventory: dict[str, Any] = {} + for name, version in packages.items(): + record = f"{name}=={version}\n".encode() + canonical = capture._canonical_distribution_name(name) + inventory[name] = { + "canonical_name": canonical, + "version": version, + "record_sha256": capture.sha256_bytes(record), + "record_size_bytes": len(record), + "files": [ + { + "path": f"{canonical}-{version}.dist-info/RECORD", + "sha256": capture.sha256_bytes(record), + "size_bytes": len(record), + } + ], + } + executable = b"fixture-python" + return { + "schema": capture.RULER_RUNTIME_MANIFEST_SCHEMA, + "python": capture.RULER_RUNTIME_PYTHON_VERSION, + "implementation": "cpython", + "cache_tag": "cpython-311", + "platform": "fixture-platform", + "machine": "fixture-machine", + "flags": { + "ignore_environment": 1, + "isolated": 1, + "no_user_site": 1, + }, + "startup_policy": dict(capture.RULER_SEALED_STARTUP_POLICY), + "excluded_startup_files": [ + { + "name": name, + "sha256": digest, + "size_bytes": size, + } + for name, (size, digest) in sorted( + capture.RULER_EXCLUDED_VIRTUALENV_STARTUP_FILES.items() + ) + ], + "source_python": { + "name": "source/python.exe", + "sha256": capture.sha256_bytes(b"fixture-source-python"), + "size_bytes": len(b"fixture-source-python"), + }, + "source_pyvenv_config": { + "name": "source/pyvenv.cfg", + "sha256": capture.sha256_bytes(b"fixture-pyvenv-config"), + "size_bytes": len(b"fixture-pyvenv-config"), + }, + "python_runtime_files": [ + { + "name": name, + "sha256": capture.sha256_bytes(data), + "size_bytes": len(data), + } + for name, data in ( + ("python.exe", b"fixture-runtime-python"), + ("python3.dll", b"fixture-python3-dll"), + ("python311.dll", b"fixture-python311-dll"), + ) + ], + "executable": { + "name": "python.exe", + "sha256": capture.sha256_bytes(executable), + "size_bytes": len(executable), + }, + "packages": packages, + "installed_distributions": { + capture._canonical_distribution_name(name): version + for name, version in packages.items() + }, + "distribution_file_inventory": inventory, + "forbidden_modules": {name: False for name in capture.RULER_FORBIDDEN_RUNTIME_MODULES}, + } + + @staticmethod + def _raw_row(receipt: dict[str, Any]) -> bytes: + input_text = receipt["input"] + first_output = receipt["outputs"][0] + niah = receipt["config"] in capture.RULER_NIAH_CONFIGS + row: dict[str, Any] = { + "index": input_text.find(first_output) if niah else 0, + "input": input_text, + "outputs": receipt["outputs"], + "length": receipt["generator_reported_length"], + "length_w_model_temp": receipt["generator_reported_length"], + "answer_prefix": receipt["answer_prefix"], + } + if niah: + row["token_position_answer"] = len( + FakeTokenizer().encode( + input_text[: input_text.find(first_output)], add_special_tokens=False + ) + ) + return capture.canonical_json_bytes(row) + + def ruler_receipt_bytes( + self, *, category: str, config: str, configured_length: int, seed: int + ) -> bytes: + return capture.canonical_json_bytes( + self.ruler_receipt( + category=category, + config=config, + configured_length=configured_length, + seed=seed, + ) + ) + + def ruler_generation_manifest_bytes(self) -> bytes: + generator_manifest = capture._ruler_generator_manifest(self.generator_files) + runtime_manifest = self._runtime_manifest() + static_inputs = capture._expected_ruler_static_inputs( + source_manifest=generator_manifest, + runtime_manifest=runtime_manifest, + ) + results = [] + for identity in capture.required_ruler_receipts(): + receipt_bytes = self.ruler_receipt_bytes( + category=identity["category"], + config=identity["config"], + configured_length=identity["configured_length"], + seed=identity["seed"], + ) + receipt = json.loads(receipt_bytes) + command = _fake_command(identity) + command_bytes = capture.canonical_json_bytes(command) + raw_data = self._raw_row(receipt) + results.append( + { + "category": identity["category"], + "command_manifest": command, + "command_manifest_file": { + "name": "generator/command-manifest.json", + "sha256": capture.sha256_bytes(command_bytes), + "size_bytes": len(command_bytes), + }, + "config": identity["config"], + "configured_length": identity["configured_length"], + "filename": identity["filename"], + "generator_reported_length": receipt["generator_reported_length"], + "phase": identity["phase"], + "raw_validation_base64": base64.b64encode(raw_data).decode("ascii"), + "raw_validation_file": { + "name": "generator/raw-validation.jsonl", + "sha256": capture.sha256_bytes(raw_data), + "size_bytes": len(raw_data), + }, + "seed": identity["seed"], + "sha256": capture.sha256_bytes(receipt_bytes), + "size_bytes": len(receipt_bytes), + } + ) + launcher = next( + item + for item in static_inputs + if item["name"] == "launcher/generate_static_q468_ruler_receipts.py" + ) + manifest = { + "schema": capture.RULER_GENERATION_MANIFEST_SCHEMA, + "launcher_revision": capture.RULER_LAUNCHER_REVISION, + "launcher_source": launcher, + "ruler_revision": resolver.RULER_REVISION, + "source_manifest": generator_manifest, + "source_manifest_sha256": capture.sha256_bytes( + capture.canonical_json_bytes(generator_manifest) + ), + "runtime_manifest": runtime_manifest, + "runtime_manifest_sha256": capture.sha256_bytes( + capture.canonical_json_bytes(runtime_manifest) + ), + "static_inputs": static_inputs, + "receipt_count": 20, + "receipts": results, + } + if self.manifest_mutator is not None: + self.manifest_mutator(manifest) + return capture.canonical_json_bytes(manifest) + def humaneval_projection(self) -> tuple[Any, ...]: self.accesses.append("humaneval_projection") return tuple(capture.ProjectionRow(f"HumanEval/{offset}", offset) for offset in range(164)) @@ -224,6 +634,57 @@ def humaneval_row(self, *, offset: int, expected_task_id: str) -> dict[str, Any] } +def _fixture_execution_authentication( + runtime_context: Any, + *, + runner: object | None = None, +) -> Any: + return capture._AuthenticatedExecutionBindings( + bindings={ + field: capture.sha256_bytes(data) + for field, data in sorted(FIXTURE_EXECUTION_ARTIFACTS.items()) + }, + source_manifest={}, + runtime_manifest=object(), + model_manifest=object(), + runner=object() if runner is None else runner, + runtime_context=runtime_context, + ) + + +@pytest.mark.parametrize( + "flags", + [ + {"ignore_environment": 0, "isolated": 1, "no_user_site": 1}, + {"ignore_environment": 1, "isolated": 0, "no_user_site": 1}, + {"ignore_environment": 1, "isolated": 1, "no_user_site": 0}, + {"ignore_environment": True, "isolated": 1, "no_user_site": 1}, + { + "ignore_environment": 1, + "isolated": 1, + "no_user_site": 1, + "unexpected": 1, + }, + ], +) +def test_ruler_runtime_manifest_rejects_nonisolated_or_extra_flags( + flags: dict[str, int], +) -> None: + manifest = FakeSource._runtime_manifest() + manifest["flags"] = flags + + with pytest.raises(ValueError, match="isolation flags drifted"): + capture._normalize_ruler_runtime_manifest(manifest) + + +def test_ruler_runtime_manifest_rejects_boolean_numeric_startup_attestation() -> None: + manifest = FakeSource._runtime_manifest() + manifest["startup_policy"]["no_site"] = True + + with pytest.raises(ValueError, match="sealed-startup policy drifted"): + capture._normalize_ruler_runtime_manifest(manifest) + + def _binding() -> bytes: return FIXTURE_BINDING_ARTIFACT @@ -286,6 +747,181 @@ def test_calibration_capture_is_deterministic_and_resolver_compatible() -> None: ) +def test_execution_artifacts_are_authenticated_before_and_after_all_data_access( + monkeypatch: pytest.MonkeyPatch, +) -> None: + source = FakeSource() + + def authenticate( + _artifacts: Any, + *, + runtime_context: Any, + previous: Any = None, + ) -> Any: + source.accesses.append( + "execution_auth:post" if previous is not None else "execution_auth:pre" + ) + if previous is not None: + return previous + return _fixture_execution_authentication(runtime_context) + + monkeypatch.setattr(capture, "_authenticate_execution_binding_artifacts", authenticate) + capture.capture_identity_input(phase="calibration", source=source) + + assert source.accesses[0] == "execution_auth:pre" + assert source.accesses[1] == "source_heads" + assert source.accesses[-2:] == ["source_heads", "execution_auth:post"] + + +def test_pre_capture_execution_authentication_failure_touches_no_data_source( + monkeypatch: pytest.MonkeyPatch, +) -> None: + source = FakeSource() + + def reject(*_args: Any, **_kwargs: Any) -> Any: + raise ValueError("pinned model Hub metadata authentication failed") + + monkeypatch.setattr(capture, "_authenticate_execution_binding_artifacts", reject) + + with pytest.raises(ValueError, match="model Hub metadata"): + capture.capture_identity_input(phase="calibration", source=source) + assert source.accesses == [] + + +def test_post_capture_execution_authentication_drift_is_rejected( + monkeypatch: pytest.MonkeyPatch, +) -> None: + source = FakeSource() + + def authenticate( + _artifacts: Any, + *, + runtime_context: Any, + previous: Any = None, + ) -> Any: + if previous is not None: + raise ValueError("execution-binding artifacts changed during capture") + return _fixture_execution_authentication(runtime_context) + + monkeypatch.setattr(capture, "_authenticate_execution_binding_artifacts", authenticate) + + with pytest.raises(ValueError, match="changed during capture"): + capture.capture_identity_input(phase="calibration", source=source) + assert "humaneval_projection" in source.accesses + assert source.accesses[-1] == "source_heads" + + +def test_preloaded_calibration_runner_is_rejected() -> None: + sentinel = object() + sys.modules[capture._CALIBRATION_RUNNER_MODULE_NAME] = sentinel # type: ignore[assignment] + try: + with pytest.raises(RuntimeError, match="preloaded"): + capture._load_calibration_runner_module() + finally: + if sys.modules.get(capture._CALIBRATION_RUNNER_MODULE_NAME) is sentinel: + sys.modules.pop(capture._CALIBRATION_RUNNER_MODULE_NAME, None) + + +def test_public_calibration_materialization_is_the_exact_capture_with_tokens() -> None: + captured = capture.capture_identity_input(phase="calibration", source=FakeSource()) + shared_result, shared_tokens = capture._capture_identity_input_with_tokens( + phase="calibration", + source=FakeSource(), + collect_tokens=True, + ) + source = FakeSource() + materialized = capture.materialize_calibration_identity_sequences(source=source) + + assert capture.canonical_json_bytes(shared_result) == capture.canonical_json_bytes(captured) + assert len(shared_tokens) == 160 + assert materialized.capture_input_sha256 == capture.sha256_bytes( + capture.canonical_json_bytes(captured) + ) + assert capture.canonical_json_bytes(materialized.identity_records) == ( + capture.canonical_json_bytes(captured["records"]) + ) + assert len(materialized.sequences) == len(materialized.by_identity_record_sha256) == 160 + assert source.head_calls == 2 + assert source.accesses.count("tokenizer_material") == 1 + assert source.accesses.count("ruler_generator_files") == 1 + + for sequence, expected_record in zip(materialized.sequences, captured["records"], strict=True): + record = sequence.identity_record + assert capture.canonical_json_bytes(record) == capture.canonical_json_bytes(expected_record) + assert isinstance(sequence.prompt_token_ids, tuple) + assert isinstance(sequence.target_token_ids, tuple) + assert sequence.sequence_token_ids == ( + sequence.prompt_token_ids + sequence.target_token_ids + ) + assert record["prompt_token_ids_sha256"] == capture._token_hash(sequence.prompt_token_ids) + assert record["target_token_ids_sha256"] == capture._token_hash(sequence.target_token_ids) + assert record["sequence_token_ids_sha256"] == capture._token_hash( + sequence.sequence_token_ids + ) + assert record["sequence_length"] == len(sequence.sequence_token_ids) + assert record["token_span"] == { + "prefill_start": 0, + "prefill_stop": len(sequence.prompt_token_ids), + "scored_start": len(sequence.prompt_token_ids), + "scored_stop": len(sequence.sequence_token_ids), + "cache_exposed_start": len(sequence.sequence_token_ids), + "cache_exposed_stop": len(sequence.sequence_token_ids), + } + assert (bool(sequence.target_token_ids)) is (record["family"] == "mbpp") + + for sequence in reversed(materialized.sequences): + assert materialized.lookup(sequence.identity_record_sha256) is sequence + first_copy = materialized.sequences[0].identity_record + first_copy["token_span"]["prefill_start"] = 99 + assert materialized.sequences[0].identity_record["token_span"]["prefill_start"] == 0 + assert len(materialized.token_sequence_manifest_sha256) == 64 + + +def test_public_calibration_materialization_returns_no_raw_content() -> None: + materialized = capture.materialize_calibration_identity_sequences(source=FakeSource()) + record_bytes = b"".join( + capture.canonical_json_bytes(record) for record in materialized.identity_records + ) + + assert b"Return 601" not in record_bytes + assert b"RULER retrieval" not in record_bytes + assert b"answer_prefix" not in record_bytes + assert b"auxiliary_files" not in record_bytes + assert b"source_payload" not in record_bytes + assert b"formatted_payload" not in record_bytes + + +@pytest.mark.parametrize( + ("mutate", "message"), + [ + ( + lambda source: setattr(source, "drift_after_capture", True), + "post-capture source HEAD", + ), + ( + lambda source: source.extra_tokenizer_files.update({"model.safetensors": b"forbidden"}), + "model weight-like file is forbidden", + ), + ( + lambda source: setattr( + source, + "manifest_mutator", + lambda manifest: manifest["receipts"].pop(), + ), + "all 20 receipt results", + ), + ], +) +def test_public_calibration_materialization_preserves_capture_failures( + mutate: Any, message: str +) -> None: + source = FakeSource() + mutate(source) + + with pytest.raises(ValueError, match=message): + capture.materialize_calibration_identity_sequences(source=source) + + def test_frozen_calibration_identity_decoder_recomputes_capture_lineage() -> None: captured = capture.capture_identity_input(phase="calibration", source=FakeSource()) candidate = resolver.build_candidate( @@ -305,6 +941,14 @@ def test_frozen_calibration_identity_decoder_recomputes_capture_lineage() -> Non assert decoded.canonical_evidence_sha256 == frozen["canonical_evidence_sha256"] assert len(decoded.records) == 160 assert len(decoded.assignment) == 160 + assert decoded.execution_bindings == { + field: capture.sha256_bytes(data) + for field, data in sorted(FIXTURE_EXECUTION_ARTIFACTS.items()) + } + assert ( + decoded.parquet_materialization_manifest_file_sha256 + == resolver.PARQUET_MATERIALIZATION_MANIFEST_FILE_SHA256 + ) assert ( decoded.assignment_sha256 == frozen["evidence"]["calibration_split_half"]["assignment_sha256"] @@ -518,6 +1162,107 @@ def test_ruler_receipt_required_output_cardinality_and_uniqueness_fail_closed() ) +@pytest.mark.parametrize( + ("config", "mutate", "message"), + [ + ( + "niah_multiquery", + lambda receipt: receipt.update( + { + "input": receipt["input"].replace( + "What are all the special magic numbers", + "Which values", + ) + } + ), + "input task markers drifted", + ), + ( + "vt", + lambda receipt: receipt.update({"answer_prefix": " Answer:"}), + "answer-prefix boundaries drifted", + ), + ( + "fwe", + lambda receipt: receipt.update({"outputs": ["INVALID", "bbbbbb", "cccccc"]}), + "output format drifted", + ), + ( + "vt", + lambda receipt: receipt.update( + {"outputs": ["ZZZZZ", *receipt["outputs"][1:]]} + ), + "required answer is absent", + ), + ], +) +def test_ruler_receipt_replays_frozen_task_semantics( + config: str, + mutate: Any, + message: str, +) -> None: + category = next( + category + for category, configs in capture.RULER_CONFIGS_BY_CATEGORY.items() + if config in configs + ) + receipt = FakeSource().ruler_receipt( + category=category, + config=config, + configured_length=4_096, + seed=2_339, + ) + mutate(receipt) + + with pytest.raises(ValueError, match=message): + capture._normalize_ruler_receipt( + receipt, + category=category, + config=config, + configured_length=4_096, + seed=2_339, + ) + + +def test_ruler_receipt_rejects_boolean_sample_index() -> None: + receipt = FakeSource().ruler_receipt( + category="retrieval", + config="niah_multiquery", + configured_length=4_096, + seed=2_339, + ) + receipt["sample_index"] = False + + with pytest.raises(ValueError, match="sample_index must be an integer"): + capture._normalize_ruler_receipt( + receipt, + category="retrieval", + config="niah_multiquery", + configured_length=4_096, + seed=2_339, + ) + + +@pytest.mark.parametrize("field", ["index", "token_position_answer"]) +def test_ruler_raw_row_rejects_boolean_numeric_fields(field: str) -> None: + source = FakeSource() + receipt = source.ruler_receipt( + category="retrieval", + config="niah_multiquery", + configured_length=4_096, + seed=2_339, + ) + row = json.loads(source._raw_row(receipt)) + row[field] = False + + with pytest.raises(ValueError, match="must be an integer"): + capture._verify_ruler_raw_row( + capture.canonical_json_bytes(row), + receipt=receipt, + tokenizer=FakeTokenizer(), + ) + + def test_pg19_ranks_all_ids_before_text_and_skips_ineligible_rows() -> None: source = FakeSource() projection = source.pg19_projection("train") @@ -738,6 +1483,42 @@ def test_ruler_generator_source_tamper_is_rejected() -> None: capture.capture_identity_input(phase="calibration", source=source) +def test_incomplete_ruler_generation_manifest_is_rejected() -> None: + source = FakeSource() + source.manifest_mutator = lambda manifest: manifest["receipts"].pop() + + with pytest.raises(ValueError, match="all 20 receipt results"): + capture.capture_identity_input(phase="calibration", source=source) + + +def test_ruler_command_manifest_argv_tamper_is_rejected() -> None: + source = FakeSource() + source.manifest_mutator = lambda manifest: manifest["receipts"][0]["command_manifest"][ + "argv" + ].append("--unbound") + + with pytest.raises(ValueError, match="command argv drifted"): + capture.capture_identity_input(phase="calibration", source=source) + + +def test_ruler_receipt_rejects_arbitrary_auxiliary_claim() -> None: + source = FakeSource() + original = source.ruler_receipt_bytes + + def with_extra_auxiliary(**kwargs: Any) -> bytes: + value = json.loads(original(**kwargs)) + value["auxiliary_files"].append( + {"name": "unbound/claim.txt", "sha256": "7" * 64, "size_bytes": 1} + ) + value["auxiliary_files"].sort(key=lambda item: item["name"]) + return capture.canonical_json_bytes(value) + + source.ruler_receipt_bytes = with_extra_auxiliary # type: ignore[method-assign] + + with pytest.raises(ValueError, match="auxiliary inventory drifted"): + capture.capture_identity_input(phase="calibration", source=source) + + def test_duplicate_projection_identity_fails_closed() -> None: source = FakeSource() original = source.pg19_projection @@ -771,65 +1552,14 @@ def test_duplicate_json_keys_are_rejected() -> None: capture._strict_json(b'{"a":1,"a":2}', context="fixture") -def test_active_parquet_manifest_excludes_stale_repo_tree_files( - monkeypatch: pytest.MonkeyPatch, -) -> None: - payload = { - "parquet_files": [ - { - "dataset": resolver.PG19_DATASET_ID, - "config": "default", - "split": "train", - "url": ( - "https://huggingface.co/datasets/emozilla/pg19/resolve/" - "refs%2Fconvert%2Fparquet/default/partial-train/0001.parquet" - ), - "filename": "0001.parquet", - "size": 200, - }, - { - "dataset": resolver.PG19_DATASET_ID, - "config": "default", - "split": "train", - "url": ( - "https://huggingface.co/datasets/emozilla/pg19/resolve/" - "refs%2Fconvert%2Fparquet/default/partial-train/0000.parquet" - ), - "filename": "0000.parquet", - "size": 100, - }, - ], - "repo_tree_files": [ - "data/train-00022-of-00023-stale-sibling.parquet", - ], - } +def test_live_capture_source_contains_no_mutable_dataset_viewer_paths() -> None: + source = SCRIPT_PATH.read_text(encoding="utf-8") - class Response(io.BytesIO): - headers = {"x-revision": resolver.PG19_REVISION} - - def __enter__(self) -> Response: - return self - - def __exit__(self, *_args: object) -> None: - self.close() - - def fake_urlopen(_request: object, timeout: int) -> Response: - assert timeout == 60 - return Response(json.dumps(payload).encode()) - - monkeypatch.setattr(capture.urllib.request, "urlopen", fake_urlopen) - aliases = capture.LiveCaptureSource._active_parquet_aliases( - dataset_id=resolver.PG19_DATASET_ID, - revision=resolver.PG19_REVISION, - config="default", - split="train", - ) - - assert aliases == ( - "datasets/emozilla/pg19@~parquet/default/partial-train/0000.parquet", - "datasets/emozilla/pg19@~parquet/default/partial-train/0001.parquet", - ) - assert all("stale" not in alias for alias in aliases) + assert '"/rows"' not in source + assert '"/parquet"' not in source + assert "@~parquet" not in source + assert "project_experiment013_parquet_columns" in source + assert "read_experiment013_parquet_row" in source def test_calibration_binding_requires_verified_artifact_and_is_normalized() -> None: @@ -851,6 +1581,355 @@ def test_calibration_binding_requires_verified_artifact_and_is_normalized() -> N ) +def test_execution_bindings_are_derived_from_verified_artifact_bytes() -> None: + captured = capture.capture_identity_input(phase="calibration", source=FakeSource()) + + assert captured["execution_bindings"] == { + field: capture.sha256_bytes(data) + for field, data in sorted(FIXTURE_EXECUTION_ARTIFACTS.items()) + } + source = FakeSource() + source.execution_binding_artifacts = lambda: { # type: ignore[method-assign] + **FIXTURE_EXECUTION_ARTIFACTS, + "repository_source_manifest_file_sha256": b"not-json", + } + with pytest.raises(ValueError, match="repository source manifest"): + capture.capture_identity_input(phase="calibration", source=source) + + +def test_execution_artifact_decoders_run_before_file_hash_binding( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from recurquant import experiment013_source + + source_payload: dict[str, Any] = { + "schema": experiment013_source.EXPERIMENT013_SOURCE_MANIFEST_SCHEMA, + "profile": experiment013_source.EXPERIMENT013_SOURCE_MANIFEST_PROFILE, + "object_format": "sha1", + "source_commit": "a" * 40, + "repository_binding": { + "schema": experiment013_source.EXPERIMENT013_REPOSITORY_BINDING_SCHEMA, + "worktree_layout": "primary", + **{field: True for field in experiment013_source._TRUE_BINDING_FIELDS}, + }, + "paths": [ + { + "path": path, + "mode": "100644", + "git_blob_oid": "b" * 40, + "index_blob_oid": "b" * 40, + "worktree_blob_oid": "b" * 40, + "raw_sha256": "c" * 64, + } + for path in experiment013_source.EXPERIMENT013_SOURCE_PATHS + ], + } + source_payload["canonical_manifest_sha256"] = ( + experiment013_source.canonical_experiment013_source_manifest_sha256(source_payload) + ) + source_bytes = experiment013_source._canonical_json_bytes(source_payload) + runtime_bytes = b"strict-runtime-manifest\n" + model_bytes = b"strict-model-manifest\n" + + class FakeRunner: + @staticmethod + def parse_calibration_runtime_manifest(data: bytes) -> Any: + if data != runtime_bytes: + raise ValueError("calibration runtime manifest is invalid") + return SimpleNamespace(file_sha256=capture.sha256_bytes(data)) + + @staticmethod + def parse_model_file_manifest(data: bytes) -> Any: + if data != model_bytes: + raise ValueError("model file manifest is invalid") + return SimpleNamespace( + file_sha256=capture.sha256_bytes(data), + model_id=resolver.PRIMARY_MODEL_ID, + revision=resolver.PRIMARY_MODEL_REVISION, + transformers_version=resolver.TRANSFORMERS_VERSION, + ) + + monkeypatch.setattr(capture, "_load_calibration_runner_module", lambda: FakeRunner()) + artifacts = { + "repository_source_manifest_file_sha256": source_bytes, + "calibration_runtime_manifest_file_sha256": runtime_bytes, + "model_file_manifest_file_sha256": model_bytes, + "parquet_materialization_manifest_file_sha256": FIXTURE_EXECUTION_ARTIFACTS[ + "parquet_materialization_manifest_file_sha256" + ], + } + + assert capture._validate_execution_binding_artifacts(artifacts) == { + field: capture.sha256_bytes(data) for field, data in sorted(artifacts.items()) + } + with pytest.raises(ValueError, match="calibration runtime manifest"): + capture._validate_execution_binding_artifacts( + {**artifacts, "calibration_runtime_manifest_file_sha256": b"{}\n"} + ) + with pytest.raises(ValueError, match="model file manifest"): + capture._validate_execution_binding_artifacts( + {**artifacts, "model_file_manifest_file_sha256": b"{}\n"} + ) + + +def test_point_of_use_authentication_rechecks_source_runtime_modules_and_model( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + events: list[str] = [] + package_root = tmp_path / "packages" + (package_root / "Lib" / "site-packages").mkdir(parents=True) + runtime_context = capture._normalize_runtime_authentication_context( + { + "base_runtime_root": tmp_path / "runtime", + "staged_interpreter": tmp_path / "runtime" / "python.exe", + "package_runtime_roots": {"packages": package_root}, + "package_import_paths": {"packages": "Lib/site-packages"}, + } + ) + artifacts = { + "repository_source_manifest_file_sha256": b"source\n", + "calibration_runtime_manifest_file_sha256": b"runtime\n", + "model_file_manifest_file_sha256": b"model\n", + "parquet_materialization_manifest_file_sha256": b"parquet\n", + } + bindings = { + field: capture.sha256_bytes(data) for field, data in sorted(artifacts.items()) + } + source_manifest = {"paths": []} + + class SourceModule: + @staticmethod + def verify_experiment013_source_manifest(manifest: Any, *, repo_root: Path) -> Any: + assert repo_root == REPOSITORY_ROOT + events.append("source") + return dict(manifest) + + @staticmethod + def verify_loaded_experiment013_recurquant_modules( + _manifest: Any, + _root: Path, + names: Any, + ) -> None: + assert names == ( + "recurquant.experiment013_source", + "recurquant.experiment013_parquet", + ) + events.append("loaded-source-parquet") + + runtime_manifest = SimpleNamespace( + package_roots=(SimpleNamespace(name="packages", import_path="Lib/site-packages"),) + ) + + class Runner: + @staticmethod + def authenticate_calibration_runtime( + _manifest: Any, + *, + base_runtime_root: Path, + package_roots: Any, + interpreter_path: Path, + ) -> Any: + assert base_runtime_root == runtime_context.base_runtime_root + assert package_roots == runtime_context.package_runtime_roots + assert interpreter_path == runtime_context.staged_interpreter + events.append("runtime") + return SimpleNamespace( + manifest_file_sha256=bindings[ + "calibration_runtime_manifest_file_sha256" + ] + ) + + @staticmethod + def capture_model_file_manifest_from_hub( + model_id: str, + revision: str, + *, + transformers_version: str, + ) -> bytes: + assert (model_id, revision, transformers_version) == ( + resolver.PRIMARY_MODEL_ID, + resolver.PRIMARY_MODEL_REVISION, + resolver.TRANSFORMERS_VERSION, + ) + events.append("model") + return artifacts["model_file_manifest_file_sha256"] + + runner = Runner() + decoded = capture._DecodedExecutionBindingArtifacts( + bindings=bindings, + source_manifest=source_manifest, + runtime_manifest=runtime_manifest, + model_manifest=object(), + source_module=SourceModule(), + parquet_module=object(), + ) + + def load_runner() -> Any: + events.append("load-runner") + sys.modules[capture._CALIBRATION_RUNNER_MODULE_NAME] = runner # type: ignore[assignment] + return runner + + def decode(_artifacts: Any, *, runner: Any) -> Any: + assert runner is not None + events.append("decode") + return decoded + + monkeypatch.setattr(capture, "_load_calibration_runner_module", load_runner) + monkeypatch.setattr(capture, "_decode_execution_binding_artifacts", decode) + monkeypatch.setattr( + capture, + "_verify_loaded_runner_source", + lambda *_args: events.append("runner-source"), + ) + try: + first = capture._authenticate_execution_binding_artifacts( + artifacts, + runtime_context=runtime_context, + ) + capture._authenticate_execution_binding_artifacts( + artifacts, + runtime_context=runtime_context, + previous=first, + ) + finally: + if sys.modules.get(capture._CALIBRATION_RUNNER_MODULE_NAME) is runner: + sys.modules.pop(capture._CALIBRATION_RUNNER_MODULE_NAME, None) + + assert events == [ + "load-runner", + "decode", + "source", + "loaded-source-parquet", + "runner-source", + "runtime", + "model", + "decode", + "source", + "loaded-source-parquet", + "runner-source", + "runtime", + "model", + ] + + +def test_loaded_calibration_runner_source_is_bound_to_source_manifest() -> None: + runner = SimpleNamespace(__file__=str(capture.CALIBRATION_RUNNER_PATH)) + manifest = { + "paths": [ + { + "path": "scripts/run_static_q468_calibration.py", + "raw_sha256": capture.sha256_bytes( + capture.CALIBRATION_RUNNER_PATH.read_bytes() + ), + } + ] + } + + capture._verify_loaded_runner_source(runner, manifest) + manifest["paths"][0]["raw_sha256"] = "0" * 64 + with pytest.raises(RuntimeError, match="runner source bytes drifted"): + capture._verify_loaded_runner_source(runner, manifest) + + +@pytest.mark.parametrize( + "mutate", + [ + lambda context: context.update({"extra": Path("unused")}), + lambda context: context.update({"base_runtime_root": "not-a-path"}), + lambda context: context.update({"package_runtime_roots": {"packages": False}}), + lambda context: context.update( + {"package_import_paths": {"packages": "../site-packages"}} + ), + lambda context: context.update( + {"package_import_paths": {"packages": "Lib\\site-packages"}} + ), + lambda context: context.update( + {"package_import_paths": {"different": "Lib/site-packages"}} + ), + ], +) +def test_runtime_authentication_context_rejects_noncanonical_values(mutate: Any) -> None: + context = copy.deepcopy(FIXTURE_RUNTIME_CONTEXT) + mutate(context) + + with pytest.raises(ValueError): + capture._normalize_runtime_authentication_context(context) + + +def test_cli_runtime_context_rejects_duplicate_package_names() -> None: + with pytest.raises(ValueError, match="duplicate name"): + capture._parse_named_cli_values( + ["packages=first", "packages=second"], + context="--package-root", + paths=True, + ) + + +def test_cli_requires_all_four_execution_artifact_files(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="repository-source-manifest"): + capture.main( + [ + "--phase", + "calibration", + "--ruler-receipt-dir", + str(tmp_path / "receipts"), + "--dry-run", + "--base-runtime-root", + str(tmp_path / "runtime"), + "--staged-interpreter", + str(tmp_path / "runtime" / "python.exe"), + "--package-root", + f"packages={tmp_path / 'packages'}", + "--package-import-path", + "packages=Lib/site-packages", + ] + ) + + +def test_live_source_probes_only_the_frozen_objects( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + observed_hf: list[tuple[str, str, str]] = [] + observed_urls: list[str] = [] + + class FakeApi: + @staticmethod + def model_info(repo_id: str, *, revision: str) -> Any: + observed_hf.append(("model", repo_id, revision)) + return SimpleNamespace(sha=revision) + + @staticmethod + def dataset_info(repo_id: str, *, revision: str) -> Any: + observed_hf.append(("dataset", repo_id, revision)) + return SimpleNamespace(sha=revision) + + class Response(io.BytesIO): + def __enter__(self) -> Response: + return self + + def __exit__(self, *_args: object) -> None: + self.close() + + def fake_urlopen(request: Any, timeout: int) -> Response: + assert timeout == 30 + observed_urls.append(request.full_url) + revision = urllib.parse.unquote(request.full_url.rsplit("/", 1)[1]) + return Response(json.dumps({"sha": revision}).encode()) + + monkeypatch.setitem(sys.modules, "huggingface_hub", SimpleNamespace(HfApi=FakeApi)) + monkeypatch.setattr(capture.urllib.request, "urlopen", fake_urlopen) + source = capture.LiveCaptureSource( + cache_dir=tmp_path / "cache", ruler_receipt_dir=tmp_path / "receipts" + ) + + assert source.source_heads() == capture.EXPECTED_SOURCE_HEADS + assert all(revision in capture.EXPECTED_SOURCE_HEADS.values() for _, _, revision in observed_hf) + assert any(url.endswith(resolver.RULER_REVISION) for url in observed_urls) + assert any(url.endswith(resolver.EVALPLUS_SOURCE_REVISION) for url in observed_urls) + assert all("default_branch" not in url for url in observed_urls) + + def test_capture_output_contains_no_raw_model_or_weight_claim() -> None: captured = capture.capture_identity_input(phase="calibration", source=FakeSource()) serialized = capture.canonical_json_bytes(copy.deepcopy(captured)) diff --git a/tests/test_experiment013_calibration_api.py b/tests/test_experiment013_calibration_api.py new file mode 100644 index 0000000..1053ca6 --- /dev/null +++ b/tests/test_experiment013_calibration_api.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from recurquant.experiment013_calibration_api import ( + AdapterConstructionContext, + AuthenticatedModelFiles, + AuthenticatedSequence, + CalibrationAdapter, + ModelFileIdentity, + StepObservation, +) + +ROOT = Path(__file__).resolve().parents[1] +API_PATH = ROOT / "src" / "recurquant" / "experiment013_calibration_api.py" + + +def binding_artifacts() -> dict[str, bytes]: + return { + "calibration_runtime_manifest_bytes": b"runtime\n", + "model_file_manifest_bytes": b"model\n", + "parquet_materialization_manifest_bytes": b"parquet\n", + "repository_source_manifest_bytes": b"source\n", + } + + +def runtime_context() -> dict[str, object]: + return { + "base_runtime_root": ROOT / "runtime" / "base", + "staged_interpreter": ROOT / "runtime" / "base" / "python.exe", + "package_runtime_roots": {"calibration": ROOT / "runtime" / "packages"}, + "package_import_paths": {"calibration": "Lib/site-packages"}, + } + + +def test_context_copy_normalizes_exact_binding_bytes() -> None: + source = binding_artifacts() + context = AdapterConstructionContext( + repository_root=ROOT, + model_root=ROOT / "model", + cache_root=ROOT / "cache", + ruler_root=ROOT / "ruler", + runtime_authentication_context=runtime_context(), + execution_binding_artifacts=source, + ) + + source["model_file_manifest_bytes"] = b"changed" + assert context.execution_binding_artifacts["model_file_manifest_bytes"] == b"model\n" + with pytest.raises(TypeError): + context.execution_binding_artifacts["new"] = b"no" # type: ignore[index] + + incomplete = binding_artifacts() + incomplete.pop("model_file_manifest_bytes") + with pytest.raises(ValueError, match="keys differ"): + AdapterConstructionContext(ROOT, ROOT, ROOT, ROOT, runtime_context(), incomplete) + + +def test_context_normalizes_and_freezes_runtime_authentication_paths() -> None: + source = runtime_context() + context = AdapterConstructionContext( + ROOT, + ROOT, + ROOT, + ROOT, + source, + binding_artifacts(), + ) + source["package_runtime_roots"] = {} + assert context.runtime_authentication_context["base_runtime_root"] == ( + ROOT / "runtime" / "base" + ) + with pytest.raises(TypeError): + context.runtime_authentication_context["new"] = ROOT # type: ignore[index] + + malformed = runtime_context() + malformed["package_import_paths"] = {"calibration": "../site-packages"} + with pytest.raises(ValueError, match="not canonical"): + AdapterConstructionContext(ROOT, ROOT, ROOT, ROOT, malformed, binding_artifacts()) + + +def test_adapter_facing_values_have_one_stable_importable_identity() -> None: + sequence = AuthenticatedSequence((1, 2), "a" * 64, "b" * 64, None, "c" * 64) + observation = StepObservation(0, 1, (0,), object(), None, (1,)) + file_identity = ModelFileIdentity( + "model.safetensors", + 1, + "d" * 64, + "e" * 40, + "d" * 64, + 1, + ) + authenticated = AuthenticatedModelFiles( + ROOT, + "example/model", + "f" * 40, + "5.14.1", + (file_identity,), + "1" * 64, + "2" * 64, + ) + + assert sequence.token_ids == (1, 2) + assert observation.successful_kernel_calls_per_layer == (1,) + assert authenticated.files == (file_identity,) + + +def test_api_import_is_stdlib_only_even_when_loaded_under_external_name() -> None: + code = f""" +import importlib.util +import sys +from pathlib import Path +path = Path({str(API_PATH)!r}) +spec = importlib.util.spec_from_file_location('external_experiment013_api', path) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +assert 'torch' not in sys.modules +assert 'transformers' not in sys.modules +assert 'datasets' not in sys.modules +assert module.AuthenticatedSequence.__module__ == 'external_experiment013_api' +""" + subprocess.run( + [str(ROOT / ".venv" / "Scripts" / "python.exe"), "-I", "-c", code], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + timeout=60, + ) + + +def test_runtime_protocol_accepts_structural_adapter() -> None: + class Adapter: + def materialize_sequence(self, record: object) -> object: + del record + return object() + + def load_model(self, authenticated: object) -> object: + del authenticated + return object() + + def begin_sequence(self, model: object, record: object) -> None: + del model, record + + def step_token(self, model: object, **kwargs: object) -> object: + del model, kwargs + return object() + + def end_sequence(self, model: object, record: object) -> None: + del model, record + + def close_model(self, model: object) -> None: + del model + + def runtime_metadata(self) -> dict[str, object]: + return {} + + assert isinstance(Adapter(), CalibrationAdapter) diff --git a/tests/test_experiment013_parquet.py b/tests/test_experiment013_parquet.py new file mode 100644 index 0000000..18d5f5b --- /dev/null +++ b/tests/test_experiment013_parquet.py @@ -0,0 +1,675 @@ +from __future__ import annotations + +import copy +import hashlib +import json +from collections import defaultdict +from dataclasses import replace +from pathlib import Path + +import pytest + +import recurquant.experiment013_parquet as parquet_module +from recurquant.experiment013_parquet import ( + EXPERIMENT013_PARQUET_MANIFEST_PATH, + EXPERIMENT013_PARQUET_MANIFEST_SCHEMA, + EXPERIMENT013_PARQUET_MANIFEST_SHA256, + EXPERIMENT013_PARQUET_MANIFEST_SIZE_BYTES, + Experiment013ParquetError, + Experiment013ParquetOffsetError, + HubDatasetMetadata, + HubFileMetadata, + ParquetFileLayout, + canonical_experiment013_parquet_manifest_bytes, + load_experiment013_parquet_manifest, + locate_experiment013_parquet_row, + project_experiment013_parquet_columns, + read_experiment013_parquet_row, + validate_experiment013_parquet_manifest, +) + + +def _raw_manifest() -> dict[str, object]: + payload = json.loads(EXPERIMENT013_PARQUET_MANIFEST_PATH.read_text(encoding="utf-8")) + assert isinstance(payload, dict) + return payload + + +class _FakeHubBackend: + def __init__(self, *, events: list[str] | None = None) -> None: + self.manifest = load_experiment013_parquet_manifest() + self.events = events if events is not None else [] + self.calls: list[tuple[object, ...]] = [] + self.revision_calls: defaultdict[str, int] = defaultdict(int) + self.snapshot_calls: defaultdict[str, int] = defaultdict(int) + self.revision_overrides: dict[tuple[str, int], str] = {} + self.snapshot_commit_overrides: dict[tuple[str, int], str] = {} + self.file_overrides: dict[tuple[str, int], dict[str, object]] = {} + self.reverse_snapshot: set[tuple[str, int]] = set() + + def resolve_dataset_revision(self, *, repo_id: str, revision: str) -> str: + call_index = self.revision_calls[revision] + self.revision_calls[revision] += 1 + self.calls.append(("resolve", repo_id, revision, call_index)) + self.events.append(f"hub-resolve-{call_index}") + return self.revision_overrides.get((revision, call_index), revision) + + def snapshot_parquet_files( + self, + *, + repo_id: str, + revision: str, + paths: tuple[str, ...], + ) -> HubDatasetMetadata: + call_index = self.snapshot_calls[revision] + self.snapshot_calls[revision] += 1 + self.calls.append(("snapshot", repo_id, revision, paths, call_index)) + self.events.append(f"hub-snapshot-{call_index}") + dataset = next( + dataset + for dataset in self.manifest.datasets + if dataset.dataset_id == repo_id and dataset.conversion_revision == revision + ) + expected_by_path = {file.immutable_path: file for file in dataset.files} + metadata: list[HubFileMetadata] = [] + for path in paths: + file = expected_by_path[path] + observed = HubFileMetadata( + path=file.immutable_path, + commit_hash=dataset.conversion_revision, + size_bytes=file.size_bytes, + git_blob_oid=file.git_blob_oid, + lfs_sha256=file.lfs_sha256, + lfs_size_bytes=file.lfs_size_bytes, + etag=file.lfs_sha256, + ) + override = self.file_overrides.get((path, call_index)) + if override: + observed = replace(observed, **override) + metadata.append(observed) + if (revision, call_index) in self.reverse_snapshot: + metadata.reverse() + return HubDatasetMetadata( + commit_hash=self.snapshot_commit_overrides.get( + (revision, call_index), dataset.conversion_revision + ), + files=tuple(metadata), + ) + + +class _FakeParquetBackend: + def __init__( + self, + layouts: dict[str, ParquetFileLayout], + *, + events: list[str] | None = None, + ) -> None: + self.layouts = layouts + self.events = events if events is not None else [] + self.inspect_calls: list[str] = [] + self.read_calls: list[tuple[str, int, int, tuple[str, ...]]] = [] + self.projection_calls: list[tuple[str, int, tuple[str, ...]]] = [] + self.projection_overreturn = False + self.projection_extra_column = False + + def inspect(self, uri: str) -> ParquetFileLayout: + self.inspect_calls.append(uri) + self.events.append("parquet-inspect") + return self.layouts[uri] + + def read_row( + self, + uri: str, + *, + row_group_index: int, + row_index_in_group: int, + columns: tuple[str, ...], + ) -> dict[str, object]: + self.read_calls.append((uri, row_group_index, row_index_in_group, columns)) + self.events.append("parquet-read") + return { + column: f"{column}:group={row_group_index}:row={row_index_in_group}" + for column in columns + } + + def read_row_group_projection( + self, + uri: str, + *, + row_group_index: int, + columns: tuple[str, ...], + ) -> tuple[dict[str, object], ...]: + self.projection_calls.append((uri, row_group_index, columns)) + self.events.append("parquet-project") + count = self.layouts[uri].row_group_rows[row_group_index] + if self.projection_overreturn: + count += 1 + rows = [] + for row_index in range(count): + row: dict[str, object] = { + column: f"{column}:{uri.rsplit('/', 1)[-1]}:{row_group_index}:{row_index}" + for column in columns + } + if self.projection_extra_column: + row["text"] = "must-not-be-returned" + rows.append(row) + return tuple(rows) + + +def _uri(dataset_id: str, revision: str, path: str) -> str: + return f"hf://datasets/{dataset_id}@{revision}/{path}" + + +def _parquet_backend( + dataset_key: str, + *, + layouts_by_path: dict[str, tuple[int, ...]] | None = None, + events: list[str] | None = None, +) -> _FakeParquetBackend: + manifest = load_experiment013_parquet_manifest() + dataset = manifest.dataset(dataset_key) + schema_columns = ( + ("url", "text") + if dataset_key == "pg19" + else ("prompt", "task_id", "canonical_solution") + ) + layouts: dict[str, ParquetFileLayout] = {} + for file in dataset.files: + row_groups = (1,) + if layouts_by_path is not None: + row_groups = layouts_by_path.get(file.immutable_path, row_groups) + layouts[_uri(dataset.dataset_id, dataset.conversion_revision, file.immutable_path)] = ( + ParquetFileLayout( + row_group_rows=row_groups, + columns=schema_columns, + ) + ) + return _FakeParquetBackend(layouts, events=events) + + +def test_checked_in_manifest_is_canonical_byte_bound_and_exact() -> None: + raw = EXPERIMENT013_PARQUET_MANIFEST_PATH.read_bytes() + manifest = load_experiment013_parquet_manifest() + parsed = _raw_manifest() + + assert len(raw) == EXPERIMENT013_PARQUET_MANIFEST_SIZE_BYTES + assert hashlib.sha256(raw).hexdigest() == EXPERIMENT013_PARQUET_MANIFEST_SHA256 + assert canonical_experiment013_parquet_manifest_bytes(parsed) == raw + assert manifest.schema == EXPERIMENT013_PARQUET_MANIFEST_SCHEMA + assert tuple(dataset.key for dataset in manifest.datasets) == ("humaneval_plus", "pg19") + assert manifest.dataset("humaneval_plus").dataset_id == "evalplus/humanevalplus" + assert manifest.dataset("pg19").dataset_id == "emozilla/pg19" + + +def test_loader_rejects_locally_changed_manifest_bytes(tmp_path: Path) -> None: + raw = bytearray(EXPERIMENT013_PARQUET_MANIFEST_PATH.read_bytes()) + marker = raw.index(b"humaneval_plus") + raw[marker] = ord("j") + changed = tmp_path / "materializations.json" + changed.write_bytes(raw) + + with pytest.raises(Experiment013ParquetError, match="SHA-256"): + load_experiment013_parquet_manifest(changed) + + +@pytest.mark.parametrize("level", ["top", "dataset", "file"]) +def test_validation_rejects_extra_fields_at_every_level(level: str) -> None: + payload = _raw_manifest() + if level == "top": + payload["mutable_revision"] = "main" + elif level == "dataset": + payload["datasets"]["pg19"]["endpoint"] = "viewer" # type: ignore[index] + else: + payload["datasets"]["pg19"]["files"][0]["url"] = "mutable" # type: ignore[index] + + with pytest.raises(Experiment013ParquetError, match="fields drifted"): + validate_experiment013_parquet_manifest(payload) + + +@pytest.mark.parametrize( + "case", + [ + "dataset_repo", + "source_alias", + "conversion_alias", + "split", + "file_path", + "file_order", + "size", + "git_hash", + "lfs_hash", + "lfs_size", + "partial_type", + "pending", + ], +) +def test_validation_rejects_every_frozen_identity_drift(case: str) -> None: + payload = copy.deepcopy(_raw_manifest()) + pg19 = payload["datasets"]["pg19"] # type: ignore[index] + files = pg19["files"] + if case == "dataset_repo": + pg19["dataset_id"] = "someone/pg19" + elif case == "source_alias": + pg19["source_revision"] = "main" + elif case == "conversion_alias": + pg19["conversion_revision"] = "refs/convert/parquet" + elif case == "split": + pg19["selected_splits"] = ["validation", "train"] + elif case == "file_path": + files[0]["immutable_path"] = "default/train/0000.parquet" + elif case == "file_order": + files[0], files[1] = files[1], files[0] + elif case == "size": + files[0]["size_bytes"] += 1 + elif case == "git_hash": + files[0]["git_blob_oid"] = "0" * 40 + elif case == "lfs_hash": + files[0]["lfs_sha256"] = "0" * 64 + elif case == "lfs_size": + files[0]["lfs_size_bytes"] += 1 + elif case == "partial_type": + pg19["partial"] = 1 + else: + pg19["pending"] = ["default/partial-train/0007.parquet"] + + with pytest.raises(Experiment013ParquetError): + validate_experiment013_parquet_manifest(payload) + + +def test_validation_rejects_malformed_manifest_shapes() -> None: + payload = _raw_manifest() + payload["datasets"]["pg19"]["files"] = "not-a-file-list" # type: ignore[index] + with pytest.raises(Experiment013ParquetError, match="files inventory"): + validate_experiment013_parquet_manifest(payload) + + with pytest.raises(Experiment013ParquetError, match="must be a mapping"): + validate_experiment013_parquet_manifest([]) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("offset", "file_index", "group_index", "row_in_group"), + [ + (0, 0, 0, 0), + (1, 0, 0, 1), + (2, 0, 1, 0), + (3, 1, 0, 0), + (4, 1, 0, 1), + (8, 5, 0, 0), + ], +) +def test_global_offset_maps_across_file_and_row_group_boundaries( + offset: int, + file_index: int, + group_index: int, + row_in_group: int, +) -> None: + manifest = load_experiment013_parquet_manifest() + dataset = manifest.dataset("pg19") + train_paths = [file.immutable_path for file in dataset.files if file.logical_split == "train"] + layouts = {path: (1,) for path in train_paths} + layouts[train_paths[0]] = (2, 1) + layouts[train_paths[1]] = (2,) + hub = _FakeHubBackend() + parquet = _parquet_backend("pg19", layouts_by_path=layouts) + + location = locate_experiment013_parquet_row( + "pg19", + "train", + offset, + hub_backend=hub, + parquet_backend=parquet, + ) + + assert location.split_row_count == 9 + assert location.split_file_index == file_index + assert location.row_group_index == group_index + assert location.row_index_in_group == row_in_group + + +@pytest.mark.parametrize("offset", [-1, 9, True]) +def test_global_offset_rejects_negative_upper_boundary_and_bool(offset: int) -> None: + manifest = load_experiment013_parquet_manifest() + dataset = manifest.dataset("pg19") + train_paths = [file.immutable_path for file in dataset.files if file.logical_split == "train"] + layouts = {path: (1,) for path in train_paths} + layouts[train_paths[0]] = (2, 1) + layouts[train_paths[1]] = (2,) + + with pytest.raises(Experiment013ParquetOffsetError): + locate_experiment013_parquet_row( + "pg19", + "train", + offset, + hub_backend=_FakeHubBackend(), + parquet_backend=_parquet_backend("pg19", layouts_by_path=layouts), + ) + + +def test_read_projects_one_row_group_and_uses_only_immutable_hf_uris() -> None: + events: list[str] = [] + manifest = load_experiment013_parquet_manifest() + dataset = manifest.dataset("humaneval_plus") + file = dataset.files[0] + hub = _FakeHubBackend(events=events) + parquet = _parquet_backend( + "humaneval_plus", + layouts_by_path={file.immutable_path: (2, 3)}, + events=events, + ) + + row = read_experiment013_parquet_row( + "humaneval_plus", + "test", + 3, + columns=("task_id", "prompt"), + hub_backend=hub, + parquet_backend=parquet, + ) + + expected_uri = _uri(dataset.dataset_id, dataset.conversion_revision, file.immutable_path) + assert row.location.immutable_uri == expected_uri + assert row.location.row_group_index == 1 + assert row.location.row_index_in_group == 1 + assert row.columns == ("task_id", "prompt") + assert tuple(row.values) == row.columns + assert parquet.read_calls == [(expected_uri, 1, 1, ("task_id", "prompt"))] + assert events == [ + "hub-resolve-0", + "hub-snapshot-0", + "parquet-inspect", + "parquet-read", + "hub-resolve-1", + "hub-snapshot-1", + ] + contacted = repr(hub.calls + parquet.inspect_calls + parquet.read_calls) + assert "/rows" not in contacted + assert "@~parquet" not in contacted + assert all(uri.startswith("hf://datasets/") for uri in parquet.inspect_calls) + + +def test_implementation_contains_no_dataset_viewer_or_mutable_parquet_endpoint() -> None: + source = Path(parquet_module.__file__).read_text(encoding="utf-8") + + assert "/rows" not in source + assert "@~parquet" not in source + + +def test_bulk_projection_is_ordered_counted_immutable_and_authenticated_once() -> None: + events: list[str] = [] + manifest = load_experiment013_parquet_manifest() + dataset = manifest.dataset("pg19") + train_files = tuple(file for file in dataset.files if file.logical_split == "train") + layouts = {file.immutable_path: (1,) for file in train_files} + layouts[train_files[0].immutable_path] = (2, 1) + hub = _FakeHubBackend(events=events) + parquet = _parquet_backend("pg19", layouts_by_path=layouts, events=events) + + projection = project_experiment013_parquet_columns( + "pg19", + "train", + columns=("url",), + expected_count=8, + hub_backend=hub, + parquet_backend=parquet, + ) + + assert projection.columns == ("url",) + assert len(projection.rows) == 8 + assert tuple(row.global_offset for row in projection.rows) == tuple(range(8)) + assert projection.rows[0].values[0].endswith("0000.parquet:0:0") + assert projection.rows[1].values[0].endswith("0000.parquet:0:1") + assert projection.rows[2].values[0].endswith("0000.parquet:1:0") + assert projection.rows[3].values[0].endswith("0001.parquet:0:0") + assert len(projection.canonical_projection_sha256) == 64 + assert hub.revision_calls[dataset.source_revision] == 2 + assert hub.snapshot_calls[dataset.conversion_revision] == 2 + assert len(parquet.inspect_calls) == 6 + assert len(parquet.projection_calls) == 7 + assert events[:2] == ["hub-resolve-0", "hub-snapshot-0"] + assert events[-2:] == ["hub-resolve-1", "hub-snapshot-1"] + assert all(call[2] == ("url",) for call in parquet.projection_calls) + assert "/rows" not in repr(parquet.projection_calls) + assert "@~parquet" not in repr(parquet.projection_calls) + + +@pytest.mark.parametrize( + ("dataset_key", "logical_split", "columns"), + [ + ("pg19", "train", ("text",)), + ("humaneval_plus", "test", ("prompt", "canonical_solution")), + ], +) +def test_bulk_projection_rejects_content_columns_before_external_access( + dataset_key: str, + logical_split: str, + columns: tuple[str, ...], +) -> None: + hub = _FakeHubBackend() + parquet = _parquet_backend(dataset_key) + + with pytest.raises(Experiment013ParquetError, match="canonical-ID"): + project_experiment013_parquet_columns( + dataset_key, + logical_split, + columns=columns, + hub_backend=hub, + parquet_backend=parquet, + ) + + assert hub.calls == [] + assert parquet.inspect_calls == [] + assert parquet.projection_calls == [] + + +def test_bulk_projection_rejects_schema_count_drift_before_reading_values() -> None: + manifest = load_experiment013_parquet_manifest() + file = manifest.dataset("humaneval_plus").files[0] + hub = _FakeHubBackend() + parquet = _parquet_backend( + "humaneval_plus", + layouts_by_path={file.immutable_path: (2, 3)}, + ) + + with pytest.raises(Experiment013ParquetError, match="population"): + project_experiment013_parquet_columns( + "humaneval_plus", + "test", + columns=("task_id",), + expected_count=4, + hub_backend=hub, + parquet_backend=parquet, + ) + assert parquet.projection_calls == [] + + with pytest.raises(Experiment013ParquetError, match="canonical-ID"): + project_experiment013_parquet_columns( + "humaneval_plus", + "test", + columns=("unknown_id",), + hub_backend=_FakeHubBackend(), + parquet_backend=parquet, + ) + + +@pytest.mark.parametrize("failure", ["overreturn", "extra_column"]) +def test_bulk_projection_rejects_backend_overreturn(failure: str) -> None: + parquet = _parquet_backend("humaneval_plus") + if failure == "overreturn": + parquet.projection_overreturn = True + message = "row count" + else: + parquet.projection_extra_column = True + message = "outside the projection" + + with pytest.raises(Experiment013ParquetError, match=message): + project_experiment013_parquet_columns( + "humaneval_plus", + "test", + columns=("task_id",), + expected_count=1, + hub_backend=_FakeHubBackend(), + parquet_backend=parquet, + ) + + +def test_bulk_projection_rejects_metadata_drift_after_projection() -> None: + manifest = load_experiment013_parquet_manifest() + dataset = manifest.dataset("humaneval_plus") + file = dataset.files[0] + hub = _FakeHubBackend() + hub.file_overrides[(file.immutable_path, 1)] = {"etag": "0" * 64} + + with pytest.raises(Experiment013ParquetError, match="after"): + project_experiment013_parquet_columns( + "humaneval_plus", + "test", + columns=("task_id",), + expected_count=1, + hub_backend=hub, + parquet_backend=_parquet_backend("humaneval_plus"), + ) + + +@pytest.mark.parametrize( + ("field", "wrong_value", "message"), + [ + ("path", "default/test/9999.parquet", "path"), + ("commit_hash", "0" * 40, "commit"), + ("size_bytes", 1, "size"), + ("git_blob_oid", "0" * 40, "git_blob_oid"), + ("lfs_sha256", "0" * 64, "lfs_sha256"), + ("lfs_size_bytes", 1, "lfs_size_bytes"), + ("etag", "0" * 64, "etag"), + ], +) +def test_point_of_use_metadata_rejects_wrong_file_identity_before_read( + field: str, + wrong_value: object, + message: str, +) -> None: + manifest = load_experiment013_parquet_manifest() + dataset = manifest.dataset("humaneval_plus") + file = dataset.files[0] + hub = _FakeHubBackend() + hub.file_overrides[(file.immutable_path, 0)] = {field: wrong_value} + parquet = _parquet_backend("humaneval_plus") + + with pytest.raises(Experiment013ParquetError, match=message): + read_experiment013_parquet_row( + "humaneval_plus", + "test", + 0, + hub_backend=hub, + parquet_backend=parquet, + ) + + assert parquet.inspect_calls == [] + assert parquet.read_calls == [] + + +def test_point_of_use_metadata_rejects_source_conversion_and_order_drift() -> None: + manifest = load_experiment013_parquet_manifest() + dataset = manifest.dataset("pg19") + + wrong_source = _FakeHubBackend() + wrong_source.revision_overrides[(dataset.source_revision, 0)] = "0" * 40 + with pytest.raises(Experiment013ParquetError, match="source commit"): + locate_experiment013_parquet_row( + "pg19", + "train", + 0, + hub_backend=wrong_source, + parquet_backend=_parquet_backend("pg19"), + ) + + wrong_conversion = _FakeHubBackend() + wrong_conversion.snapshot_commit_overrides[(dataset.conversion_revision, 0)] = "0" * 40 + with pytest.raises(Experiment013ParquetError, match="conversion commit"): + locate_experiment013_parquet_row( + "pg19", + "train", + 0, + hub_backend=wrong_conversion, + parquet_backend=_parquet_backend("pg19"), + ) + + wrong_order = _FakeHubBackend() + wrong_order.reverse_snapshot.add((dataset.conversion_revision, 0)) + with pytest.raises(Experiment013ParquetError, match="path"): + locate_experiment013_parquet_row( + "pg19", + "train", + 0, + hub_backend=wrong_order, + parquet_backend=_parquet_backend("pg19"), + ) + + +@pytest.mark.parametrize("drift", ["etag", "source", "conversion"]) +def test_metadata_drift_after_row_read_fails_closed(drift: str) -> None: + manifest = load_experiment013_parquet_manifest() + dataset = manifest.dataset("humaneval_plus") + file = dataset.files[0] + hub = _FakeHubBackend() + if drift == "etag": + hub.file_overrides[(file.immutable_path, 1)] = {"etag": "0" * 64} + elif drift == "source": + hub.revision_overrides[(dataset.source_revision, 1)] = "0" * 40 + else: + hub.snapshot_commit_overrides[(dataset.conversion_revision, 1)] = "0" * 40 + parquet = _parquet_backend("humaneval_plus") + + with pytest.raises(Experiment013ParquetError, match="after"): + read_experiment013_parquet_row( + "humaneval_plus", + "test", + 0, + columns=("prompt",), + hub_backend=hub, + parquet_backend=parquet, + ) + + assert len(parquet.read_calls) == 1 + + +def test_projection_rejects_unknown_duplicate_and_backend_extra_columns() -> None: + hub = _FakeHubBackend() + parquet = _parquet_backend("humaneval_plus") + with pytest.raises(Experiment013ParquetError, match="absent"): + read_experiment013_parquet_row( + "humaneval_plus", + "test", + 0, + columns=("missing",), + hub_backend=hub, + parquet_backend=parquet, + ) + + with pytest.raises(Experiment013ParquetError, match="unique"): + read_experiment013_parquet_row( + "humaneval_plus", + "test", + 0, + columns=("prompt", "prompt"), + hub_backend=_FakeHubBackend(), + parquet_backend=_parquet_backend("humaneval_plus"), + ) + + +def test_rejects_mutable_or_unknown_selection_at_api_boundary() -> None: + with pytest.raises(Experiment013ParquetError, match="unknown"): + locate_experiment013_parquet_row( + "pg19@main", + "train", + 0, + hub_backend=_FakeHubBackend(), + parquet_backend=_parquet_backend("pg19"), + ) + with pytest.raises(Experiment013ParquetError, match="not frozen"): + locate_experiment013_parquet_row( + "pg19", + "train@latest", + 0, + hub_backend=_FakeHubBackend(), + parquet_backend=_parquet_backend("pg19"), + ) diff --git a/tests/test_experiment013_qwen35_adapter.py b/tests/test_experiment013_qwen35_adapter.py new file mode 100644 index 0000000..00dd791 --- /dev/null +++ b/tests/test_experiment013_qwen35_adapter.py @@ -0,0 +1,1124 @@ +from __future__ import annotations + +import gc +import hashlib +import json +import sys +import types +import weakref +from collections.abc import Mapping +from pathlib import Path + +import pytest +import torch + +from recurquant import experiment013_qwen35_adapter as adapter_module +from recurquant.experiment013_calibration_api import ( + AdapterConstructionContext, + AuthenticatedModelFiles, + AuthenticatedSequence, + CalibrationAdapter, + ModelFileIdentity, +) + + +def _sha(value: int) -> str: + return f"{value:064x}" + + +def _context(tmp_path: Path) -> AdapterConstructionContext: + return AdapterConstructionContext( + repository_root=tmp_path / "repository-does-not-need-to-exist-at-construction", + model_root=tmp_path / "model-does-not-need-to-exist-at-construction", + cache_root=tmp_path / "cache-does-not-need-to-exist-at-construction", + ruler_root=tmp_path / "ruler-does-not-need-to-exist-at-construction", + runtime_authentication_context={ + "base_runtime_root": tmp_path / "runtime" / "base", + "staged_interpreter": tmp_path / "runtime" / "base" / "python.exe", + "package_runtime_roots": {"calibration": tmp_path / "runtime" / "packages"}, + "package_import_paths": {"calibration": "Lib/site-packages"}, + }, + execution_binding_artifacts={ + "repository_source_manifest_bytes": b"source-manifest\n", + "calibration_runtime_manifest_bytes": b"runtime-manifest\n", + "model_file_manifest_bytes": b"model-manifest\n", + "parquet_materialization_manifest_bytes": b"parquet-manifest\n", + }, + ) + + +def _fake_capture_module(**attributes: object) -> types.ModuleType: + module = types.ModuleType(adapter_module.CAPTURE_MODULE_NAME) + for name, value in attributes.items(): + setattr(module, name, value) + return module + + +def _fake_capture_binding( + tmp_path: Path, + module: types.ModuleType, + *, + payload: bytes = b"# authenticated test capture source\n", +) -> adapter_module._CaptureModuleBinding: + root = _context(tmp_path).repository_root + source_path = root / Path(adapter_module.CAPTURE_SOURCE_PATH) + source_path.parent.mkdir(parents=True, exist_ok=True) + source_path.write_bytes(payload) + module.__file__ = str(source_path) + return adapter_module._CaptureModuleBinding( + module=module, + repository_root=root, + source_path=source_path, + raw_sha256=hashlib.sha256(payload).hexdigest(), + ) + + +def _source_manifest_bytes(payload: bytes) -> bytes: + return json.dumps( + { + "schema": adapter_module.SOURCE_MANIFEST_SCHEMA, + "paths": [ + { + "path": adapter_module.CAPTURE_SOURCE_PATH, + "raw_sha256": hashlib.sha256(payload).hexdigest(), + } + ], + } + ).encode("utf-8") + + +def _identity_record(index: int, token_ids: tuple[int, ...]) -> dict[str, object]: + return { + "identity_record_sha256": _sha(index + 1), + "source_content_sha256": _sha(1_000 + index), + "formatted_content_sha256": _sha(2_000 + index), + "generator_receipt_sha256": None, + "tokenizer_manifest_sha256": _sha(3_000), + "sequence_length": len(token_ids), + } + + +class _FakeMaterializedSequence: + def __init__(self, record: dict[str, object], token_ids: tuple[int, ...]) -> None: + self.identity_record = dict(record) + self.identity_record_sha256 = record["identity_record_sha256"] + self.sequence_token_ids = token_ids + + +class _FakeMaterialization: + def __init__(self) -> None: + self.sequences = tuple( + _FakeMaterializedSequence( + _identity_record(index, (index, index + 1)), + (index, index + 1), + ) + for index in range(160) + ) + self.tokenizer_manifest_sha256 = _sha(3_000) + self.capture_input_sha256 = _sha(4_000) + self.token_sequence_manifest_sha256 = _sha(5_000) + self.private_source_text = "must-not-be-retained" + + +def test_factory_is_zero_io_and_uses_the_shared_contract( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setattr( + adapter_module, + "_load_capture_module", + lambda *_args: pytest.fail("factory touched capture source"), + ) + monkeypatch.setattr( + adapter_module, + "_load_transformers_runtime", + lambda: pytest.fail("factory imported the model runtime"), + ) + + adapter = adapter_module.create_adapter(_context(tmp_path)) + + assert isinstance(adapter, adapter_module.Experiment013Qwen35Adapter) + assert isinstance(adapter, CalibrationAdapter) + assert adapter.runtime_metadata()["materialization_attempted"] is False + assert adapter.runtime_metadata()["model_loaded"] is False + + +def test_canonical_materialization_runs_once_and_retains_only_tokens_and_hashes( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + calls: list[dict[str, bytes]] = [] + runtime_calls: list[Mapping[str, object]] = [] + source_references: list[weakref.ReferenceType[object]] = [] + materialization_references: list[weakref.ReferenceType[object]] = [] + + class FakeLiveCaptureSource: + def __init__(self, *, cache_dir: Path, ruler_receipt_dir: Path) -> None: + self.cache_dir = cache_dir + self.ruler_receipt_dir = ruler_receipt_dir + self.private_source_text = "must-not-be-retained" + source_references.append(weakref.ref(self)) + + def materialize( + *, + source: object, + execution_binding_artifacts: dict[str, bytes], + runtime_authentication_context: Mapping[str, object], + ): + assert isinstance(source, FakeLiveCaptureSource) + calls.append(dict(execution_binding_artifacts)) + runtime_calls.append(runtime_authentication_context) + result = _FakeMaterialization() + materialization_references.append(weakref.ref(result)) + return result + + capture_module = _fake_capture_module( + LiveCaptureSource=FakeLiveCaptureSource, + materialize_calibration_identity_sequences=materialize, + ) + capture_binding = _fake_capture_binding(tmp_path, capture_module) + monkeypatch.setattr( + adapter_module, + "_load_capture_module", + lambda _root, _manifest: capture_binding, + ) + adapter = adapter_module.create_adapter(_context(tmp_path)) + + first_record = _identity_record(0, (0, 1)) + first = adapter.materialize_sequence(first_record) + second = adapter.materialize_sequence(_identity_record(1, (1, 2))) + gc.collect() + + assert first == AuthenticatedSequence( + token_ids=(0, 1), + source_content_sha256=_sha(1_000), + formatted_content_sha256=_sha(2_000), + generator_receipt_sha256=None, + tokenizer_manifest_sha256=_sha(3_000), + ) + assert second.token_ids == (1, 2) + assert calls == [ + { + "repository_source_manifest_file_sha256": b"source-manifest\n", + "calibration_runtime_manifest_file_sha256": b"runtime-manifest\n", + "model_file_manifest_file_sha256": b"model-manifest\n", + "parquet_materialization_manifest_file_sha256": b"parquet-manifest\n", + } + ] + assert source_references[0]() is None + assert materialization_references[0]() is None + assert adapter._execution_binding_artifacts is None + assert adapter._runtime_authentication_context is None + assert runtime_calls[0]["staged_interpreter"] == (tmp_path / "runtime" / "base" / "python.exe") + assert set(adapter._materialized_sequences or {}) == {_sha(index + 1) for index in range(160)} + assert all( + isinstance(value, AuthenticatedSequence) + for value in (adapter._materialized_sequences or {}).values() + ) + assert "must-not-be-retained" not in repr(adapter.__dict__) + metadata = adapter.runtime_metadata() + assert metadata["materialized_sequence_count"] == 160 + assert metadata["capture_input_sha256"] == _sha(4_000) + assert metadata["token_sequence_manifest_sha256"] == _sha(5_000) + + +def test_materialization_rejects_frozen_commitment_mismatch( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + capture_module = _fake_capture_module( + LiveCaptureSource=lambda **_kwargs: object(), + materialize_calibration_identity_sequences=lambda **_kwargs: _FakeMaterialization(), + ) + capture_binding = _fake_capture_binding(tmp_path, capture_module) + monkeypatch.setattr( + adapter_module, + "_load_capture_module", + lambda _root, _manifest: capture_binding, + ) + adapter = adapter_module.create_adapter(_context(tmp_path)) + record = _identity_record(0, (0, 1)) + record["formatted_content_sha256"] = _sha(99_999) + + with pytest.raises(adapter_module.Experiment013AdapterError, match="formatted_content"): + adapter.materialize_sequence(record) + + +def test_capture_loader_executes_manifest_bound_bytes_without_leaving_a_module( + tmp_path: Path, +) -> None: + root = tmp_path / "repository" + path = root / Path(adapter_module.CAPTURE_SOURCE_PATH) + path.parent.mkdir(parents=True) + payload = b"AUTHENTICATED_VALUE = 17\n" + path.write_bytes(payload) + + binding = adapter_module._load_capture_module(root, _source_manifest_bytes(payload)) + + assert binding.module.AUTHENTICATED_VALUE == 17 + assert binding.raw_sha256 == hashlib.sha256(payload).hexdigest() + assert adapter_module.CAPTURE_MODULE_NAME not in sys.modules + adapter_module._verify_capture_binding(binding) + + +def test_capture_loader_rejects_every_preloaded_capture_module(tmp_path: Path) -> None: + root = tmp_path / "repository" + path = root / Path(adapter_module.CAPTURE_SOURCE_PATH) + path.parent.mkdir(parents=True) + payload = b"AUTHENTICATED_VALUE = 17\n" + path.write_bytes(payload) + preloaded = types.ModuleType(adapter_module.CAPTURE_MODULE_NAME) + preloaded.__file__ = str(path) + sys.modules[adapter_module.CAPTURE_MODULE_NAME] = preloaded + try: + with pytest.raises(adapter_module.Experiment013AdapterError, match="already loaded"): + adapter_module._load_capture_module(root, _source_manifest_bytes(payload)) + finally: + sys.modules.pop(adapter_module.CAPTURE_MODULE_NAME, None) + sys.modules[adapter_module.CAPTURE_MODULE_NAME] = None + try: + with pytest.raises(adapter_module.Experiment013AdapterError, match="already loaded"): + adapter_module._load_capture_module(root, _source_manifest_bytes(payload)) + finally: + sys.modules.pop(adapter_module.CAPTURE_MODULE_NAME, None) + + +def test_capture_loader_rechecks_source_after_execution(tmp_path: Path) -> None: + root = tmp_path / "repository" + path = root / Path(adapter_module.CAPTURE_SOURCE_PATH) + path.parent.mkdir(parents=True) + payload = ( + b"from pathlib import Path\n" + b"Path(__file__).write_bytes(b'tampered after authenticated execution')\n" + ) + path.write_bytes(payload) + + with pytest.raises( + adapter_module.Experiment013AdapterError, + match="differs from the repository source manifest", + ): + adapter_module._load_capture_module(root, _source_manifest_bytes(payload)) + + assert adapter_module.CAPTURE_MODULE_NAME not in sys.modules + + +def test_capture_loader_rejects_link_or_reparse_components( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + root = tmp_path / "repository" + path = root / Path(adapter_module.CAPTURE_SOURCE_PATH) + path.parent.mkdir(parents=True) + payload = b"AUTHENTICATED_VALUE = 17\n" + path.write_bytes(payload) + monkeypatch.setattr(adapter_module.stat, "S_ISLNK", lambda _mode: True) + + with pytest.raises(adapter_module.Experiment013AdapterError, match="link or reparse"): + adapter_module._load_capture_module(root, _source_manifest_bytes(payload)) + + +def test_materialization_rechecks_capture_source_after_call( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + capture_module = _fake_capture_module() + capture_binding = _fake_capture_binding(tmp_path, capture_module) + + class FakeLiveCaptureSource: + def __init__(self, **_kwargs: object) -> None: + pass + + def materialize(**_kwargs: object) -> _FakeMaterialization: + capture_binding.source_path.write_bytes(b"changed during materialization\n") + return _FakeMaterialization() + + capture_module.LiveCaptureSource = FakeLiveCaptureSource + capture_module.materialize_calibration_identity_sequences = materialize + monkeypatch.setattr( + adapter_module, + "_load_capture_module", + lambda _root, _manifest: capture_binding, + ) + adapter = adapter_module.create_adapter(_context(tmp_path)) + + with pytest.raises( + adapter_module.Experiment013AdapterError, + match="differs from the repository source manifest", + ): + adapter.materialize_sequence(_identity_record(0, (0, 1))) + + assert adapter._materialized_sequences is None + assert adapter._execution_binding_artifacts is None + assert adapter._runtime_authentication_context is None + + +class _FakeCacheLayer: + def __init__(self) -> None: + self.recurrent_states: dict[int, torch.Tensor | None] = {0: None} + + +class _FakeDynamicCache: + corrupt_layer: int | None = None + + def __init__(self, *, config: object) -> None: + del config + self.layers = [_FakeCacheLayer() for _index in range(24)] + self.sequence_length = 0 + + def get_seq_length(self) -> int: + return self.sequence_length + + def update_recurrent_state(self, state: torch.Tensor, layer_index: int) -> torch.Tensor: + cached = self.layers[layer_index].recurrent_states[0] + if cached is None: + cached = torch.empty_like(state) + self.layers[layer_index].recurrent_states[0] = cached + cached.copy_(state) + if layer_index == self.corrupt_layer: + cached.add_(1) + return cached + + +class Qwen3_5GatedDeltaNet: + def __init__(self, layer_idx: int) -> None: + self.layer_idx = layer_idx + self.layer_type = "linear_attention" + for name, value in adapter_module._GATED_DELTA_GEOMETRY.items(): + setattr(self, name, value) + self.calls: list[str] = [] + self.fail = False + self.double_call = False + self.bad_query = False + self.bad_query_dtype = False + self.bad_state = False + self.causal_conv1d_fn = None + self.causal_conv1d_update = lambda *args, **kwargs: None + self.chunk_gated_delta_rule = self._chunk + self.recurrent_gated_delta_rule = self._recurrent + + def _state(self, initial_state: torch.Tensor | None) -> torch.Tensor: + shape = (1, 16, 128, 127) if self.bad_state else adapter_module.STATE_SHAPE + if initial_state is None: + return torch.full( + shape, + float(self.layer_idx + 1), + dtype=torch.float32, + ) + return initial_state + 1.0 + + def _chunk( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + chunk_size: int = 64, + initial_state: torch.Tensor | None = None, + output_final_state: bool = False, + use_qk_l2norm_in_kernel: bool = False, + **kwargs: object, + ) -> tuple[torch.Tensor, torch.Tensor]: + del key, value, g, beta, chunk_size, output_final_state, use_qk_l2norm_in_kernel, kwargs + self.calls.append("chunk") + if self.fail: + raise RuntimeError("fake kernel failure") + return query, self._state(initial_state) + + def _recurrent( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + initial_state: torch.Tensor | None, + output_final_state: bool, + use_qk_l2norm_in_kernel: bool = False, + ) -> tuple[torch.Tensor, torch.Tensor]: + del key, value, g, beta, output_final_state, use_qk_l2norm_in_kernel + self.calls.append("recurrent") + if self.fail: + raise RuntimeError("fake kernel failure") + return query, self._state(initial_state) + + +class _FakeDecoderLayer: + def __init__(self, index: int) -> None: + if index in adapter_module.RECURRENT_LAYER_INDICES: + self.linear_attn = Qwen3_5GatedDeltaNet(index) + + +class _FakeTextModel: + def __init__(self) -> None: + self.layers = [_FakeDecoderLayer(index) for index in range(24)] + self.skip_layer: int | None = None + self.force_recurrent_at_zero = False + self.clone_initial_state = False + self.return_different_cache = False + self.sequence_increment = 1 + + def __call__( + self, + *, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + past_key_values: _FakeDynamicCache, + use_cache: bool, + ) -> object: + assert tuple(input_ids.shape) == (1, 1) + assert tuple(position_ids.shape) == (1, 1) + assert use_cache is True + assert torch.is_inference_mode_enabled() + position = int(position_ids.item()) + for layer_index in adapter_module.RECURRENT_LAYER_INDICES: + if layer_index == self.skip_layer: + continue + module = self.layers[layer_index].linear_attn + query_shape = (1, 1, 16, 127) if module.bad_query else adapter_module.QUERY_SHAPE + query_dtype = torch.float32 if module.bad_query_dtype else torch.bfloat16 + query = torch.full(query_shape, float(layer_index + position), dtype=query_dtype) + key = torch.zeros_like(query) + value = torch.zeros(adapter_module.QUERY_SHAPE, dtype=query_dtype) + g = torch.zeros((1, 1, 16), dtype=torch.float32) + beta = torch.zeros((1, 1, 16), dtype=torch.float32) + cached = past_key_values.layers[layer_index].recurrent_states[0] + kernel_initial = ( + cached.clone() + if self.clone_initial_state and isinstance(cached, torch.Tensor) + else cached + ) + if position == 0 and not self.force_recurrent_at_zero: + output = module.chunk_gated_delta_rule( + query, + key, + value, + g=g, + beta=beta, + initial_state=None, + output_final_state=True, + use_qk_l2norm_in_kernel=True, + ) + if module.double_call: + module.chunk_gated_delta_rule( + query, + key, + value, + g=g, + beta=beta, + initial_state=None, + output_final_state=True, + use_qk_l2norm_in_kernel=True, + ) + else: + output = module.recurrent_gated_delta_rule( + query, + key, + value, + g=g, + beta=beta, + initial_state=kernel_initial, + output_final_state=True, + use_qk_l2norm_in_kernel=True, + ) + past_key_values.update_recurrent_state(output[1], layer_index) + past_key_values.sequence_length += self.sequence_increment + returned_cache = object() if self.return_different_cache else past_key_values + return types.SimpleNamespace(past_key_values=returned_cache) + + +class _FakeLiveModel: + def __init__(self) -> None: + self.config = types.SimpleNamespace() + self.model = _FakeTextModel() + self.training = False + + +def _bind_fake_model( + adapter: adapter_module.Experiment013Qwen35Adapter, +) -> tuple[_FakeLiveModel, adapter_module._Qwen35StepObserver]: + model = _FakeLiveModel() + runtime = adapter_module._TransformersRuntime( + version=adapter_module.TRANSFORMERS_VERSION, + qwen_config_class=object, + qwen_model_class=object, + qwen_gated_delta_net_class=Qwen3_5GatedDeltaNet, + dynamic_cache_class=_FakeDynamicCache, + torch_chunk_gated_delta_rule=lambda *args, **kwargs: None, + torch_recurrent_gated_delta_rule=lambda *args, **kwargs: None, + torch_causal_conv1d_update=lambda *args, **kwargs: None, + ) + modules = adapter_module._qwen_modules(model, runtime) + observer = adapter_module._Qwen35StepObserver( + modules, + query_device=torch.device("cpu"), + _allow_test_non_cuda=True, + ) + observer.install() + adapter._runtime = runtime + adapter._model = model + adapter._model_device = torch.device("cpu") + adapter._observer = observer + return model, observer + + +def test_qwen_modules_require_exact_authenticated_class_and_geometry(tmp_path: Path) -> None: + adapter = adapter_module.create_adapter(_context(tmp_path)) + model, _observer = _bind_fake_model(adapter) + assert adapter._runtime is not None + authenticated_runtime = adapter._runtime + adapter.close_model(model) + + class SameNamedSubclass(Qwen3_5GatedDeltaNet): + pass + + impostor_model = _FakeLiveModel() + impostor_model.model.layers[0].linear_attn = SameNamedSubclass(0) + with pytest.raises(adapter_module.Experiment013AdapterError, match="pinned Gated DeltaNet"): + adapter_module._qwen_modules(impostor_model, authenticated_runtime) + + geometry_model = _FakeLiveModel() + geometry_model.model.layers[0].linear_attn.num_k_heads = 8 + runtime = adapter_module._TransformersRuntime( + version=adapter_module.TRANSFORMERS_VERSION, + qwen_config_class=object, + qwen_model_class=object, + qwen_gated_delta_net_class=Qwen3_5GatedDeltaNet, + dynamic_cache_class=_FakeDynamicCache, + torch_chunk_gated_delta_rule=lambda *args, **kwargs: None, + torch_recurrent_gated_delta_rule=lambda *args, **kwargs: None, + torch_causal_conv1d_update=lambda *args, **kwargs: None, + ) + with pytest.raises(adapter_module.Experiment013AdapterError, match="num_k_heads geometry"): + adapter_module._qwen_modules(geometry_model, runtime) + + +def test_production_observer_rejects_a_non_cuda_query_contract(tmp_path: Path) -> None: + adapter = adapter_module.create_adapter(_context(tmp_path)) + model, observer = _bind_fake_model(adapter) + modules = observer.modules + adapter.close_model(model) + + with pytest.raises(adapter_module.Experiment013AdapterError, match="must be CUDA"): + adapter_module._Qwen35StepObserver(modules, query_device=torch.device("cpu")) + + +def _sequence_record(length: int = 2) -> dict[str, object]: + return { + "identity_record_sha256": _sha(77), + "sequence_length": length, + } + + +def test_one_token_chunk_then_recurrent_and_anchor_only_state(tmp_path: Path) -> None: + adapter = adapter_module.create_adapter(_context(tmp_path)) + model, observer = _bind_fake_model(adapter) + record = _sequence_record() + adapter.begin_sequence(model, record) + + first = adapter.step_token(model, token_id=11, position=0, capture_state=False) + second = adapter.step_token(model, token_id=12, position=1, capture_state=True) + + assert first.recurrence_query.shape == (18, 16, 128) + assert first.recurrent_state is None + assert second.recurrence_query.shape == (18, 16, 128) + assert second.recurrent_state is not None + assert second.recurrent_state.shape == (18, 16, 128, 128) + assert second.recurrent_state.dtype == torch.float32 + assert first.layer_indices == adapter_module.RECURRENT_LAYER_INDICES + assert first.successful_kernel_calls_per_layer == (1,) * 18 + for layer_index in adapter_module.RECURRENT_LAYER_INDICES: + assert model.model.layers[layer_index].linear_attn.calls == ["chunk", "recurrent"] + assert observer.is_idle + adapter.end_sequence(model, record) + adapter.close_model(model) + + +@pytest.mark.parametrize( + "failure", + [ + "duplicate", + "missing", + "bad_query", + "bad_query_dtype", + "bad_state", + "wrong_kernel", + ], +) +def test_one_call_receipts_and_shapes_fail_closed(tmp_path: Path, failure: str) -> None: + adapter = adapter_module.create_adapter(_context(tmp_path)) + model, observer = _bind_fake_model(adapter) + if failure == "duplicate": + model.model.layers[0].linear_attn.double_call = True + elif failure == "missing": + model.model.skip_layer = 22 + elif failure == "bad_query": + model.model.layers[0].linear_attn.bad_query = True + elif failure == "bad_query_dtype": + model.model.layers[0].linear_attn.bad_query_dtype = True + elif failure == "bad_state": + model.model.layers[0].linear_attn.bad_state = True + else: + model.model.force_recurrent_at_zero = True + record = _sequence_record(length=1) + adapter.begin_sequence(model, record) + + with pytest.raises(adapter_module.Experiment013AdapterError): + adapter.step_token(model, token_id=1, position=0, capture_state=False) + + assert observer.is_idle + with pytest.raises(adapter_module.Experiment013AdapterError, match="healthy"): + adapter.step_token(model, token_id=1, position=0, capture_state=False) + adapter.end_sequence(model, record) + adapter.close_model(model) + + +def test_recurrent_step_requires_the_exact_cached_state_object(tmp_path: Path) -> None: + adapter = adapter_module.create_adapter(_context(tmp_path)) + model, observer = _bind_fake_model(adapter) + record = _sequence_record() + adapter.begin_sequence(model, record) + adapter.step_token(model, token_id=1, position=0, capture_state=False) + model.model.clone_initial_state = True + + with pytest.raises(adapter_module.Experiment013AdapterError, match="exact persistent state"): + adapter.step_token(model, token_id=2, position=1, capture_state=False) + + assert observer.is_idle + adapter.end_sequence(model, record) + adapter.close_model(model) + + +@pytest.mark.parametrize("failure", ["cache_identity", "sequence_length"]) +def test_dynamic_cache_identity_and_length_are_proven(tmp_path: Path, failure: str) -> None: + adapter = adapter_module.create_adapter(_context(tmp_path)) + model, observer = _bind_fake_model(adapter) + if failure == "cache_identity": + model.model.return_different_cache = True + expected = "different DynamicCache" + else: + model.model.sequence_increment = 2 + expected = "exactly one token" + record = _sequence_record(length=1) + adapter.begin_sequence(model, record) + + with pytest.raises(adapter_module.Experiment013AdapterError, match=expected): + adapter.step_token(model, token_id=1, position=0, capture_state=False) + + assert observer.is_idle + adapter.end_sequence(model, record) + adapter.close_model(model) + + +def test_kernel_exception_resets_context_and_invalidates_sequence(tmp_path: Path) -> None: + adapter = adapter_module.create_adapter(_context(tmp_path)) + model, observer = _bind_fake_model(adapter) + model.model.layers[0].linear_attn.fail = True + record = _sequence_record(length=1) + adapter.begin_sequence(model, record) + + with pytest.raises(RuntimeError, match="fake kernel failure"): + adapter.step_token(model, token_id=1, position=0, capture_state=False) + + assert observer.is_idle + assert adapter._sequence_failed is True + adapter.end_sequence(model, record) + adapter.close_model(model) + + +def test_failed_kernel_never_appends_a_receipt(tmp_path: Path) -> None: + adapter = adapter_module.create_adapter(_context(tmp_path)) + model, observer = _bind_fake_model(adapter) + module = model.model.layers[0].linear_attn + module.fail = True + cache = _FakeDynamicCache(config=model.config) + capture = adapter_module._StepCapture(cache=cache, position=0, receipts={}) + token = observer.activate(capture) + query = torch.zeros(adapter_module.QUERY_SHAPE, dtype=torch.bfloat16) + try: + with pytest.raises(RuntimeError, match="fake kernel failure"): + module.chunk_gated_delta_rule( + query, + query, + query, + g=torch.zeros((1, 1, 16)), + beta=torch.zeros((1, 1, 16)), + initial_state=None, + output_final_state=True, + use_qk_l2norm_in_kernel=True, + ) + assert capture.receipts == {} + finally: + observer.deactivate(token) + adapter.close_model(model) + + +def test_post_forward_cache_equality_is_required(tmp_path: Path) -> None: + adapter = adapter_module.create_adapter(_context(tmp_path)) + model, observer = _bind_fake_model(adapter) + _FakeDynamicCache.corrupt_layer = 5 + record = _sequence_record(length=1) + adapter.begin_sequence(model, record) + try: + with pytest.raises(adapter_module.Experiment013AdapterError, match="state differs"): + adapter.step_token(model, token_id=1, position=0, capture_state=True) + assert observer.is_idle + finally: + _FakeDynamicCache.corrupt_layer = None + adapter.end_sequence(model, record) + adapter.close_model(model) + + +class _FakeDeviceValue: + def __init__(self) -> None: + self.device = torch.device("cpu") + self.requires_grad = True + self.dtype = torch.bfloat16 + + @staticmethod + def is_floating_point() -> bool: + return True + + +class _LoaderModel: + def __init__(self, config: object, events: list[tuple[str, object]]) -> None: + self.config = config + self.model = types.SimpleNamespace(layers=[_FakeDecoderLayer(index) for index in range(24)]) + self.training = True + self._parameter = _FakeDeviceValue() + self._events = events + + def to(self, device: torch.device) -> _LoaderModel: + self._events.append(("to", device)) + self._parameter.device = device + return self + + def eval(self) -> _LoaderModel: + self._events.append(("eval", None)) + self.training = False + return self + + def requires_grad_(self, enabled: bool) -> _LoaderModel: + self._events.append(("requires_grad", enabled)) + self._parameter.requires_grad = enabled + return self + + def parameters(self): + return iter((self._parameter,)) + + def buffers(self): + return iter(()) + + +def _qwen_config() -> object: + return types.SimpleNamespace( + model_type="qwen3_5_text", + hidden_size=1024, + num_hidden_layers=24, + linear_num_key_heads=16, + linear_num_value_heads=16, + linear_key_head_dim=128, + linear_value_head_dim=128, + linear_conv_kernel_dim=4, + layer_types=list(adapter_module.LAYER_TYPES), + _attn_implementation="eager", + ) + + +def _model_identity(model_root: Path) -> AuthenticatedModelFiles: + names = sorted( + ( + "config.json", + "model.safetensors-00001-of-00001.safetensors", + "model.safetensors.index.json", + ) + ) + files = tuple( + ModelFileIdentity( + name=name, + size_bytes=1, + sha256=_sha(index + 10) if name.endswith(".safetensors") else None, + git_blob_oid="1" * 40, + lfs_sha256=_sha(index + 10) if name.endswith(".safetensors") else None, + lfs_size_bytes=1 if name.endswith(".safetensors") else None, + ) + for index, name in enumerate(names) + ) + return AuthenticatedModelFiles( + model_root=model_root, + model_id=adapter_module.MODEL_ID, + revision=adapter_module.MODEL_REVISION, + transformers_version=adapter_module.TRANSFORMERS_VERSION, + files=files, + hub_tree_manifest_sha256=_sha(88), + manifest_file_sha256=_sha(89), + ) + + +def _loader_adapter( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + result_factory: object, +) -> tuple[ + adapter_module.Experiment013Qwen35Adapter, + AuthenticatedModelFiles, + list[tuple[str, object]], +]: + model_root = tmp_path / "model" + model_root.mkdir() + context = _context(tmp_path) + context = AdapterConstructionContext( + repository_root=context.repository_root, + model_root=model_root, + cache_root=context.cache_root, + ruler_root=context.ruler_root, + runtime_authentication_context=context.runtime_authentication_context, + execution_binding_artifacts=context.execution_binding_artifacts, + ) + events: list[tuple[str, object]] = [] + config = _qwen_config() + + class FakeConfigClass: + @classmethod + def from_pretrained(cls, path: str, **kwargs: object) -> object: + events.append(("config", (path, kwargs))) + return config + + class FakeModelClass: + @classmethod + def from_pretrained(cls, path: str, **kwargs: object) -> object: + events.append(("model", (path, kwargs))) + model = _LoaderModel(config, events) + if not callable(result_factory): + raise AssertionError("result_factory must be callable") + return result_factory(model) + + runtime = adapter_module._TransformersRuntime( + version=adapter_module.TRANSFORMERS_VERSION, + qwen_config_class=FakeConfigClass, + qwen_model_class=FakeModelClass, + qwen_gated_delta_net_class=Qwen3_5GatedDeltaNet, + dynamic_cache_class=_FakeDynamicCache, + torch_chunk_gated_delta_rule=lambda *args, **kwargs: None, + torch_recurrent_gated_delta_rule=lambda *args, **kwargs: None, + torch_causal_conv1d_update=lambda *args, **kwargs: None, + ) + monkeypatch.setattr(adapter_module, "_load_transformers_runtime", lambda: runtime) + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "current_device", lambda: 0) + return ( + adapter_module.create_adapter(context), + _model_identity(model_root), + events, + ) + + +@pytest.mark.parametrize( + ("diagnostic_name", "diagnostic_value"), + [ + ("missing_keys", {"model.layers.0.weight"}), + ("unexpected_keys", {"unknown.weight"}), + ("mismatched_keys", {("model.weight", (1,), (2,))}), + ("error_msgs", ["checkpoint load failed"]), + ], +) +def test_loader_rejects_each_non_empty_transformers_diagnostic_before_cuda_transfer( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + diagnostic_name: str, + diagnostic_value: object, +) -> None: + def result(model: object) -> object: + diagnostics: dict[str, object] = { + "missing_keys": set(), + "unexpected_keys": set(), + "mismatched_keys": set(), + "error_msgs": [], + } + diagnostics[diagnostic_name] = diagnostic_value + return model, diagnostics + + adapter, identity, events = _loader_adapter(monkeypatch, tmp_path, result) + + with pytest.raises(adapter_module.Experiment013AdapterError, match=diagnostic_name): + adapter.load_model(identity) + + assert [name for name, _value in events] == ["config", "model"] + assert adapter.runtime_metadata()["model_loaded"] is False + assert adapter.runtime_metadata()["model_loading_diagnostic_counts"] is None + + +@pytest.mark.parametrize( + "result_factory", + [ + lambda model: model, + lambda model: [model, {}], + lambda model: (model, {}, None), + ], +) +def test_loader_requires_exact_transformers_model_loading_tuple( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + result_factory: object, +) -> None: + adapter, identity, events = _loader_adapter(monkeypatch, tmp_path, result_factory) + + with pytest.raises(adapter_module.Experiment013AdapterError, match="exactly"): + adapter.load_model(identity) + + assert [name for name, _value in events] == ["config", "model"] + + +@pytest.mark.parametrize( + "loading_info", + [ + { + "missing_keys": [], + "unexpected_keys": set(), + "mismatched_keys": set(), + "error_msgs": [], + }, + { + "missing_keys": set(), + "unexpected_keys": set(), + "mismatched_keys": set(), + }, + { + "missing_keys": set(), + "unexpected_keys": set(), + "mismatched_keys": set(), + "error_msgs": [], + "conversion_errors": {}, + }, + ], +) +def test_loader_requires_exact_transformers_loading_diagnostic_schema( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + loading_info: dict[str, object], +) -> None: + adapter, identity, events = _loader_adapter( + monkeypatch, + tmp_path, + lambda model: (model, loading_info), + ) + + with pytest.raises(adapter_module.Experiment013AdapterError, match="diagnostic"): + adapter.load_model(identity) + + assert [name for name, _value in events] == ["config", "model"] + + +def test_loader_is_local_bf16_safetensors_eager_and_freezes_backends( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + model_root = tmp_path / "model" + model_root.mkdir() + context = _context(tmp_path) + context = AdapterConstructionContext( + repository_root=context.repository_root, + model_root=model_root, + cache_root=context.cache_root, + ruler_root=context.ruler_root, + runtime_authentication_context=context.runtime_authentication_context, + execution_binding_artifacts=context.execution_binding_artifacts, + ) + events: list[tuple[str, object]] = [] + config = _qwen_config() + + class FakeConfigClass: + @classmethod + def from_pretrained(cls, path: str, **kwargs: object) -> object: + events.append(("config", (path, kwargs))) + return config + + class FakeModelClass: + @classmethod + def from_pretrained(cls, path: str, **kwargs: object) -> object: + events.append(("model", (path, kwargs))) + return ( + _LoaderModel(config, events), + { + "missing_keys": set(), + "unexpected_keys": set(), + "mismatched_keys": set(), + "error_msgs": [], + }, + ) + + def torch_chunk(*args: object, **kwargs: object): + del args, kwargs + + def torch_recurrent(*args: object, **kwargs: object): + del args, kwargs + + def torch_conv(*args: object, **kwargs: object): + del args, kwargs + + runtime = adapter_module._TransformersRuntime( + version=adapter_module.TRANSFORMERS_VERSION, + qwen_config_class=FakeConfigClass, + qwen_model_class=FakeModelClass, + qwen_gated_delta_net_class=Qwen3_5GatedDeltaNet, + dynamic_cache_class=_FakeDynamicCache, + torch_chunk_gated_delta_rule=torch_chunk, + torch_recurrent_gated_delta_rule=torch_recurrent, + torch_causal_conv1d_update=torch_conv, + ) + monkeypatch.setattr(adapter_module, "_load_transformers_runtime", lambda: runtime) + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "current_device", lambda: 0) + adapter = adapter_module.create_adapter(context) + + model = adapter.load_model(_model_identity(model_root)) + + assert [name for name, _value in events] == [ + "config", + "model", + "to", + "eval", + "requires_grad", + ] + config_call = events[0][1] + assert isinstance(config_call, tuple) + assert config_call[1] == {"local_files_only": True, "trust_remote_code": False} + model_call = events[1][1] + assert isinstance(model_call, tuple) + assert model_call[1] == { + "config": config, + "dtype": torch.bfloat16, + "attn_implementation": "eager", + "low_cpu_mem_usage": True, + "use_safetensors": True, + "weights_only": True, + "local_files_only": True, + "trust_remote_code": False, + "output_loading_info": True, + } + for layer_index in adapter_module.RECURRENT_LAYER_INDICES: + module = model.model.layers[layer_index].linear_attn + assert module.causal_conv1d_fn is None + assert module.causal_conv1d_update is torch_conv + # The state functions are wrapped after the exact fallbacks are frozen. + assert module.chunk_gated_delta_rule is not torch_chunk + assert module.recurrent_gated_delta_rule is not torch_recurrent + assert adapter.runtime_metadata()["model_loaded"] is True + assert adapter.runtime_metadata()["device"] == "cuda:0" + assert adapter.runtime_metadata()["kernel_backend"] == ( + "transformers_pure_torch_gated_delta_rule" + ) + assert adapter.runtime_metadata()["model_loading_diagnostic_counts"] == { + "missing_keys": 0, + "unexpected_keys": 0, + "mismatched_keys": 0, + "error_msgs": 0, + } + adapter.close_model(model) + assert adapter.runtime_metadata()["model_loaded"] is False + + +def test_source_binding_hashes_the_adapter_own_bytes() -> None: + binding = adapter_module.Experiment013Qwen35Adapter.source_binding() + expected = hashlib.sha256(Path(adapter_module.__file__).read_bytes()).hexdigest() + + assert binding == { + "path": "src/recurquant/experiment013_qwen35_adapter.py", + "raw_sha256": expected, + } diff --git a/tests/test_experiment013_source.py b/tests/test_experiment013_source.py new file mode 100644 index 0000000..67a35fb --- /dev/null +++ b/tests/test_experiment013_source.py @@ -0,0 +1,315 @@ +from __future__ import annotations + +import copy +import json +import os +import subprocess +import sys +import types +from pathlib import Path + +import pytest + +from recurquant.experiment013_source import ( + EXPERIMENT013_SOURCE_PATHS, + Experiment013SourceError, + canonical_experiment013_source_manifest_bytes, + canonical_experiment013_source_manifest_sha256, + capture_experiment013_source_manifest, + validate_experiment013_source_manifest, + verify_experiment013_source_manifest, + verify_loaded_experiment013_recurquant_modules, +) + + +def _git(root: Path, *arguments: str, env: dict[str, str] | None = None) -> str: + environment = os.environ.copy() + if env: + environment.update(env) + process = subprocess.run( + ["git", *arguments], + cwd=root, + check=False, + capture_output=True, + text=True, + env=environment, + ) + if process.returncode != 0: + raise AssertionError(process.stderr or process.stdout) + return process.stdout.strip() + + +def _write(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8", newline="\n") + + +def _init_repository(root: Path, *, marker: str = "fixture") -> Path: + root.mkdir(parents=True) + _git(root, "init", "-b", "main") + _git(root, "config", "user.name", "Experiment 013 Test") + _git(root, "config", "user.email", "experiment013@example.invalid") + _write(root / ".gitattributes", "* text eol=lf\n") + _write(root / ".gitignore", "artifacts/\n") + for relative in EXPERIMENT013_SOURCE_PATHS: + _write(root / relative, f"{marker}:{relative}\n") + _git(root, "add", "--", ".") + _git(root, "commit", "-m", "fixture source") + return root + + +def _rehash(manifest: dict[str, object]) -> dict[str, object]: + payload = copy.deepcopy(manifest) + payload.pop("canonical_manifest_sha256") + payload["canonical_manifest_sha256"] = canonical_experiment013_source_manifest_sha256(payload) + return payload + + +def test_frozen_inventory_covers_all_experiment013_surfaces_without_hash_constants() -> None: + required = { + "research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md", + "research/experiment013-parquet-materializations.json", + "requirements/experiment013-calibration.txt", + "scripts/capture_static_q468_identity_input.py", + "scripts/generate_static_q468_ruler_receipts.py", + "scripts/launch_static_q468_calibration.py", + "scripts/resolve_static_q468_identity.py", + "scripts/run_static_q468_calibration.py", + "requirements/experiment013-ruler.txt", + "src/recurquant/static_q468.py", + "src/recurquant/static_q468_cache.py", + "src/recurquant/static_q468_calibration.py", + "src/recurquant/experiment013_calibration_api.py", + "src/recurquant/experiment013_parquet.py", + "src/recurquant/experiment013_qwen35_adapter.py", + "src/recurquant/experiment013_source.py", + "tests/test_capture_static_q468_identity_input.py", + "tests/test_generate_static_q468_ruler_receipts.py", + "tests/test_launch_static_q468_calibration.py", + "tests/test_resolve_static_q468_identity.py", + "tests/test_run_static_q468_calibration.py", + "tests/test_static_q468.py", + "tests/test_static_q468_cache.py", + "tests/test_static_q468_calibration.py", + "tests/test_experiment013_calibration_api.py", + "tests/test_experiment013_parquet.py", + "tests/test_experiment013_qwen35_adapter.py", + "tests/test_experiment013_source.py", + } + + assert required <= set(EXPERIMENT013_SOURCE_PATHS) + assert tuple(sorted(EXPERIMENT013_SOURCE_PATHS)) == EXPERIMENT013_SOURCE_PATHS + assert len(set(EXPERIMENT013_SOURCE_PATHS)) == len(EXPERIMENT013_SOURCE_PATHS) + source = Path(__file__).resolve().parents[1] / "src" / "recurquant" / "experiment013_source.py" + assert 'canonical_manifest_sha256 = "' not in source.read_text(encoding="utf-8") + + +def test_capture_is_portable_complete_and_allows_ignored_artifacts(tmp_path: Path) -> None: + root = _init_repository(tmp_path / "repository") + _write(root / "artifacts" / "ignored-result.json", "{}\n") + + manifest = capture_experiment013_source_manifest(root) + + assert validate_experiment013_source_manifest(manifest) == manifest + assert manifest["source_commit"] == _git(root, "rev-parse", "HEAD") + assert [entry["path"] for entry in manifest["paths"]] == list( # type: ignore[index] + EXPERIMENT013_SOURCE_PATHS + ) + for entry in manifest["paths"]: # type: ignore[assignment] + assert entry["git_blob_oid"] == entry["index_blob_oid"] + assert entry["git_blob_oid"] == entry["worktree_blob_oid"] + assert len(entry["raw_sha256"]) == 64 + serialized = json.dumps(manifest, sort_keys=True) + assert str(root) not in serialized + assert str(root.resolve()) not in serialized + assert manifest["repository_binding"]["ignored_artifacts_permitted"] is True # type: ignore[index] + + payload = canonical_experiment013_source_manifest_bytes(manifest) + assert payload.endswith(b"\n") + assert json.loads(payload) == manifest + assert payload == canonical_experiment013_source_manifest_bytes(manifest) + + +@pytest.mark.parametrize("state", ["staged", "unstaged", "untracked"]) +def test_capture_rejects_non_clean_nonignored_state(tmp_path: Path, state: str) -> None: + root = _init_repository(tmp_path / "repository") + target = root / EXPERIMENT013_SOURCE_PATHS[0] + if state == "untracked": + _write(root / "unexpected.txt", "not ignored\n") + else: + target.write_text("changed\n", encoding="utf-8", newline="\n") + if state == "staged": + _git(root, "add", "--", EXPERIMENT013_SOURCE_PATHS[0]) + + with pytest.raises(Experiment013SourceError, match="changes|untracked|status"): + capture_experiment013_source_manifest(root) + + +@pytest.mark.parametrize("flag", ["--skip-worktree", "--assume-unchanged"]) +def test_capture_rejects_hidden_index_flags(tmp_path: Path, flag: str) -> None: + root = _init_repository(tmp_path / "repository") + _git(root, "update-index", flag, "--", EXPERIMENT013_SOURCE_PATHS[0]) + + with pytest.raises(Experiment013SourceError, match="skip-worktree|assume-unchanged"): + capture_experiment013_source_manifest(root) + + +def test_capture_scrubs_inherited_git_index_redirection( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = _init_repository(tmp_path / "repository") + decoy = tmp_path / "decoy-index" + decoy.write_bytes(b"not a Git index") + monkeypatch.setenv("GIT_INDEX_FILE", str(decoy)) + monkeypatch.setenv("GIT_WORK_TREE", str(tmp_path / "elsewhere")) + + manifest = capture_experiment013_source_manifest(root) + + assert manifest["source_commit"] == _git(root, "rev-parse", "HEAD") + + +def test_capture_rejects_unsafe_local_git_config(tmp_path: Path) -> None: + root = _init_repository(tmp_path / "repository") + _git(root, "config", "core.fsmonitor", "true") + + with pytest.raises(Experiment013SourceError, match="unsafe local Git config"): + capture_experiment013_source_manifest(root) + + +def test_validation_rejects_non_string_field_names(tmp_path: Path) -> None: + root = _init_repository(tmp_path / "repository") + manifest = capture_experiment013_source_manifest(root) + malformed = dict(manifest) + malformed[1] = "not a field name" + + with pytest.raises(Experiment013SourceError, match="field names must be strings"): + validate_experiment013_source_manifest(malformed) + + +def test_linked_worktree_gitdir_common_store_and_index_are_authenticated(tmp_path: Path) -> None: + primary = _init_repository(tmp_path / "primary") + linked = tmp_path / "linked" + _git(primary, "worktree", "add", "-b", "linked-source", str(linked)) + + manifest = capture_experiment013_source_manifest(linked) + + assert manifest["repository_binding"]["worktree_layout"] == "linked" # type: ignore[index] + assert verify_experiment013_source_manifest(manifest, linked) == manifest + + +def test_verify_accepts_unchanged_inventory_on_descendant_commit(tmp_path: Path) -> None: + root = _init_repository(tmp_path / "repository") + manifest = capture_experiment013_source_manifest(root) + _write(root / "notes.md", "unrelated tracked descendant\n") + _git(root, "add", "--", "notes.md") + _git(root, "commit", "-m", "unrelated descendant") + + verified = verify_experiment013_source_manifest(manifest, root) + + assert verified == manifest + assert verified["source_commit"] != _git(root, "rev-parse", "HEAD") + + +def test_verify_rejects_changed_inventory_even_on_descendant_commit(tmp_path: Path) -> None: + root = _init_repository(tmp_path / "repository") + manifest = capture_experiment013_source_manifest(root) + target = EXPERIMENT013_SOURCE_PATHS[0] + _write(root / target, "changed in descendant\n") + _git(root, "add", "--", target) + _git(root, "commit", "-m", "change frozen source") + + with pytest.raises(Experiment013SourceError, match="differ|drift"): + verify_experiment013_source_manifest(manifest, root) + + +@pytest.mark.parametrize( + "protected_path", + ["src/recurquant/experiment013_source.py", "tests/test_experiment013_source.py"], +) +def test_verify_rejects_descendant_changes_to_verifier_or_its_tests( + tmp_path: Path, protected_path: str +) -> None: + root = _init_repository(tmp_path / "repository") + manifest = capture_experiment013_source_manifest(root) + _write(root / protected_path, "replaced verifier surface\n") + _git(root, "add", "--", protected_path) + _git(root, "commit", "-m", "replace source verifier surface") + + with pytest.raises(Experiment013SourceError, match="differ|drift"): + verify_experiment013_source_manifest(manifest, root) + + +def test_verify_rejects_unavailable_or_unrelated_source_commit(tmp_path: Path) -> None: + first = _init_repository(tmp_path / "first", marker="first") + second = _init_repository(tmp_path / "second", marker="second") + manifest = capture_experiment013_source_manifest(first) + + with pytest.raises(Experiment013SourceError, match="unavailable|ancestor"): + verify_experiment013_source_manifest(manifest, second) + + +def test_validation_rejects_absolute_path_even_with_recomputed_manifest_hash( + tmp_path: Path, +) -> None: + root = _init_repository(tmp_path / "repository") + manifest = capture_experiment013_source_manifest(root) + manifest["paths"][0]["path"] = "C:/private/source.py" # type: ignore[index] + tampered = _rehash(manifest) + + with pytest.raises(Experiment013SourceError, match="repository-relative|inventory"): + validate_experiment013_source_manifest(tampered) + + +def test_verify_rejects_rehashed_raw_content_identity_tampering(tmp_path: Path) -> None: + root = _init_repository(tmp_path / "repository") + manifest = capture_experiment013_source_manifest(root) + manifest["paths"][0]["raw_sha256"] = "0" * 64 # type: ignore[index] + tampered = _rehash(manifest) + assert validate_experiment013_source_manifest(tampered) == tampered + + with pytest.raises(Experiment013SourceError, match="differs"): + verify_experiment013_source_manifest(tampered, root) + + +def test_validation_rejects_unknown_fields_and_canonical_hash_drift(tmp_path: Path) -> None: + root = _init_repository(tmp_path / "repository") + manifest = capture_experiment013_source_manifest(root) + extra = {**manifest, "absolute_root": str(root)} + with pytest.raises(Experiment013SourceError, match="fields drifted"): + validate_experiment013_source_manifest(extra) + + drifted = copy.deepcopy(manifest) + drifted["canonical_manifest_sha256"] = "0" * 64 + with pytest.raises(Experiment013SourceError, match="canonical source manifest"): + validate_experiment013_source_manifest(drifted) + + +def test_loaded_module_helper_binds_canonical_local_source_bytes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = _init_repository(tmp_path / "repository") + manifest = capture_experiment013_source_manifest(root) + module_name = "recurquant.static_q468" + module = types.ModuleType(module_name) + module.__file__ = str(root / "src" / "recurquant" / "static_q468.py") + monkeypatch.setitem(sys.modules, module_name, module) + + observed = verify_loaded_experiment013_recurquant_modules(manifest, root, [module_name]) + + assert observed == {module_name: "src/recurquant/static_q468.py"} + + +def test_loaded_module_helper_rejects_outside_or_undeclared_source( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = _init_repository(tmp_path / "repository") + manifest = capture_experiment013_source_manifest(root) + outside = tmp_path / "outside.py" + _write(outside, "pass\n") + module_name = "recurquant.static_q468" + module = types.ModuleType(module_name) + module.__file__ = str(outside) + monkeypatch.setitem(sys.modules, module_name, module) + + with pytest.raises(Experiment013SourceError, match="outside"): + verify_loaded_experiment013_recurquant_modules(manifest, root, [module_name]) diff --git a/tests/test_generate_static_q468_ruler_receipts.py b/tests/test_generate_static_q468_ruler_receipts.py index 52d869a..3d45576 100644 --- a/tests/test_generate_static_q468_ruler_receipts.py +++ b/tests/test_generate_static_q468_ruler_receipts.py @@ -22,6 +22,30 @@ def tokenize(text: str) -> list[str]: return text.split() +def _minimal_recorded_package_root(root: Path, *, pth_payload: str = "") -> Path: + root.mkdir(parents=True) + extra_paths = ("fixture.py", "hostile.pth") + (root / "fixture.py").write_text("VALUE = 'package-bound'\n", encoding="utf-8") + (root / "hostile.pth").write_text(pth_payload, encoding="utf-8") + first = True + for name, version in ruler.RUNTIME_PACKAGES.items(): + stem = ruler._canonical_distribution_name(name).replace("-", "_") + info_name = f"{stem}-{version}.dist-info" + info = root / info_name + info.mkdir() + metadata_path = info / "METADATA" + metadata_path.write_text(f"Name: {name}\nVersion: {version}\n", encoding="utf-8") + record_paths = [f"{info_name}/METADATA", f"{info_name}/RECORD"] + if first: + record_paths.extend(extra_paths) + first = False + (info / "RECORD").write_text( + "".join(f"{path},,\n" for path in record_paths), + encoding="utf-8", + ) + return root + + def _valid_niah_row() -> dict[str, object]: answers = ["8028695", "4670027", "8310696", "7938875"] input_text = ( @@ -55,6 +79,29 @@ def test_task_specs_cover_the_exact_required_receipt_configs() -> None: } +def test_capture_freezes_every_exact_launcher_command_manifest() -> None: + capture = ruler._load_capture_module() + launcher_entry = ruler._launcher_source_entry() + observed: dict[str, str] = {} + for receipt in capture.required_ruler_receipts(): + _actual, portable = ruler.generator_argv( + python=Path("python.exe"), + package_root=Path("site-packages"), + staged_root=Path("staged"), + raw_root=Path("raw"), + receipt=receipt, + ) + command = ruler._command_manifest( + receipt=receipt, + portable_argv=portable, + capture=capture, + static_entries=[launcher_entry], + ) + observed[receipt["filename"]] = ruler._sha256_bytes(ruler._canonical_json_bytes(command)) + + assert observed == capture.RULER_COMMAND_MANIFEST_SHA256_BY_FILENAME + + def test_generator_argv_preserves_multiline_template_as_one_argument(tmp_path) -> None: receipt = { "config": "niah_multiquery", @@ -64,7 +111,8 @@ def test_generator_argv_preserves_multiline_template_as_one_argument(tmp_path) - actual, portable = ruler.generator_argv( python=tmp_path / "python.exe", - ruler_root=tmp_path / "ruler", + package_root=tmp_path / "site-packages", + staged_root=tmp_path / "staged-ruler", raw_root=tmp_path / "raw", receipt=receipt, ) @@ -73,9 +121,92 @@ def test_generator_argv_preserves_multiline_template_as_one_argument(tmp_path) - assert actual[template_index] == ruler.NIAH_TEMPLATE assert "\n{context}\n" in actual[template_index] assert portable[0] == "" - assert portable[1] == "-s" - assert portable[2] == "scripts/data/synthetic/niah.py" - assert portable[4] == "" + # ``-I`` ignores PYTHONDONTWRITEBYTECODE, so ``-B`` must be explicit to + # keep imported bytecode out of the authenticated staged source tree. + assert portable[1:9] == [ + "-I", + "-S", + "-B", + "-X", + "pycache_prefix=", + "-X", + "utf8", + "-c", + ] + assert portable[10] == "" + assert portable[11] == "" + assert portable[12] == "" + assert portable[13] == "synthetic/niah.py" + assert portable[15] == "" + + +def test_isolated_bootstrap_imports_authenticated_sibling_module(tmp_path) -> None: + root = tmp_path / "data" + script_dir = root / "synthetic" + script_dir.mkdir(parents=True) + (script_dir / "constants.py").write_text("VALUE = 'bound'\n", encoding="utf-8") + (script_dir / "task.py").write_text( + "from constants import VALUE\nfrom fixture import VALUE as PACKAGE_VALUE\n" + "print(VALUE + ':' + PACKAGE_VALUE)\n", + encoding="utf-8", + ) + marker = tmp_path / "pth-executed" + package_root = _minimal_recorded_package_root( + tmp_path / "site-packages", + pth_payload=f"import pathlib; pathlib.Path({str(marker)!r}).write_text('bad')\n", + ) + adjacent_pycache = package_root / "__pycache__" + adjacent_pycache.mkdir() + (adjacent_pycache / f"fixture.{sys.implementation.cache_tag}.pyc").write_bytes( + b"unbound-adjacent-bytecode" + ) + pycache_prefix = tmp_path / "empty-pycache" + pycache_prefix.mkdir() + + result = subprocess.run( + ruler._sealed_python_argv( + python=Path(sys.executable), + package_root=package_root, + pycache_prefix=pycache_prefix, + code=ruler.ISOLATED_SOURCE_BOOTSTRAP, + arguments=(str(root), "synthetic/task.py"), + ), + check=False, + capture_output=True, + text=True, + encoding="utf-8", + errors="strict", + ) + + assert result.returncode == 0, result.stderr + assert result.stdout == "bound:package-bound\n" + assert not marker.exists() + assert not any(pycache_prefix.iterdir()) + assert not any(root.rglob("*.pyc")) + + +def test_sealed_bootstrap_rejects_unrecorded_shadow_module(tmp_path: Path) -> None: + package_root = _minimal_recorded_package_root(tmp_path / "site-packages") + (package_root / "rogue.py").write_text("VALUE = 'unbound'\n", encoding="utf-8") + pycache_prefix = tmp_path / "empty-pycache" + pycache_prefix.mkdir() + + result = subprocess.run( + ruler._sealed_python_argv( + python=Path(sys.executable), + package_root=package_root, + pycache_prefix=pycache_prefix, + code=ruler.SEALED_STARTUP_BOOTSTRAP, + ), + check=False, + capture_output=True, + text=True, + encoding="utf-8", + errors="strict", + ) + + assert result.returncode != 0 + assert "unrecorded importable code" in result.stderr def test_valid_niah_row_is_independently_recounted() -> None: @@ -175,6 +306,178 @@ def test_atomic_publish_same_is_idempotent_but_rejects_drift(tmp_path) -> None: ruler._atomic_publish_same(path, b"different\n") +def test_retry_reclaims_only_owned_staging_and_complete_diagnostic_orphans(tmp_path) -> None: + filename = "retrieval__niah_single_1__l4096__s12339.json" + config = "niah_single_1" + raw_root = tmp_path / "raw" + output_dir = tmp_path / "output" + raw_root.mkdir() + output_dir.mkdir() + receipt_key = ruler._sha256_bytes(filename.encode("utf-8"))[:12] + staging = raw_root / f".rq-{receipt_key}.fixture.staging" + staging.mkdir() + (staging / "command-manifest.json").write_bytes(b"fixture") + published = raw_root / filename.removesuffix(".json") + (published / config).mkdir(parents=True) + for relative in ( + "command-manifest.json", + "runtime-manifest.json", + "stdout.log", + "stderr.log", + f"{config}/validation.jsonl", + ): + (published / relative).write_bytes(b"fixture") + + recovered = ruler.recover_owned_receipt_orphans( + filename=filename, + config=config, + raw_root=raw_root, + output_dir=output_dir, + ) + + assert recovered == (staging.name, published.name) + assert not staging.exists() + assert not published.exists() + + +def test_retry_refuses_mutated_orphan_and_preserves_it(tmp_path) -> None: + filename = "retrieval__niah_single_1__l4096__s12339.json" + config = "niah_single_1" + raw_root = tmp_path / "raw" + output_dir = tmp_path / "output" + raw_root.mkdir() + output_dir.mkdir() + receipt_key = ruler._sha256_bytes(filename.encode("utf-8"))[:12] + staging = raw_root / f".rq-{receipt_key}.fixture.staging" + staging.mkdir() + (staging / "unowned.txt").write_bytes(b"preserve") + + with pytest.raises(ValueError, match="unexpected file"): + ruler.recover_owned_receipt_orphans( + filename=filename, + config=config, + raw_root=raw_root, + output_dir=output_dir, + ) + + assert (staging / "unowned.txt").read_bytes() == b"preserve" + + +def test_retry_never_cleans_diagnostics_beside_a_published_receipt(tmp_path) -> None: + filename = "retrieval__niah_single_1__l4096__s12339.json" + config = "niah_single_1" + raw_root = tmp_path / "raw" + output_dir = tmp_path / "output" + raw_root.mkdir() + output_dir.mkdir() + published = raw_root / filename.removesuffix(".json") + published.mkdir() + (output_dir / filename).write_bytes(b"published") + + assert ( + ruler.recover_owned_receipt_orphans( + filename=filename, + config=config, + raw_root=raw_root, + output_dir=output_dir, + ) + == () + ) + assert published.is_dir() + + +def test_two_partial_invocations_then_full_set_publish_one_complete_manifest( + monkeypatch, tmp_path +) -> None: + required = [ + { + "filename": f"receipt-{index:02d}.json", + "phase": "calibration" if index < 16 else "stage_a", + "category": "retrieval", + "config": "niah_single_1", + "configured_length": 2_048, + "seed": index, + } + for index in range(20) + ] + output_dir = tmp_path / "receipts" + output_dir.mkdir() + verified: list[str] = [] + + def fake_verify(*, path: Path, receipt, **_kwargs): + verified.append(path.name) + return { + "category": receipt["category"], + "command_manifest": {"fixture": path.name}, + "command_manifest_file": { + "name": "generator/command-manifest.json", + "sha256": "1" * 64, + "size_bytes": 1, + }, + "config": receipt["config"], + "configured_length": receipt["configured_length"], + "filename": path.name, + "generator_reported_length": 100, + "phase": receipt["phase"], + "raw_validation_base64": "e30K", + "raw_validation_file": { + "name": "generator/raw-validation.jsonl", + "sha256": "2" * 64, + "size_bytes": 3, + }, + "seed": receipt["seed"], + "sha256": "3" * 64, + "size_bytes": 4, + } + + monkeypatch.setattr(ruler, "_load_existing_receipt_result", fake_verify) + capture = type( + "Capture", + (), + {"resolver": type("Resolver", (), {"RULER_REVISION": "a" * 40})()}, + )() + kwargs = { + "required": required, + "output_dir": output_dir, + "raw_root": tmp_path / "raw", + "capture": capture, + "python": tmp_path / "python.exe", + "package_root": tmp_path / "site-packages", + "staged_root": tmp_path / "staged", + "tokenizer": object(), + "static_entries": [ + { + "name": "launcher/generate_static_q468_ruler_receipts.py", + "sha256": "4" * 64, + "size_bytes": 5, + } + ], + "source_manifest": [], + "runtime_manifest": {"schema": ruler.RUNTIME_MANIFEST_SCHEMA}, + } + + for item in required[:7]: + (output_dir / item["filename"]).write_bytes(b"partial-one") + assert ruler.finalize_generation_manifest_if_complete(**kwargs) is None + assert not (output_dir / "generation-manifest.json").exists() + for item in required[7:13]: + (output_dir / item["filename"]).write_bytes(b"partial-two") + assert ruler.finalize_generation_manifest_if_complete(**kwargs) is None + assert not (output_dir / "generation-manifest.json").exists() + for item in required[13:]: + (output_dir / item["filename"]).write_bytes(b"complete") + + manifest = ruler.finalize_generation_manifest_if_complete(**kwargs) + + assert manifest is not None + assert manifest["receipt_count"] == 20 + assert [item["filename"] for item in manifest["receipts"]] == [ + item["filename"] for item in required + ] + assert verified == [item["filename"] for item in required] + assert list(output_dir.glob("*manifest.json")) == [output_dir / "generation-manifest.json"] + + def test_subprocess_environment_removes_python_injection(monkeypatch) -> None: monkeypatch.setenv("PYTHONHOME", "untrusted-home") monkeypatch.setenv("PYTHONPATH", "untrusted-path") @@ -187,29 +490,86 @@ def test_subprocess_environment_removes_python_injection(monkeypatch) -> None: assert env["TEST_MARKER"] == "bound" +def test_isolated_stage_contains_only_verified_blob_and_corpus_bytes(tmp_path) -> None: + checkout = ruler.VerifiedRulerCheckout( + source_manifest=(), + source_files={"scripts/data/synthetic/task.py": b"print('verified')\n"}, + ) + static_inputs = ruler.VerifiedStaticInputs( + entries=(), corpus_files={"fixture.json": b"{}\n"}, sealed_runtime_files={} + ) + staged_root = tmp_path / "staged" + + manifest = ruler.stage_verified_ruler_source( + staged_root, checkout=checkout, static_inputs=static_inputs + ) + + assert sorted(manifest) == [ + "scripts/data/synthetic/json/fixture.json", + "scripts/data/synthetic/task.py", + ] + (staged_root / "unexpected.py").write_bytes(b"shadow") + with pytest.raises(ValueError, match="inventory drifted"): + ruler.verify_staged_ruler_source(staged_root, expected=manifest) + + def test_independent_tokenizer_uses_verified_isolated_python(monkeypatch, tmp_path) -> None: - python = tmp_path / "python.exe" - tokenizer_dir = tmp_path / "tokenizer" + python_runtime_root = tmp_path / "python-runtime" + python_runtime_root.mkdir() + python = python_runtime_root / "python.exe" + package_runtime_root = tmp_path / "package-runtime" + package_root = package_runtime_root / "Lib" / "site-packages" + package_root.mkdir(parents=True) + runtime_input_root = tmp_path / "runtime-inputs" + tokenizer_dir = runtime_input_root / "tokenizer" python.write_bytes(b"fixture") - tokenizer_dir.mkdir() + tokenizer_dir.mkdir(parents=True) + python_runtime_manifest = [ruler._tree_file_entry("python.exe", python.read_bytes())] observed: dict[str, object] = {} def fake_run(argv, **kwargs): observed["argv"] = argv observed["env"] = kwargs["env"] + observed["encoding"] = kwargs["encoding"] + observed["errors"] = kwargs["errors"] + observed["timeout"] = kwargs["timeout"] observed["request"] = json.loads(kwargs["input"]) return subprocess.CompletedProcess(argv, 0, stdout='{"count":3}\n', stderr="") monkeypatch.setattr(ruler.subprocess, "run", fake_run) monkeypatch.setenv("PYTHONHOME", "untrusted-home") - tokenizer = ruler.IndependentTokenizer(python=python, tokenizer_dir=tokenizer_dir) + tokenizer = ruler.IndependentTokenizer( + python=python, + tokenizer_dir=tokenizer_dir, + package_root=package_root, + package_tree_manifest=(), + runtime_input_root=runtime_input_root, + runtime_input_manifest={}, + python_runtime_root=python_runtime_root, + python_runtime_manifest=python_runtime_manifest, + ) - assert tokenizer.count_tokens("one two three") == 3 - assert observed["argv"][:2] == [str(python.resolve()), "-I"] + assert tokenizer.count_tokens("one two three \u0e04\u0e33\u0e16\u0e32\u0e21") == 3 + assert observed["argv"][:9] == [ + str(python.resolve()), + "-I", + "-S", + "-B", + "-X", + observed["argv"][5], + "-X", + "utf8", + "-c", + ] + assert str(observed["argv"][5]).startswith("pycache_prefix=") assert observed["request"] == { - "text": "one two three", + "text": "one two three \u0e04\u0e33\u0e16\u0e32\u0e21", "tokenizer_dir": str(tokenizer_dir.resolve()), } + assert observed["encoding"] == "utf-8" + assert observed["errors"] == "strict" + assert observed["timeout"] == ruler.TOKENIZER_TIMEOUT_SECONDS + assert observed["argv"][10] == str(package_root.resolve()) assert "PYTHONHOME" not in observed["env"] assert observed["env"]["TRANSFORMERS_OFFLINE"] == "1" @@ -266,3 +626,101 @@ def test_frozen_runtime_requirements_match_the_probe_contract() -> None: assert pinned == ruler.RUNTIME_PACKAGES assert "torch" not in {ruler._canonical_distribution_name(name) for name in pinned} assert "torch" in ruler.FORBIDDEN_RUNTIME_MODULES + + +def test_produced_runtime_manifest_round_trips_through_capture_contract( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + python_runtime_root = tmp_path / "python-runtime" + python_runtime_root.mkdir() + python = python_runtime_root / "python.exe" + python.write_bytes(b"fixture-ruler-python") + for name, data in ( + ("python3.dll", b"fixture-python3-dll"), + ("python311.dll", b"fixture-python311-dll"), + ): + (python_runtime_root / name).write_bytes(data) + python_runtime_manifest = [ + ruler._tree_file_entry(path.name, path.read_bytes()) + for path in sorted(python_runtime_root.iterdir(), key=lambda path: path.name) + ] + package_runtime_root = tmp_path / "package-runtime" + package_root = package_runtime_root / "Lib" / "site-packages" + package_root.mkdir(parents=True) + inventories: dict[str, object] = {} + for name, version in ruler.RUNTIME_PACKAGES.items(): + canonical_name = ruler._canonical_distribution_name(name) + record = f"{name}=={version}\n".encode() + inventories[name] = { + "canonical_name": canonical_name, + "version": version, + "record_sha256": ruler._sha256_bytes(record), + "record_size_bytes": len(record), + "files": [ + { + "path": f"{canonical_name}-{version}.dist-info/RECORD", + "sha256": ruler._sha256_bytes(record), + "size_bytes": len(record), + } + ], + } + payload = { + "python": ruler.RUNTIME_PYTHON_VERSION, + "implementation": "cpython", + "cache_tag": "cpython-311", + "executable": str(python.resolve()), + "platform": "fixture-platform", + "machine": "fixture-machine", + "flags": { + "ignore_environment": 1, + "isolated": 1, + "no_user_site": 1, + }, + "startup_policy": dict(ruler.SEALED_STARTUP_POLICY), + "packages": dict(ruler.RUNTIME_PACKAGES), + "installed_distributions": { + ruler._canonical_distribution_name(name): version + for name, version in ruler.RUNTIME_PACKAGES.items() + }, + "distribution_file_inventory": inventories, + "forbidden_modules": {name: False for name in ruler.FORBIDDEN_RUNTIME_MODULES}, + "resources": { + name: str(tmp_path / name.replace("/", "_")) + for name in ruler.EXPECTED_PACKAGE_RESOURCES + }, + } + + observed: dict[str, object] = {} + + def fake_run(argv: list[str], **kwargs: object) -> subprocess.CompletedProcess[str]: + assert argv[:5] == [str(python.resolve()), "-I", "-S", "-B", "-X"] + observed["timeout"] = kwargs["timeout"] + return subprocess.CompletedProcess( + argv, + 0, + stdout=json.dumps(payload, sort_keys=True, separators=(",", ":")), + stderr="", + ) + + monkeypatch.setattr(ruler.subprocess, "run", fake_run) + excluded_startup_files = [ + {"name": name, "sha256": digest, "size_bytes": size} + for name, (size, digest) in sorted(ruler.EXPECTED_EXCLUDED_VIRTUALENV_STARTUP_FILES.items()) + ] + produced, _resource_paths = ruler.verify_runtime( + python, + tmp_path / "nltk-data", + package_root=package_root, + package_tree_manifest=(), + excluded_startup_files=excluded_startup_files, + python_runtime_root=python_runtime_root, + python_runtime_manifest=python_runtime_manifest, + source_python=ruler._tree_file_entry("source/python.exe", b"source-python"), + source_pyvenv_config=ruler._tree_file_entry("source/pyvenv.cfg", b"source-pyvenv"), + ) + capture = ruler._load_capture_module() + + assert capture.RULER_RUNTIME_MANIFEST_SCHEMA == ruler.RUNTIME_MANIFEST_SCHEMA + assert capture.RULER_LAUNCHER_REVISION == ruler.LAUNCHER_REVISION + assert capture._normalize_ruler_runtime_manifest(produced) == produced + assert observed["timeout"] == ruler.RUNTIME_PROBE_TIMEOUT_SECONDS diff --git a/tests/test_launch_static_q468_calibration.py b/tests/test_launch_static_q468_calibration.py new file mode 100644 index 0000000..28d89ab --- /dev/null +++ b/tests/test_launch_static_q468_calibration.py @@ -0,0 +1,427 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import json +import platform +import struct +import subprocess +import sys +from pathlib import Path +from typing import Any + +import pytest + +SCRIPT = ( + Path(__file__).resolve().parents[1] + / "scripts" + / "launch_static_q468_calibration.py" +) +SPEC = importlib.util.spec_from_file_location("launch_static_q468_calibration", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +launcher = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(launcher) + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _write(path: Path, data: bytes) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return path + + +def _sealed_fixture(tmp_path: Path) -> dict[str, Any]: + base = tmp_path / "base" + packages = tmp_path / "packages" + repository = tmp_path / "repository" + artifacts = tmp_path / "artifacts" + + interpreter = _write(base / "python.exe", b"staged interpreter\n") + _write(base / "Lib" / "marker.py", b"BASE = True\n") + import_root = packages / "Lib" / "site-packages" + _write(import_root / "demo" / "__init__.py", b"VALUE = 1\n") + metadata = b"Metadata-Version: 2.1\nName: demo\nVersion: 1.0\n\n" + _write(import_root / "demo-1.0.dist-info" / "METADATA", metadata) + record = ( + b"demo-1.0.dist-info/METADATA,,\n" + b"demo-1.0.dist-info/RECORD,,\n" + b"demo/__init__.py,,\n" + ) + _write(import_root / "demo-1.0.dist-info" / "RECORD", record) + + package_roots = {"packages": packages.resolve(strict=True)} + import_paths = {"packages": "Lib/site-packages"} + base_files = list(launcher._tree_files(base, context="fixture base")) + package_files = list(launcher._tree_files(packages, context="fixture packages")) + interpreter_record = launcher._stable_file_record( + interpreter, + relative="python.exe", + context="fixture interpreter", + ) + runtime = { + "artifact_kind": launcher.RUNTIME_MANIFEST_KIND, + "base_runtime_root": launcher.BASE_RUNTIME_ROOT_NAME, + "base_sys_path": ["Lib"], + "distributions": list( + launcher._distribution_inventory(package_roots, import_paths) + ), + "interpreter": { + "relative_path": "python.exe", + "root": launcher.BASE_RUNTIME_ROOT_NAME, + "sha256": interpreter_record["sha256"], + "size_bytes": interpreter_record["size_bytes"], + }, + "launch_policy": launcher.SEALED_LAUNCH_POLICY, + "machine": { + "architecture": platform.architecture()[0], + "byteorder": sys.byteorder, + "machine": platform.machine(), + "pointer_bits": 8 * struct.calcsize("P"), + "system": platform.system(), + }, + "package_roots": [ + {"import_path": "Lib/site-packages", "name": "packages"} + ], + "python": { + "abi_flags": getattr(sys, "abiflags", ""), + "cache_tag": sys.implementation.cache_tag, + "implementation": platform.python_implementation(), + "version": platform.python_version(), + }, + "runtime_trees": [ + {"files": base_files, "kind": "base-runtime", "name": "base-runtime"}, + {"files": package_files, "kind": "packages", "name": "packages"}, + ], + "schema_version": launcher.RUNTIME_MANIFEST_SCHEMA, + } + runtime_path = _write( + artifacts / "runtime.json", + launcher._canonical_json_bytes(runtime), + ) + + runner_path = _write( + repository / launcher.RUNNER_SOURCE_PATH, + ( + b"def sealed_main(argv, *, base_runtime_root, package_roots, " + b"package_import_paths, interpreter_path, pycache_prefix):\n" + b" return 0\n" + ), + ) + source_payload = { + "object_format": "sha1", + "paths": [ + { + "git_blob_oid": "b" * 40, + "index_blob_oid": "b" * 40, + "mode": "100644", + "path": launcher.RUNNER_SOURCE_PATH, + "raw_sha256": _sha256(runner_path.read_bytes()), + "worktree_blob_oid": "b" * 40, + } + ], + "profile": "experiment-013-static-q468-frozen-source-v1", + "repository_binding": {}, + "schema": "recurquant.experiment013.source-manifest.v1", + "source_commit": "a" * 40, + } + source = { + **source_payload, + "canonical_manifest_sha256": _sha256( + launcher._pretty_json_bytes(source_payload) + ), + } + source_path = _write( + artifacts / "source.json", + launcher._pretty_json_bytes(source), + ) + model_path = _write(artifacts / "model.json", b'{"model":"fixture"}\n') + parquet_path = _write(artifacts / "parquet.json", b'{"parquet":"fixture"}\n') + bindings = { + "calibration_runtime_manifest_file_sha256": _sha256(runtime_path.read_bytes()), + "model_file_manifest_file_sha256": _sha256(model_path.read_bytes()), + "parquet_materialization_manifest_file_sha256": _sha256( + parquet_path.read_bytes() + ), + "repository_source_manifest_file_sha256": _sha256(source_path.read_bytes()), + } + evidence = { + "execution_bindings": bindings, + "identity_only": True, + "phase": "calibration", + "promotion_required": False, + "schema_version": launcher.IDENTITY_SCHEMA, + "status": "frozen", + } + identity = { + "canonical_evidence_sha256": _sha256( + launcher._canonical_json_bytes(evidence) + ), + "evidence": evidence, + } + identity_path = _write( + artifacts / "identity.json", + launcher._canonical_json_bytes(identity), + ) + runner_arguments = [ + "--frozen-identity", + str(identity_path), + "--repository-source-manifest", + str(source_path), + "--model-file-manifest", + str(model_path), + "--expected-model-file-manifest-sha256", + bindings["model_file_manifest_file_sha256"], + "--parquet-materialization-manifest", + str(parquet_path), + "--expected-parquet-materialization-manifest-sha256", + bindings["parquet_materialization_manifest_file_sha256"], + "--runtime-manifest", + str(runtime_path), + "--expected-runtime-manifest-sha256", + bindings["calibration_runtime_manifest_file_sha256"], + "--repository-root", + str(repository), + ] + host_arguments = [ + "--base-runtime-root", + str(base), + "--package-root", + f"packages={packages.resolve(strict=True)}", + "--runtime-manifest", + str(runtime_path), + "--", + *runner_arguments, + ] + return { + "base": base, + "bindings": bindings, + "host_arguments": host_arguments, + "model_path": model_path, + "packages": packages, + "parquet_path": parquet_path, + "repository": repository, + "runner_arguments": runner_arguments, + "runtime_path": runtime_path, + } + + +def test_launch_uses_exact_isolated_command_and_reauthenticates( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fixture = _sealed_fixture(tmp_path) + monkeypatch.setenv("VIRTUAL_ENV", str(tmp_path / "untrusted-venv")) + monkeypatch.setenv("VIRTUAL_ENV_PROMPT", "untrusted") + events: list[str] = [] + commands: list[tuple[list[str], Path, dict[str, str]]] = [] + verify_bound = launcher._verify_bound_artifacts + verify_runtime = launcher._verify_runtime + verify_pycache = launcher._verify_empty_pycache + + def bound_wrapper(*args: Any, **kwargs: Any) -> Any: + events.append("bound") + return verify_bound(*args, **kwargs) + + def runtime_wrapper(*args: Any, **kwargs: Any) -> Any: + events.append("runtime") + return verify_runtime(*args, **kwargs) + + def pycache_wrapper(*args: Any, **kwargs: Any) -> Any: + events.append("pycache") + return verify_pycache(*args, **kwargs) + + def run_wrapper( + command: list[str], + *, + check: bool, + cwd: Path, + env: dict[str, str], + ) -> subprocess.CompletedProcess[str]: + events.append("run") + assert check is False + commands.append((command, cwd, env)) + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(launcher, "_verify_bound_artifacts", bound_wrapper) + monkeypatch.setattr(launcher, "_verify_runtime", runtime_wrapper) + monkeypatch.setattr(launcher, "_verify_empty_pycache", pycache_wrapper) + monkeypatch.setattr(launcher.subprocess, "run", run_wrapper) + + assert launcher.launch(fixture["host_arguments"]) == 0 + assert events == [ + "bound", + "runtime", + "pycache", + "run", + "pycache", + "bound", + "runtime", + "pycache", + ] + assert len(commands) == 1 + command, cwd, environment = commands[0] + assert command[0] == str((fixture["base"] / "python.exe").resolve(strict=True)) + assert command[1:5] == ["-I", "-S", "-B", "-X"] + assert command[5].startswith("pycache_prefix=") + assert command[6:10] == ["-X", "utf8", "-c", launcher.SEALED_BOOTSTRAP] + assert command[10] == str(fixture["runtime_path"].resolve(strict=True)) + assert command[11] == str(fixture["base"].resolve(strict=True)) + assert json.loads(command[12]) == { + "packages": str(fixture["packages"].resolve(strict=True)) + } + assert command[14:] == fixture["runner_arguments"] + assert not Path(command[13]).exists() + assert cwd == fixture["base"].resolve(strict=True) + assert all(not key.upper().startswith("PYTHON") for key in environment) + assert "VIRTUAL_ENV" not in environment + assert "VIRTUAL_ENV_PROMPT" not in environment + + +def test_help_does_not_require_the_runner_separator(capsys: pytest.CaptureFixture[str]) -> None: + assert launcher.launch(["--help"]) == 0 + assert "exact run_static_q468_calibration.py arguments" in capsys.readouterr().out + + +@pytest.mark.parametrize( + ("field", "value"), + [("schema_version", 3.0), ("dont_write_bytecode", True)], +) +def test_runtime_parser_rejects_equality_compatible_json_types( + tmp_path: Path, + field: str, + value: object, +) -> None: + fixture = _sealed_fixture(tmp_path) + document = json.loads(fixture["runtime_path"].read_bytes()) + if field == "schema_version": + document[field] = value + else: + document["launch_policy"][field] = value + + with pytest.raises(launcher.SealedLaunchError): + launcher._parse_runtime_manifest(launcher._canonical_json_bytes(document)) + + +def test_identity_parser_rejects_float_schema_version(tmp_path: Path) -> None: + fixture = _sealed_fixture(tmp_path) + identity_path = fixture["runtime_path"].parent / "identity.json" + document = json.loads(identity_path.read_bytes()) + document["evidence"]["schema_version"] = 4.0 + document["canonical_evidence_sha256"] = _sha256( + launcher._canonical_json_bytes(document["evidence"]) + ) + + with pytest.raises(launcher.SealedLaunchError, match="schema"): + launcher._parse_identity(launcher._canonical_json_bytes(document)) + + +def test_embedded_bootstrap_repeats_exact_json_type_checks() -> None: + assert 'type(evidence.get("schema_version")) is not int' in launcher.SEALED_BOOTSTRAP + assert '_typed(root["launch_policy"], _policy' in launcher.SEALED_BOOTSTRAP + + +def test_identity_binding_mismatch_stops_before_subprocess( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fixture = _sealed_fixture(tmp_path) + fixture["model_path"].write_bytes(b"mutated model manifest\n") + monkeypatch.setattr( + launcher.subprocess, + "run", + lambda *_args, **_kwargs: pytest.fail("subprocess must not start"), + ) + + with pytest.raises(launcher.SealedLaunchError, match="identity binding mismatch"): + launcher.launch(fixture["host_arguments"]) + + +def test_expected_parquet_digest_mismatch_stops_before_subprocess( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fixture = _sealed_fixture(tmp_path) + arguments = list(fixture["host_arguments"]) + index = arguments.index("--expected-parquet-materialization-manifest-sha256") + arguments[index + 1] = "0" * 64 + monkeypatch.setattr( + launcher.subprocess, + "run", + lambda *_args, **_kwargs: pytest.fail("subprocess must not start"), + ) + + with pytest.raises(launcher.SealedLaunchError, match="runner digest binding mismatch"): + launcher.launch(arguments) + + +def test_complete_tree_mutation_stops_before_subprocess( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fixture = _sealed_fixture(tmp_path) + package_file = fixture["packages"] / "Lib" / "site-packages" / "demo" / "__init__.py" + package_file.write_bytes(b"VALUE = 2\n") + monkeypatch.setattr( + launcher.subprocess, + "run", + lambda *_args, **_kwargs: pytest.fail("subprocess must not start"), + ) + + with pytest.raises(launcher.SealedLaunchError, match="runtime tree packages differs"): + launcher.launch(fixture["host_arguments"]) + + +def test_nonempty_pycache_is_rejected(tmp_path: Path) -> None: + pycache = tmp_path / "pycache" + _write(pycache / "unexpected.pyc", b"not allowed") + + with pytest.raises(launcher.SealedLaunchError, match="pycache prefix is not empty"): + launcher._verify_empty_pycache(pycache) + + +def test_bootstrap_rejects_flag_drift() -> None: + completed = subprocess.run( + [sys.executable, "-S", "-c", launcher.SEALED_BOOTSTRAP], + check=False, + capture_output=True, + text=True, + ) + + assert completed.returncode != 0 + assert "sealed bootstrap startup flags drifted" in completed.stderr + + +def test_bootstrap_rejects_preloaded_sensitive_module(tmp_path: Path) -> None: + pycache = tmp_path / "empty-pycache" + pycache.mkdir() + prefix = "import sys; sys.modules['_virtualenv'] = object()\n" + completed = subprocess.run( + [ + sys.executable, + "-I", + "-S", + "-B", + "-X", + f"pycache_prefix={pycache}", + "-X", + "utf8", + "-c", + prefix + launcher.SEALED_BOOTSTRAP, + ], + check=False, + capture_output=True, + text=True, + ) + + assert completed.returncode != 0 + assert "sealed bootstrap found a preloaded sensitive module" in completed.stderr + assert not any(pycache.iterdir()) + + +@pytest.mark.parametrize("arguments", [[], ["--", "--", "runner"]]) +def test_launcher_requires_one_separator(arguments: list[str]) -> None: + with pytest.raises(launcher.SealedLaunchError, match="exactly one -- separator"): + launcher._split_host_and_runner_args(arguments) diff --git a/tests/test_resolve_static_q468_identity.py b/tests/test_resolve_static_q468_identity.py index a4779e4..738c4d1 100644 --- a/tests/test_resolve_static_q468_identity.py +++ b/tests/test_resolve_static_q468_identity.py @@ -33,6 +33,14 @@ "static_k27030_policy_file_sha256": resolver.sha256_bytes(b"k27030-policy"), "static_k29334_policy_file_sha256": resolver.sha256_bytes(b"k29334-policy"), } +FIXTURE_EXECUTION_BINDINGS = { + "repository_source_manifest_file_sha256": resolver.sha256_bytes(b"source-manifest"), + "calibration_runtime_manifest_file_sha256": resolver.sha256_bytes(b"runtime-manifest"), + "model_file_manifest_file_sha256": resolver.sha256_bytes(b"model-manifest"), + "parquet_materialization_manifest_file_sha256": ( + resolver.PARQUET_MATERIALIZATION_MANIFEST_FILE_SHA256 + ), +} def _hash(label: str) -> str: @@ -242,6 +250,7 @@ def _stage_a_source() -> dict[str, Any]: "datasets": _datasets(), "tokenizer": _tokenizer(), "records": list(reversed(records)), + "execution_bindings": dict(FIXTURE_EXECUTION_BINDINGS), "model_weights_loaded": False, "calibration_binding": dict(FIXTURE_BINDING), } @@ -289,11 +298,37 @@ def test_stage_a_candidate_is_deterministic_and_complete() -> None: *(["humaneval_plus"] * 4), ] assert evidence["tokenizer"]["file_manifest_sha256"] == _tokenizer_manifest_hash() + assert evidence["execution_bindings"] == FIXTURE_EXECUTION_BINDINGS assert evidence["content_manifest_sha256"] == resolver.sha256_bytes( resolver.canonical_json_bytes(evidence["records"]) ) +@pytest.mark.parametrize( + ("mutate", "message"), + [ + (lambda evidence: evidence.update({"identity_only": 1}), "identity_only drifted"), + ( + lambda evidence: evidence.update({"promotion_required": 1}), + "promotion_required drifted", + ), + ( + lambda evidence: evidence["protected_identity"].update({"stage_b_read": 0}), + "protected identity boundary drifted", + ), + ], +) +def test_candidate_rejects_boolean_integer_aliases(mutate: Any, message: str) -> None: + candidate = _build_candidate(_stage_a_source()) + mutate(candidate["evidence"]) + candidate["canonical_evidence_sha256"] = resolver.sha256_bytes( + resolver.canonical_json_bytes(candidate["evidence"]) + ) + + with pytest.raises(ValueError, match=message): + resolver.validate_candidate_artifact(candidate) + + def test_stage_a_candidate_requires_and_matches_a_verified_binding_artifact() -> None: source = _stage_a_source() with pytest.raises(ValueError, match="requires a verified calibration binding"): @@ -335,6 +370,18 @@ def test_raw_content_and_unknown_fields_fail_closed() -> None: lambda source: source.update({"model_weights_loaded": True}), "before model weights", ), + ( + lambda source: source["execution_bindings"].update( + {"model_file_manifest_file_sha256": "not-a-sha256"} + ), + "model_file_manifest_file_sha256", + ), + ( + lambda source: source["execution_bindings"].update( + {"parquet_materialization_manifest_file_sha256": "0" * 64} + ), + "Parquet materialization manifest file SHA-256 drifted", + ), ( lambda source: source["records"][0]["token_span"].update({"scored_start": 4_095}), "contiguous", @@ -567,6 +614,11 @@ def test_frozen_stage_a_decoder_reauthenticates_promotion_records_and_binding() assert decoded.file_sha256 == resolver.sha256_bytes(frozen_bytes) assert len(decoded.records) == 12 assert decoded.calibration_binding == FIXTURE_BINDING + assert decoded.execution_bindings == FIXTURE_EXECUTION_BINDINGS + assert ( + decoded.parquet_materialization_manifest_file_sha256 + == resolver.PARQUET_MATERIALIZATION_MANIFEST_FILE_SHA256 + ) tampered = copy.deepcopy(frozen) tampered["evidence"]["records"][0]["source_content_sha256"] = "0" * 64 diff --git a/tests/test_run_static_q468_calibration.py b/tests/test_run_static_q468_calibration.py new file mode 100644 index 0000000..82cd9c5 --- /dev/null +++ b/tests/test_run_static_q468_calibration.py @@ -0,0 +1,1782 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import json +import subprocess +import sys +from collections.abc import Mapping, Sequence +from dataclasses import replace +from pathlib import Path +from typing import Any + +import pytest +import torch + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "run_static_q468_calibration.py" +API_SCRIPT = ( + Path(__file__).resolve().parents[1] / "src" / "recurquant" / "experiment013_calibration_api.py" +) +API_SPEC = importlib.util.spec_from_file_location( + "experiment013_calibration_api_for_tests", API_SCRIPT +) +assert API_SPEC is not None and API_SPEC.loader is not None +api = importlib.util.module_from_spec(API_SPEC) +sys.modules[API_SPEC.name] = api +API_SPEC.loader.exec_module(api) +SPEC = importlib.util.spec_from_file_location("run_static_q468_calibration", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +runner = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = runner +SPEC.loader.exec_module(runner) + + +def digest(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def token_digest(values: Sequence[int]) -> str: + return digest(runner.canonical_json_bytes(list(values))) + + +def current_head() -> str: + process = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=SCRIPT.parents[1], + check=True, + capture_output=True, + text=True, + ) + return process.stdout.strip() + + +def record(token_ids: tuple[int, ...] = (1, 2, 3)) -> dict[str, object]: + prompt_stop = max(1, len(token_ids) - 1) + return { + "canonical_id": "item-1", + "config": "default", + "family": "mbpp", + "formatted_content_sha256": "b" * 64, + "generator_receipt_sha256": None, + "prompt_token_ids_sha256": token_digest(token_ids[:prompt_stop]), + "ruler_category": None, + "seed": None, + "configured_length": None, + "sequence_length": len(token_ids), + "sequence_token_ids_sha256": token_digest(token_ids), + "source_content_sha256": "a" * 64, + "target_token_ids_sha256": token_digest(token_ids[prompt_stop:]), + "token_span": { + "prefill_start": 0, + "prefill_stop": prompt_stop, + "scored_start": prompt_stop, + "scored_stop": len(token_ids), + "cache_exposed_start": len(token_ids), + "cache_exposed_stop": len(token_ids), + }, + "tokenizer_manifest_sha256": "c" * 64, + } + + +def materialized(item: Mapping[str, object], token_ids: tuple[int, ...]) -> Any: + return api.AuthenticatedSequence( + token_ids=token_ids, + source_content_sha256=item["source_content_sha256"], + formatted_content_sha256=item["formatted_content_sha256"], + generator_receipt_sha256=item["generator_receipt_sha256"], + tokenizer_manifest_sha256=item["tokenizer_manifest_sha256"], + ) + + +def identity( + records: Sequence[dict[str, object]], + *, + source_manifest_sha256: str = "7" * 64, + runtime_manifest_sha256: str = "8" * 64, + model_manifest_sha256: str = "9" * 64, + parquet_manifest_sha256: str = "a" * 64, +) -> Any: + return runner.FrozenCalibrationIdentity( + file_sha256="d" * 64, + canonical_evidence_sha256="e" * 64, + records=tuple(records), + assignment=(), + assignment_sha256="f" * 64, + tokenizer_manifest_sha256="c" * 64, + identity_input_manifest_sha256="1" * 64, + repository_source_manifest_file_sha256=source_manifest_sha256, + runtime_manifest_file_sha256=runtime_manifest_sha256, + model_file_manifest_file_sha256=model_manifest_sha256, + parquet_materialization_manifest_file_sha256=parquet_manifest_sha256, + model_id="example/model", + model_revision="2" * 40, + transformers_version="5.14.1", + artifact_bytes=b"frozen-identity", + ) + + +def bootstrap_identity_bytes( + *, + source_manifest_sha256: str, + runtime_manifest_sha256: str, + model_manifest_sha256: str, + parquet_manifest_sha256: str, +) -> bytes: + evidence = { + "execution_bindings": { + "calibration_runtime_manifest_file_sha256": runtime_manifest_sha256, + "model_file_manifest_file_sha256": model_manifest_sha256, + "parquet_materialization_manifest_file_sha256": parquet_manifest_sha256, + "repository_source_manifest_file_sha256": source_manifest_sha256, + }, + "identity_only": True, + "phase": "calibration", + "promotion_required": False, + "schema_version": 4, + "status": "frozen", + } + return runner.canonical_json_bytes( + { + "canonical_evidence_sha256": digest(runner.canonical_json_bytes(evidence)), + "evidence": evidence, + } + ) + + +def model_manifest_bytes(files: Mapping[str, bytes]) -> bytes: + file_records = [] + tree_records = [] + for index, (name, content) in enumerate(sorted(files.items())): + is_weight = name.endswith(".safetensors") + content_sha256 = digest(content) + git_blob_oid = hashlib.sha1( + f"blob {len(content)}\0".encode("ascii") + content, + usedforsecurity=False, + ).hexdigest() + item = { + "git_blob_oid": f"{index + 1:040x}" if is_weight else git_blob_oid, + "lfs_sha256": content_sha256 if is_weight else None, + "lfs_size_bytes": len(content) if is_weight else None, + "name": name, + "sha256": content_sha256 if is_weight else None, + "size_bytes": len(content), + } + file_records.append(item) + tree_records.append( + { + "git_blob_oid": item["git_blob_oid"], + "lfs_sha256": item["lfs_sha256"], + "lfs_size_bytes": item["lfs_size_bytes"], + "name": name, + } + ) + payload = { + "artifact_kind": runner.MODEL_FILE_MANIFEST_KIND, + "files": file_records, + "hub_tree_manifest_sha256": digest(runner.canonical_json_bytes(tree_records)), + "metadata_derivation": runner.MODEL_FILE_MANIFEST_DERIVATION, + "model_id": "example/model", + "revision": "2" * 40, + "schema_version": runner.MODEL_FILE_MANIFEST_SCHEMA, + "selection_profile": runner.MODEL_FILE_SELECTION_PROFILE, + "transformers_version": "5.14.1", + } + return runner.canonical_json_bytes(payload) + + +def runtime_manifest_bytes() -> bytes: + interpreter_sha256, interpreter_size = runner._stream_file_sha256( + Path(sys.executable).resolve(strict=True) + ) + machine = runner._current_machine_identity() + base_files = [ + { + "path": "Lib/os.py", + "sha256": "b" * 64, + "size_bytes": 1, + }, + { + "path": "python.exe", + "sha256": interpreter_sha256, + "size_bytes": interpreter_size, + }, + ] + package_files = [ + { + "path": "Lib/site-packages/transformers-5.14.1.dist-info/RECORD", + "sha256": "c" * 64, + "size_bytes": 1, + }, + { + "path": "Lib/site-packages/transformers/__init__.py", + "sha256": "d" * 64, + "size_bytes": 1, + }, + ] + payload = { + "artifact_kind": runner.RUNTIME_MANIFEST_KIND, + "base_runtime_root": runner.BASE_RUNTIME_ROOT_NAME, + "base_sys_path": ["Lib"], + "distributions": [ + { + "files": [item["path"] for item in package_files], + "name": "transformers", + "package_root": "packages", + "version": "5.14.1", + } + ], + "interpreter": { + "relative_path": "python.exe", + "root": runner.BASE_RUNTIME_ROOT_NAME, + "sha256": interpreter_sha256, + "size_bytes": interpreter_size, + }, + "launch_policy": dict(runner.SEALED_LAUNCH_POLICY), + "machine": dict( + zip( + ("system", "architecture", "machine", "byteorder", "pointer_bits"), + machine, + strict=True, + ) + ), + "package_roots": [{"import_path": "Lib/site-packages", "name": "packages"}], + "python": { + "abi_flags": getattr(sys, "abiflags", ""), + "cache_tag": sys.implementation.cache_tag, + "implementation": runner.platform.python_implementation(), + "version": runner.platform.python_version(), + }, + "runtime_trees": [ + { + "files": base_files, + "kind": "base-runtime", + "name": runner.BASE_RUNTIME_ROOT_NAME, + }, + { + "files": package_files, + "kind": "packages", + "name": "packages", + }, + ], + "schema_version": runner.RUNTIME_MANIFEST_SCHEMA, + } + return runner.canonical_json_bytes(payload) + + +@pytest.mark.parametrize( + ("field", "value"), + [("schema_version", 3.0), ("dont_write_bytecode", True)], +) +def test_runtime_manifest_rejects_equality_compatible_json_types( + field: str, + value: object, +) -> None: + document = json.loads(runtime_manifest_bytes()) + if field == "schema_version": + document[field] = value + else: + document["launch_policy"][field] = value + + with pytest.raises(ValueError): + runner.parse_calibration_runtime_manifest(runner.canonical_json_bytes(document)) + + +def test_bootstrap_identity_rejects_float_schema_version() -> None: + data = bootstrap_identity_bytes( + source_manifest_sha256="7" * 64, + runtime_manifest_sha256="8" * 64, + model_manifest_sha256="9" * 64, + parquet_manifest_sha256="a" * 64, + ) + document = json.loads(data) + document["evidence"]["schema_version"] = 4.0 + document["canonical_evidence_sha256"] = digest( + runner.canonical_json_bytes(document["evidence"]) + ) + + with pytest.raises(runner.CalibrationRunError, match="state"): + runner._bootstrap_identity_bindings(runner.canonical_json_bytes(document)) + + +def test_model_manifest_rejects_float_schema_version() -> None: + document = json.loads( + model_manifest_bytes( + { + "config.json": b"{}", + "model.safetensors.index.json": b"{}", + "model.safetensors-00001-of-00001.safetensors": b"weights", + } + ) + ) + document["schema_version"] = float(runner.MODEL_FILE_MANIFEST_SCHEMA) + + with pytest.raises(ValueError, match="schema"): + runner.parse_model_file_manifest(runner.canonical_json_bytes(document)) + + +def write_model_root(path: Path, files: Mapping[str, bytes]) -> None: + for name, content in files.items(): + target = path / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + + +class FakeBackend: + geometry = runner.Geometry(layer_indices=(0, 2), heads=2, key_rows=4, value_width=4) + + def __init__( + self, + frozen_identity: Any, + events: list[str], + *, + stability_passed: bool = True, + decode_error: BaseException | None = None, + ) -> None: + self.frozen_identity = frozen_identity + self.events = events + self.stability_passed = stability_passed + self.decode_error = decode_error + self.captured: list[Any] = [] + + def decode_identity(self, data: bytes) -> Any: + self.events.append("decode_identity") + assert data == b"identity" + if self.decode_error is not None: + raise self.decode_error + return self.frozen_identity + + def reduce_sequence( + self, + item: Mapping[str, object], + token_ids: tuple[int, ...], + captured: Any, + ) -> object: + self.events.append("reduce_sequence") + self.captured.append((item, token_ids, captured)) + return {"canonical_id": item["canonical_id"]} + + def finalize( + self, + scores: Sequence[object], + *, + identity: Any, + source_commit: str, + ) -> Any: + self.events.append("finalize") + assert len(scores) == len(identity.records) + assert source_commit == current_head() + stability = { + "checks": [{"name": "fake", "passed": self.stability_passed}], + "passed": self.stability_passed, + } + if not self.stability_passed: + return runner.FinalizationResult( + passed=False, + stability=stability, + artifacts=None, + ) + artifacts = runner.CalibrationArtifacts( + score=b"score", + split_half=b"split", + static_k27030=b"k27030", + static_k29334=b"k29334", + static_q48=b"q48", + stage_a_binding=b"binding", + stability=stability, + calibration_scores_sha256="3" * 64, + sequence_score_manifest_sha256="4" * 64, + ) + return runner.FinalizationResult( + passed=True, + stability=stability, + artifacts=artifacts, + ) + + +class FakeAdapter: + def __init__( + self, + sequence_by_id: Mapping[str, Any], + events: list[str], + *, + invalid_kernel_receipt: bool = False, + ) -> None: + self.sequence_by_id = sequence_by_id + self.events = events + self.invalid_kernel_receipt = invalid_kernel_receipt + self.closed = False + self.capture_flags: list[bool] = [] + + def materialize_sequence(self, item: Mapping[str, object]) -> Any: + self.events.append("materialize_sequence") + return self.sequence_by_id[str(item["canonical_id"])] + + def load_model(self, authenticated: Any) -> object: + self.events.append("load_model") + assert authenticated.revision == "2" * 40 + return object() + + def begin_sequence(self, model: object, item: Mapping[str, object]) -> None: + del model, item + self.events.append("begin_sequence") + + def step_token( + self, + model: object, + *, + token_id: int, + position: int, + capture_state: bool, + ) -> Any: + del model + self.events.append("step_token") + self.capture_flags.append(capture_state) + geometry = FakeBackend.geometry + return api.StepObservation( + position=position, + token_id=token_id, + layer_indices=geometry.layer_indices, + recurrence_query=torch.ones( + geometry.layers, + geometry.heads, + geometry.key_rows, + ), + recurrent_state=( + torch.ones( + geometry.layers, + geometry.heads, + geometry.key_rows, + geometry.value_width, + ) + if capture_state + else None + ), + successful_kernel_calls_per_layer=( + (2,) * geometry.layers if self.invalid_kernel_receipt else (1,) * geometry.layers + ), + ) + + def end_sequence(self, model: object, item: Mapping[str, object]) -> None: + del model, item + self.events.append("end_sequence") + + def close_model(self, model: object) -> None: + del model + self.events.append("close_model") + self.closed = True + + def runtime_metadata(self) -> Mapping[str, object]: + self.events.append("runtime_metadata") + return { + "model_open": not self.closed, + "name": "fake", + "one_token_calls": self.events.count("step_token"), + } + + +def fake_distortions(state: torch.Tensor, geometry: Any) -> Any: + assert tuple(state.shape) == ( + geometry.layers, + geometry.heads, + geometry.key_rows, + geometry.value_width, + ) + shape = (geometry.layers, geometry.heads, geometry.key_rows) + return tuple(torch.full(shape, value, dtype=torch.float64) for value in (3.0, 2.0, 1.0)) + + +def source_verifier(events: list[str], *, fail_on_call: int | None = None) -> Any: + calls = 0 + + def verify(expected: Mapping[str, object], root: Path) -> Any: + nonlocal calls + calls += 1 + events.append("verify_source") + assert expected == {"manifest": "expected", "source_commit": current_head()} + assert root == SCRIPT.parents[1] + if fail_on_call == calls: + raise runner.CalibrationRunError("source drift") + return {"manifest": "expected", "source_commit": current_head()}, "5" * 64 + + return verify + + +def configured_run( + tmp_path: Path, + *, + records: Sequence[dict[str, object]] | None = None, + stability_passed: bool = True, + source_fail_on_call: int | None = None, + decode_error: BaseException | None = None, +) -> tuple[Any, Any, Any, Any]: + selected_records = list(records or [record()]) + files = {"config.json": b"{}", "model.safetensors": b"safe-test-placeholder"} + source_bytes = runner.canonical_json_bytes( + {"manifest": "expected", "source_commit": current_head()} + ) + model_bytes = model_manifest_bytes(files) + runtime_bytes = runtime_manifest_bytes() + parquet_bytes = b'{"artifact_kind":"test-parquet-materializations"}\n' + frozen = identity( + selected_records, + source_manifest_sha256=digest(source_bytes), + runtime_manifest_sha256=digest(runtime_bytes), + model_manifest_sha256=digest(model_bytes), + parquet_manifest_sha256=digest(parquet_bytes), + ) + model_root = tmp_path / "model" + write_model_root(model_root, files) + events: list[str] = [] + backend = FakeBackend( + frozen, + events, + stability_passed=stability_passed, + decode_error=decode_error, + ) + sequence_map = { + str(item["canonical_id"]): materialized( + item, + tuple(range(1, int(item["sequence_length"]) + 1)), + ) + for item in selected_records + } + adapter = FakeAdapter(sequence_map, events) + + def authenticate(root: Path, manifest: Any) -> Any: + events.append("authenticate_model_files") + return runner.authenticate_local_model_files(root, manifest, calibration_api=api) + + def authenticate_runtime(manifest: Any) -> Any: + events.append("authenticate_runtime") + return runner.AuthenticatedRuntime( + manifest_file_sha256=manifest.file_sha256, + python_implementation=manifest.python_implementation, + python_version=manifest.python_version, + python_cache_tag=manifest.python_cache_tag, + interpreter_sha256=manifest.interpreter_sha256, + machine_name=manifest.machine_name, + base_runtime_file_count=len(manifest.runtime_trees[0].files), + package_root_count=len(manifest.package_roots), + distributions=(("transformers", "5.14.1"),), + distribution_count=1, + file_count=sum(len(tree.files) for tree in manifest.runtime_trees), + ) + + services = runner.RunnerServices( + backend=backend, + calibration_api=api, + verify_repository_source=source_verifier(events, fail_on_call=source_fail_on_call), + validate_adapter=lambda _adapter: events.append("validate_adapter"), + distortion_function=fake_distortions, + authenticate_model_files=authenticate, + authenticate_runtime=authenticate_runtime, + ) + config = runner.CalibrationRunConfig( + frozen_identity_bytes=b"identity", + repository_source_manifest_bytes=source_bytes, + model_file_manifest_bytes=model_bytes, + parquet_materialization_manifest_bytes=parquet_bytes, + runtime_manifest_bytes=runtime_bytes, + model_root=model_root, + repository_root=SCRIPT.parents[1], + expected_source_commit=current_head(), + expected_model_file_manifest_sha256=digest(model_bytes), + expected_parquet_materialization_manifest_sha256=digest(parquet_bytes), + expected_runtime_manifest_sha256=digest(runtime_bytes), + output_dir=tmp_path / "output", + require_cuda=False, + ) + return config, adapter, services, events + + +def test_success_authenticates_every_boundary_and_publishes_complete_set(tmp_path: Path) -> None: + config, adapter, services, events = configured_run(tmp_path) + + result = runner.run_calibration(config, adapter, services=services) + + assert result["status"] == "passed" + assert set(path.name for path in config.output_dir.iterdir()) == { + runner.SCORE_FILENAME, + runner.SPLIT_FILENAME, + runner.K27030_FILENAME, + runner.K29334_FILENAME, + runner.Q48_FILENAME, + runner.BINDING_FILENAME, + runner.REPORT_FILENAME, + runner.COMPLETE_FILENAME, + } + assert events[0] == "decode_identity" + assert events.index("verify_source") < events.index("validate_adapter") + assert events.index("validate_adapter") < events.index("authenticate_runtime") + assert events.index("authenticate_runtime") < events.index("materialize_sequence") + assert events.index("materialize_sequence") < events.index("authenticate_model_files") + assert events.index("authenticate_model_files") < events.index("load_model") + assert events.count("verify_source") == 3 + assert events.count("authenticate_runtime") == 4 + assert events[-6:] == [ + "runtime_metadata", + "close_model", + "authenticate_model_files", + "verify_source", + "authenticate_runtime", + "authenticate_model_files", + ] + assert adapter.closed + report = json.loads((config.output_dir / runner.REPORT_FILENAME).read_text()) + assert report["evidence"]["status"] == "passed" + assert report["evidence"]["calibration"] == { + "anchor_count": 3, + "sequence_count": 1, + "token_count": 3, + } + assert report["evidence"]["runtime"]["adapter"]["model_open"] is True + assert report["evidence"]["identity"]["execution_bindings"] == { + "calibration_runtime_manifest_file_sha256": digest(config.runtime_manifest_bytes), + "model_file_manifest_file_sha256": digest(config.model_file_manifest_bytes), + "parquet_materialization_manifest_file_sha256": digest( + config.parquet_materialization_manifest_bytes + ), + "repository_source_manifest_file_sha256": digest(config.repository_source_manifest_bytes), + } + + +def test_identity_view_consumes_strict_schema_v4_execution_bindings( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + bindings = { + "calibration_runtime_manifest_file_sha256": "1" * 64, + "model_file_manifest_file_sha256": "2" * 64, + "parquet_materialization_manifest_file_sha256": "9" * 64, + "repository_source_manifest_file_sha256": "3" * 64, + } + evidence = { + "execution_bindings": bindings, + "model_contracts": {"primary": {"id": "example/model", "revision": "4" * 40}}, + "source_manifest_sha256": "5" * 64, + "tokenizer": {"transformers_version": "5.14.1"}, + } + payload = runner.canonical_json_bytes({"evidence": evidence}) + + class Decoded: + file_sha256 = digest(payload) + canonical_evidence_sha256 = "6" * 64 + records = () + assignment = () + assignment_sha256 = "7" * 64 + tokenizer_manifest_sha256 = "8" * 64 + execution_bindings = bindings + + class Resolver: + @staticmethod + def deserialize_frozen_calibration_identity_artifact(data: bytes) -> Decoded: + assert data == payload + return Decoded() + + monkeypatch.setattr(runner, "_load_identity_resolver", lambda _root: Resolver()) + decoded = runner._identity_view(payload, tmp_path) + + assert decoded.repository_source_manifest_file_sha256 == "3" * 64 + assert decoded.runtime_manifest_file_sha256 == "1" * 64 + assert decoded.model_file_manifest_file_sha256 == "2" * 64 + assert decoded.parquet_materialization_manifest_file_sha256 == "9" * 64 + + +def test_failed_stability_publishes_only_report_and_never_binding(tmp_path: Path) -> None: + config, adapter, services, _events = configured_run(tmp_path, stability_passed=False) + + with pytest.raises(runner.CalibrationStabilityFailure, match="stability gate failed"): + runner.run_calibration(config, adapter, services=services) + + assert {path.name for path in config.output_dir.iterdir()} == { + runner.REPORT_FILENAME, + runner.COMPLETE_FILENAME, + } + report = json.loads((config.output_dir / runner.REPORT_FILENAME).read_text()) + assert report["evidence"]["status"] == "stability_failed" + assert report["evidence"]["artifacts"] == {} + assert adapter.closed + + +def test_identity_decode_is_first_and_failure_touches_no_other_boundary(tmp_path: Path) -> None: + error = ValueError("identity invalid") + config, _adapter, services, events = configured_run(tmp_path, decode_error=error) + config = replace( + config, + repository_root=tmp_path / "does-not-exist", + model_root=tmp_path / "also-missing", + ) + + with pytest.raises(ValueError, match="identity invalid"): + runner.run_calibration(config, FakeAdapter({}, events), services=services) + + assert events == ["decode_identity"] + assert not config.output_dir.exists() + + +def test_existing_output_stops_after_identity_before_source_or_data(tmp_path: Path) -> None: + config, adapter, services, events = configured_run(tmp_path) + config.output_dir.mkdir() + + with pytest.raises(FileExistsError, match="refusing to overwrite"): + runner.run_calibration(config, adapter, services=services) + + assert events == ["decode_identity"] + + +def test_source_drift_stops_before_dataset_and_model_access(tmp_path: Path) -> None: + config, adapter, services, events = configured_run(tmp_path, source_fail_on_call=1) + + with pytest.raises(runner.CalibrationRunError, match="source drift"): + runner.run_calibration(config, adapter, services=services) + + assert events == ["decode_identity", "verify_source"] + assert not config.output_dir.exists() + + +def test_source_manifest_commit_must_equal_reported_head(tmp_path: Path) -> None: + config, adapter, services, events = configured_run(tmp_path) + + def wrong_commit(expected: Mapping[str, object], root: Path) -> Any: + del root + events.append("verify_source") + return {**expected, "source_commit": "0" * 40}, "5" * 64 + + services = replace(services, verify_repository_source=wrong_commit) + with pytest.raises(runner.CalibrationRunError, match="must equal"): + runner.run_calibration(config, adapter, services=services) + + assert "materialize_sequence" not in events + + +def test_source_manifest_exact_bytes_are_identity_bound_before_verifier(tmp_path: Path) -> None: + config, adapter, services, events = configured_run(tmp_path) + config = replace(config, repository_source_manifest_bytes=b'{"manifest":"changed"}\n') + + with pytest.raises(runner.CalibrationRunError, match="frozen identity binding"): + runner.run_calibration(config, adapter, services=services) + + assert events == ["decode_identity"] + + +def test_materialized_token_mismatch_stops_before_model_file_open_or_load(tmp_path: Path) -> None: + config, adapter, services, events = configured_run(tmp_path) + item = services.backend.frozen_identity.records[0] + adapter.sequence_by_id["item-1"] = materialized(item, (9, 9, 9)) + config.model_root.joinpath("model.safetensors").unlink() + + with pytest.raises(runner.CalibrationRunError, match="token IDs differ"): + runner.run_calibration(config, adapter, services=services) + + assert "authenticate_model_files" not in events + assert "load_model" not in events + + +def test_model_manifest_commitment_is_checked_before_data_access(tmp_path: Path) -> None: + config, adapter, services, events = configured_run(tmp_path) + config = replace(config, expected_model_file_manifest_sha256="9" * 64) + + with pytest.raises(runner.CalibrationRunError, match="identity/config binding"): + runner.run_calibration(config, adapter, services=services) + + assert "validate_adapter" in events + assert "materialize_sequence" not in events + assert "authenticate_model_files" not in events + assert "load_model" not in events + + +def test_parquet_manifest_commitment_is_checked_before_data_access(tmp_path: Path) -> None: + config, adapter, services, events = configured_run(tmp_path) + config = replace( + config, + expected_parquet_materialization_manifest_sha256="9" * 64, + ) + + with pytest.raises(runner.CalibrationRunError, match="parquet materialization manifest"): + runner.run_calibration(config, adapter, services=services) + + assert "materialize_sequence" not in events + assert "authenticate_model_files" not in events + assert "load_model" not in events + + +def test_runtime_manifest_commitment_and_authentication_precede_data(tmp_path: Path) -> None: + config, adapter, services, events = configured_run(tmp_path) + config = replace(config, expected_runtime_manifest_sha256="9" * 64) + + with pytest.raises(runner.CalibrationRunError, match="runtime manifest bytes"): + runner.run_calibration(config, adapter, services=services) + + assert "authenticate_runtime" not in events + assert "materialize_sequence" not in events + assert "load_model" not in events + + +def test_point_used_runtime_drift_stops_before_data_access(tmp_path: Path) -> None: + config, adapter, services, events = configured_run(tmp_path) + original = services.authenticate_runtime + calls = 0 + + def drifting(manifest: Any) -> Any: + nonlocal calls + calls += 1 + authenticated = original(manifest) + return ( + replace(authenticated, machine_name="drifted") + if calls == 2 + else authenticated + ) + + services = replace(services, authenticate_runtime=drifting) + with pytest.raises(runner.CalibrationRunError, match="before data access"): + runner.run_calibration(config, adapter, services=services) + + assert "materialize_sequence" not in events + assert "load_model" not in events + + +def test_empty_calibration_target_is_hash_checked_and_allowed() -> None: + token_ids = (10, 11, 12) + item = record(token_ids) + span = dict(item["token_span"]) + span["prefill_stop"] = len(token_ids) + span["scored_start"] = len(token_ids) + item["token_span"] = span + item["prompt_token_ids_sha256"] = token_digest(token_ids) + item["target_token_ids_sha256"] = token_digest(()) + candidate = materialized(item, token_ids) + + assert runner.validate_materialized_sequence(item, candidate, calibration_api=api) == token_ids + + +def test_exact_local_model_file_mismatch_stops_before_model_load(tmp_path: Path) -> None: + config, adapter, services, events = configured_run(tmp_path) + config.model_root.joinpath("model.safetensors").write_bytes(b"tampered") + + with pytest.raises(runner.CalibrationRunError, match="authentication failed"): + runner.run_calibration(config, adapter, services=services) + + assert "authenticate_model_files" in events + assert "load_model" not in events + + +def test_model_files_are_reauthenticated_immediately_after_load(tmp_path: Path) -> None: + config, adapter, services, events = configured_run(tmp_path) + original_load = adapter.load_model + + def mutating_load(authenticated: Any) -> object: + model = original_load(authenticated) + config.model_root.joinpath("model.safetensors").write_bytes(b"changed-after-auth") + return model + + adapter.load_model = mutating_load # type: ignore[method-assign] + with pytest.raises(runner.CalibrationRunError, match="authentication failed"): + runner.run_calibration(config, adapter, services=services) + + assert "load_model" in events + assert "step_token" not in events + assert adapter.closed + + +def test_model_authentication_rejects_reparse_or_symlink_entries( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + files = {"config.json": b"{}", "model.safetensors": b"weights"} + root = tmp_path / "model" + write_model_root(root, files) + manifest = runner.parse_model_file_manifest(model_manifest_bytes(files)) + real_check = runner._is_link_or_reparse + + def selected_link(path: Path) -> bool: + return path.name == "model.safetensors" or real_check(path) + + monkeypatch.setattr(runner, "_is_link_or_reparse", selected_link) + with pytest.raises(runner.CalibrationRunError, match="link or reparse"): + runner.authenticate_local_model_files(root, manifest, calibration_api=api) + + +def test_second_source_verification_stops_before_model_file_authentication(tmp_path: Path) -> None: + config, adapter, services, events = configured_run(tmp_path, source_fail_on_call=2) + + with pytest.raises(runner.CalibrationRunError, match="source drift"): + runner.run_calibration(config, adapter, services=services) + + assert events.count("materialize_sequence") == 1 + assert "authenticate_model_files" not in events + assert "load_model" not in events + + +def test_bad_kernel_receipt_closes_model_and_publishes_nothing(tmp_path: Path) -> None: + config, adapter, services, events = configured_run(tmp_path) + adapter.invalid_kernel_receipt = True + + with pytest.raises(runner.CalibrationRunError, match="one successful causal kernel"): + runner.run_calibration(config, adapter, services=services) + + assert events[-2:] == ["end_sequence", "close_model"] + assert adapter.closed + assert not config.output_dir.exists() + + +def test_capture_uses_frozen_ema_and_only_requests_state_at_anchors() -> None: + geometry = runner.Geometry(layer_indices=(0,), heads=1, key_rows=4, value_width=4) + tokens = tuple(range(17)) + item = record(tokens) + events: list[str] = [] + + class QueryAdapter(FakeAdapter): + def step_token( + self, + model: object, + *, + token_id: int, + position: int, + capture_state: bool, + ) -> Any: + del model + self.capture_flags.append(capture_state) + query = torch.zeros(1, 1, 4) + query[..., position % 4] = 1.0 + return api.StepObservation( + position=position, + token_id=token_id, + layer_indices=(0,), + recurrence_query=query, + recurrent_state=torch.ones(1, 1, 4, 4) if capture_state else None, + successful_kernel_calls_per_layer=(1,), + ) + + adapter = QueryAdapter({}, events) + captured = runner.capture_sequence_causally( + adapter, + object(), + item, + tokens, + geometry=geometry, + calibration_api=api, + require_cuda=False, + distortion_function=fake_distortions, + ) + + assert captured.anchor_positions == runner.frozen_anchor_positions(17) + assert adapter.capture_flags[15] is False + assert sum(adapter.capture_flags) == 16 + first_energy = torch.tensor( + [1.0 / (1.0 + runner.QUERY_ENERGY_EPSILON), 0.0, 0.0, 0.0], + dtype=torch.float64, + ) + expected = runner.QUERY_EMA_DECAY * torch.full((4,), 0.25, dtype=torch.float64) + expected += (1.0 - runner.QUERY_EMA_DECAY) * first_energy + torch.testing.assert_close(captured.query_energy[0, 0, 0], expected, rtol=1e-6, atol=1e-8) + assert captured.query_energy.dtype == torch.float64 + assert captured.q4_mse.shape == (16, 1, 1, 4) + + +def test_official_capture_rejects_cpu_queries_even_when_cuda_might_exist() -> None: + geometry = FakeBackend.geometry + tokens = (1,) + adapter = FakeAdapter({}, []) + + with pytest.raises(runner.CalibrationRunError, match="actual CUDA"): + runner.capture_sequence_causally( + adapter, + object(), + record(tokens), + tokens, + geometry=geometry, + calibration_api=api, + require_cuda=True, + distortion_function=fake_distortions, + ) + + +def test_capture_rejects_non_fp32_reference_state() -> None: + geometry = runner.Geometry(layer_indices=(0,), heads=1, key_rows=4, value_width=4) + + class Bf16Adapter(FakeAdapter): + def step_token(self, model: object, **kwargs: object) -> Any: + del model + return api.StepObservation( + position=kwargs["position"], + token_id=kwargs["token_id"], + layer_indices=(0,), + recurrence_query=torch.ones(1, 1, 4), + recurrent_state=torch.ones(1, 1, 4, 4, dtype=torch.bfloat16), + successful_kernel_calls_per_layer=(1,), + ) + + with pytest.raises(runner.CalibrationRunError, match="must be FP32"): + runner.capture_sequence_causally( + Bf16Adapter({}, []), + object(), + record((1,)), + (1,), + geometry=geometry, + calibration_api=api, + require_cuda=False, + distortion_function=fake_distortions, + ) + + +def test_compute_anchor_distortions_is_cpu_fp64_and_precision_ordered() -> None: + geometry = runner.Geometry(layer_indices=(3,), heads=1, key_rows=2, value_width=4) + state = torch.tensor( + [[[[1.2, -0.1, 0.4, 2.3], [0.2, 0.7, -1.4, 0.5]]]], + dtype=torch.float32, + ) + + d4, d6, d8 = runner.compute_anchor_distortions(state, geometry) + + for tensor in (d4, d6, d8): + assert tensor.shape == (1, 1, 2) + assert tensor.device.type == "cpu" + assert tensor.dtype == torch.float64 + assert torch.isfinite(tensor).all() + assert (tensor >= 0).all() + assert torch.all(d8 <= d4) + + +def test_model_manifest_rejects_noncanonical_duplicate_and_traversal() -> None: + files = {"config.json": b"{}", "model.safetensors": b"weights"} + valid = model_manifest_bytes(files) + parsed = runner.parse_model_file_manifest(valid) + assert parsed.file_sha256 == digest(valid) + assert [item.name for item in parsed.files] == ["config.json", "model.safetensors"] + + with pytest.raises(ValueError, match="canonical"): + runner.parse_model_file_manifest(valid.rstrip(b"\n") + b" \n") + duplicate = valid.replace( + b'"model_id":"example/model"', b'"model_id":"x","model_id":"example/model"' + ) + with pytest.raises(ValueError, match="duplicate JSON key"): + runner.parse_model_file_manifest(duplicate) + payload = json.loads(valid) + payload["files"][0]["name"] = "../config.json" + with pytest.raises(ValueError, match="name is invalid|relative POSIX"): + runner.parse_model_file_manifest(runner.canonical_json_bytes(payload)) + + +def hub_tree_entries() -> list[dict[str, object]]: + return [ + { + "blob_id": "1" * 40, + "lfs": None, + "path": "config.json", + "size": 123, + }, + { + "blob_id": "2" * 40, + "lfs": {"sha256": "a" * 64, "size": 1_024}, + "path": "model.safetensors", + "size": 1_024, + }, + { + "blob_id": "3" * 40, + "lfs": None, + "path": "README.md", + "size": 50, + }, + ] + + +def test_model_manifest_capture_uses_only_pinned_hub_tree_lfs_metadata() -> None: + revision = "4" * 40 + payload = runner.capture_model_file_manifest_from_hub( + "example/model", + revision, + transformers_version="5.14.1", + tree_entries=hub_tree_entries(), + resolved_revision=revision, + ) + parsed = runner.parse_model_file_manifest(payload) + + assert [item.name for item in parsed.files] == ["config.json", "model.safetensors"] + assert parsed.files[0].sha256 is None + assert parsed.files[0].git_blob_oid == "1" * 40 + assert parsed.files[1].sha256 == "a" * 64 + assert parsed.files[1].lfs_sha256 == "a" * 64 + assert b"README.md" not in payload + + +def test_model_manifest_accepts_the_pinned_qwen35_weight_filename() -> None: + revision = "4" * 40 + weight_name = "model.safetensors-00001-of-00001.safetensors" + entries = [ + hub_tree_entries()[0], + { + "blob_id": "2" * 40, + "lfs": {"sha256": "a" * 64, "size": 1_024}, + "path": weight_name, + "size": 1_024, + }, + { + "blob_id": "3" * 40, + "lfs": None, + "path": "model.safetensors.index.json", + "size": 321, + }, + ] + + payload = runner.capture_model_file_manifest_from_hub( + "Qwen/Qwen3.5-0.8B-Base", + revision, + transformers_version="5.14.1", + tree_entries=entries, + resolved_revision=revision, + ) + parsed = runner.parse_model_file_manifest(payload) + + assert [item.name for item in parsed.files] == [ + "config.json", + "model.safetensors-00001-of-00001.safetensors", + "model.safetensors.index.json", + ] + assert parsed.files[1].lfs_sha256 == "a" * 64 + + +def test_model_manifest_capture_calls_only_metadata_api_surfaces() -> None: + revision = "4" * 40 + calls: list[str] = [] + + class Info: + sha = revision + + class Api: + @staticmethod + def model_info(*args: object, **kwargs: object) -> Info: + del args, kwargs + calls.append("model_info") + return Info() + + @staticmethod + def list_repo_tree(*args: object, **kwargs: object) -> list[dict[str, object]]: + del args, kwargs + calls.append("list_repo_tree") + return hub_tree_entries() + + runner.capture_model_file_manifest_from_hub( + "example/model", + revision, + transformers_version="5.14.1", + api=Api(), + ) + + assert calls == ["model_info", "list_repo_tree"] + + +@pytest.mark.parametrize( + ("mutator", "message"), + [ + ( + lambda entries: entries.append(dict(entries[0])), + "duplicate path", + ), + ( + lambda entries: entries.append( + {"blob_id": "5" * 40, "lfs": None, "path": "../escape", "size": 1} + ), + "repository-relative", + ), + ( + lambda entries: entries[1].update({"lfs": None}), + "lacks pinned LFS", + ), + ( + lambda entries: entries[1].update({"lfs": {"sha256": "a" * 64, "size": 999}}), + "differs from LFS", + ), + ], +) +def test_model_manifest_capture_rejects_malformed_hub_metadata( + mutator: Any, + message: str, +) -> None: + entries = hub_tree_entries() + mutator(entries) + + with pytest.raises(ValueError, match=message): + runner.capture_model_file_manifest_from_hub( + "example/model", + "4" * 40, + transformers_version="5.14.1", + tree_entries=entries, + resolved_revision="4" * 40, + ) + + +def test_model_manifest_parser_detects_tree_metadata_tamper() -> None: + payload = runner.capture_model_file_manifest_from_hub( + "example/model", + "4" * 40, + transformers_version="5.14.1", + tree_entries=hub_tree_entries(), + resolved_revision="4" * 40, + ) + document = json.loads(payload) + document["files"][0]["git_blob_oid"] = "9" * 40 + + with pytest.raises(ValueError, match="tree metadata manifest"): + runner.parse_model_file_manifest(runner.canonical_json_bytes(document)) + + +def test_model_authentication_rejects_extra_files(tmp_path: Path) -> None: + files = {"config.json": b"{}", "model.safetensors": b"weights"} + root = tmp_path / "model" + write_model_root(root, {**files, "unbound.txt": b"extra"}) + manifest = runner.parse_model_file_manifest(model_manifest_bytes(files)) + + with pytest.raises(runner.CalibrationRunError, match="file set differs"): + runner.authenticate_local_model_files(root, manifest, calibration_api=api) + + +def test_runtime_manifest_hashes_complete_record_inventory(tmp_path: Path) -> None: + base_root = tmp_path / "base" + package_root = tmp_path / "packages" + import_root = package_root / "Lib" / "site-packages" + interpreter = base_root / "python.exe" + stdlib_file = base_root / "Lib" / "os.py" + first = import_root / "package" / "__init__.py" + second = import_root / "package-1.0.dist-info" / "RECORD" + interpreter.parent.mkdir(parents=True) + stdlib_file.parent.mkdir(parents=True) + first.parent.mkdir(parents=True) + second.parent.mkdir(parents=True) + interpreter.write_bytes(b"fake-python") + stdlib_file.write_bytes(b"stdlib") + first.write_bytes(b"VALUE = 1\n") + second.write_text( + "package/__init__.py,,\npackage-1.0.dist-info/RECORD,,\n", + encoding="utf-8", + ) + + class Distribution: + metadata = {"Name": "Package"} + version = "1.0" + files = ("package/__init__.py", "package-1.0.dist-info/RECORD") + + @staticmethod + def locate_file(path: object) -> Path: + return import_root / str(path) + + @staticmethod + def read_text(name: str) -> str | None: + return second.read_text(encoding="utf-8") if name == "RECORD" else None + + distribution = Distribution() + payload = runner.capture_calibration_runtime_manifest( + base_runtime_root=base_root, + base_sys_path=("Lib",), + distributions=[distribution], + interpreter_path=interpreter, + package_import_paths={"packages": "Lib/site-packages"}, + package_roots={"packages": package_root}, + ) + parsed = runner.parse_calibration_runtime_manifest(payload) + authenticated = runner.authenticate_calibration_runtime( + parsed, + base_runtime_root=base_root, + distributions=[distribution], + interpreter_path=interpreter, + package_roots={"packages": package_root}, + ) + + assert authenticated.distributions == (("package", "1.0"),) + assert authenticated.file_count == 4 + first.write_bytes(b"VALUE = 2\n") + with pytest.raises(runner.CalibrationRunError, match="tree identity drifted"): + runner.authenticate_calibration_runtime( + parsed, + base_runtime_root=base_root, + distributions=[distribution], + interpreter_path=interpreter, + package_roots={"packages": package_root}, + ) + + +@pytest.mark.parametrize("unsafe", ["../escape.py", "C:/escape.py", "/escape.py"]) +def test_runtime_manifest_rejects_unsafe_serialized_paths(unsafe: str) -> None: + payload = json.loads(runtime_manifest_bytes()) + payload["runtime_trees"][1]["files"][0]["path"] = unsafe + + with pytest.raises((ValueError, runner.CalibrationRunError), match="repository-relative"): + runner.parse_calibration_runtime_manifest(runner.canonical_json_bytes(payload)) + + +@pytest.mark.parametrize( + "unsafe", + [ + "Lib/site-packages/startup.pth", + "Lib/site-packages/module.pyc", + "Lib/site-packages/__pycache__/module.py", + ], +) +def test_runtime_manifest_rejects_startup_hooks_and_bytecode(unsafe: str) -> None: + payload = json.loads(runtime_manifest_bytes()) + payload["runtime_trees"][1]["files"][0]["path"] = unsafe + + with pytest.raises(ValueError, match="forbidden"): + runner.parse_calibration_runtime_manifest(runner.canonical_json_bytes(payload)) + + +def test_runtime_manifest_rejects_unrecorded_importable_package_file() -> None: + payload = json.loads(runtime_manifest_bytes()) + payload["runtime_trees"][1]["files"].append( + { + "path": "Lib/site-packages/unrecorded.py", + "sha256": "e" * 64, + "size_bytes": 1, + } + ) + + with pytest.raises(ValueError, match="distribution inventory"): + runner.parse_calibration_runtime_manifest(runner.canonical_json_bytes(payload)) + + +def test_runtime_tree_rejects_link_or_reparse_entry( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "tree" + root.mkdir() + linked = root / "module.py" + linked.write_bytes(b"value = 1\n") + real_check = runner._is_link_or_reparse + + def selected_link(path: Path) -> bool: + return path.name == "module.py" or real_check(path) + + monkeypatch.setattr(runner, "_is_link_or_reparse", selected_link) + with pytest.raises(runner.CalibrationRunError, match="link or reparse"): + runner._runtime_tree_files(root, kind="packages") + + +def test_nonempty_pycache_prefix_is_rejected(tmp_path: Path) -> None: + root = tmp_path / "pycache" + root.mkdir() + root.joinpath("injected.pyc").write_bytes(b"bytecode") + + with pytest.raises(runner.CalibrationRunError, match="not empty"): + runner._verify_empty_pycache_prefix(root) + + +def test_base_runtime_staging_omits_bytecode_hooks_and_global_packages(tmp_path: Path) -> None: + source = tmp_path / "source" + destination = tmp_path / "destination" + for relative, content in { + "python.exe": b"python", + "python311.dll": b"dll", + "DLLs/_ssl.pyd": b"ssl", + "Lib/os.py": b"os", + "Lib/startup.pth": b"hook", + "Lib/sitecustomize.py": b"hook", + "Lib/__pycache__/os.pyc": b"bytecode", + "Lib/site-packages/untrusted.py": b"untrusted", + }.items(): + path = source / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + destination.mkdir() + + runner._copy_base_runtime(source.resolve(), destination) + + copied = { + path.relative_to(destination).as_posix() + for path in destination.rglob("*") + if path.is_file() + } + assert copied == { + "DLLs/_ssl.pyd", + "Lib/os.py", + "python.exe", + "python311.dll", + } + + +def test_record_only_staging_preserves_scripts_outside_site_packages(tmp_path: Path) -> None: + source = tmp_path / "source" + destination = tmp_path / "destination" + site = source / "Lib" / "site-packages" + package = site / "demo" / "__init__.py" + record_path = site / "demo-1.0.dist-info" / "RECORD" + script = source / "Scripts" / "demo.exe" + for path, content in ((package, b"demo"), (script, b"script")): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + record_path.parent.mkdir(parents=True, exist_ok=True) + record_text = ( + "demo/__init__.py,,\n" + "demo-1.0.dist-info/RECORD,,\n" + "../../Scripts/demo.exe,,\n" + ) + record_path.write_text(record_text, encoding="utf-8") + destination.mkdir() + + class Distribution: + metadata = {"Name": "demo"} + version = "1.0" + files = ( + "demo/__init__.py", + "demo-1.0.dist-info/RECORD", + "../../Scripts/demo.exe", + ) + + @staticmethod + def locate_file(path: object) -> Path: + return site / str(path) + + @staticmethod + def read_text(name: str) -> str | None: + return record_text if name == "RECORD" else None + + runner._copy_record_only_packages(source.resolve(), destination, [Distribution()]) + + assert (destination / "Scripts" / "demo.exe").read_bytes() == b"script" + assert not list(destination.rglob("*.pth")) + + +def test_prepare_runtime_is_strictly_no_overwrite(tmp_path: Path) -> None: + output = tmp_path / "runtime" + output.mkdir() + + with pytest.raises(FileExistsError, match="refusing to overwrite"): + runner.prepare_calibration_runtime( + source_python=tmp_path / "missing-python.exe", + requirements_file=tmp_path / "missing-requirements.txt", + output_root=output, + ) + + +def test_adapter_contract_uses_the_shared_api_identity() -> None: + events: list[str] = [] + adapter = FakeAdapter({}, events) + + runner.validate_adapter_contract(adapter, calibration_api=api) + with pytest.raises(TypeError, match="does not implement"): + runner.validate_adapter_contract(object(), calibration_api=api) + + +def test_atomic_publish_refuses_overwrite(tmp_path: Path) -> None: + path = tmp_path / "artifact.json" + runner._atomic_publish_new(path, b"first") + + with pytest.raises(FileExistsError, match="refusing to overwrite"): + runner._atomic_publish_new(path, b"second") + + assert path.read_bytes() == b"first" + + +def test_output_directory_fault_never_exposes_a_partial_final_directory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + output = tmp_path / "final" + real_publish = runner._atomic_publish_new + calls = 0 + + def fail_second(path: Path, payload: bytes) -> None: + nonlocal calls + calls += 1 + if calls == 2: + raise OSError("injected publication failure") + real_publish(path, payload) + + monkeypatch.setattr(runner, "_atomic_publish_new", fail_second) + with pytest.raises(OSError, match="injected"): + runner._publish_output_directory(output, {"a.json": b"a", "b.json": b"b"}) + + assert not output.exists() + assert not list(tmp_path.glob(".final.staging-*")) + + +def test_capture_manifest_cli_modes_are_no_overwrite( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository_root = tmp_path / "repository" + repository_root.mkdir() + source_output = tmp_path / "source.json" + runtime_output = tmp_path / "runtime.json" + model_output = tmp_path / "model.json" + source_manifest = { + "canonical_manifest_sha256": "6" * 64, + "source_commit": "4" * 40, + } + source_payload = b'{"source":"verified"}\n' + verification_calls: list[dict[str, object]] = [] + + class FakeSourceCapture: + @staticmethod + def capture_experiment013_source_manifest(root: Path) -> dict[str, object]: + assert root == repository_root + return source_manifest + + @staticmethod + def validate_experiment013_source_manifest( + manifest: Mapping[str, object], + ) -> dict[str, object]: + assert manifest is source_manifest + return source_manifest + + @staticmethod + def canonical_experiment013_source_manifest_bytes( + manifest: Mapping[str, object], + ) -> bytes: + assert manifest is source_manifest + return source_payload + + @staticmethod + def verify_experiment013_source_manifest( + manifest: Mapping[str, object], + root: Path, + ) -> dict[str, object]: + assert manifest is source_manifest + assert root == repository_root + verification_calls.append(dict(manifest)) + return source_manifest + + monkeypatch.setattr(runner, "_load_source_capture_module", lambda _root: FakeSourceCapture) + runtime_calls: list[dict[str, object]] = [] + + def runtime_capture(**kwargs: object) -> bytes: + runtime_calls.append(dict(kwargs)) + return b"runtime\n" + + monkeypatch.setattr(runner, "capture_calibration_runtime_manifest", runtime_capture) + + assert ( + runner.main( + [ + "capture-source-manifest", + "--repository-root", + str(repository_root), + "--output", + str(source_output), + ] + ) + == 0 + ) + assert source_output.read_bytes() == source_payload + assert len(verification_calls) == 2 + with pytest.raises(FileExistsError, match="refusing to overwrite"): + runner.main( + [ + "capture-source-manifest", + "--repository-root", + str(repository_root), + "--output", + str(source_output), + ] + ) + + runtime_args = [ + "capture-runtime-manifest", + "--output", + str(runtime_output), + "--base-runtime-root", + str(tmp_path / "base"), + "--staged-interpreter", + str(tmp_path / "base" / "python.exe"), + "--package-root", + f"packages={tmp_path / 'packages'}", + "--package-import-path", + "packages=Lib/site-packages", + ] + assert runner.main(runtime_args) == 0 + assert runtime_output.read_bytes() == b"runtime\n" + assert runtime_calls == [ + { + "base_runtime_root": tmp_path / "base", + "interpreter_path": tmp_path / "base" / "python.exe", + "package_import_paths": {"packages": "Lib/site-packages"}, + "package_roots": {"packages": tmp_path / "packages"}, + } + ] + with pytest.raises(FileExistsError, match="refusing to overwrite"): + runner.main(runtime_args) + + def model_capture( + model_id: str, + revision: str, + *, + transformers_version: str, + ) -> bytes: + assert (model_id, revision, transformers_version) == ( + "example/model", + "4" * 40, + "5.14.1", + ) + return b"model\n" + + monkeypatch.setattr(runner, "capture_model_file_manifest_from_hub", model_capture) + assert ( + runner.main( + [ + "capture-model-manifest", + "--output", + str(model_output), + "--model-id", + "example/model", + "--revision", + "4" * 40, + "--transformers-version", + "5.14.1", + ] + ) + == 0 + ) + assert model_output.read_bytes() == b"model\n" + + +def test_source_manifest_output_location_allows_only_external_or_ignored_paths( + tmp_path: Path, +) -> None: + repository = tmp_path / "repository" + repository.mkdir() + subprocess.run(["git", "init", "-b", "main"], cwd=repository, check=True, capture_output=True) + (repository / ".gitignore").write_text("artifacts/\n", encoding="utf-8") + + outside = tmp_path / "outside.json" + ignored = repository / "artifacts" / "source.json" + assert runner._assert_source_manifest_output_location(repository, outside) == outside.resolve() + assert runner._assert_source_manifest_output_location(repository, ignored) == ignored.resolve() + + with pytest.raises(runner.CalibrationRunError, match="must be ignored"): + runner._assert_source_manifest_output_location(repository, repository / "source.json") + with pytest.raises(runner.CalibrationRunError, match="repository metadata"): + runner._assert_source_manifest_output_location(repository, repository / ".git" / "new.json") + + +def test_public_main_binds_manifest_bytes_before_adapter_import( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_bytes = b"not-the-frozen-source\n" + runtime_bytes = b"runtime-metadata\n" + model_bytes = b"model-metadata\n" + parquet_bytes = b"parquet-metadata\n" + identity_bytes = bootstrap_identity_bytes( + source_manifest_sha256="0" * 64, + runtime_manifest_sha256=digest(runtime_bytes), + model_manifest_sha256=digest(model_bytes), + parquet_manifest_sha256=digest(parquet_bytes), + ) + paths = { + "identity": tmp_path / "identity.json", + "source": tmp_path / "source.json", + "runtime": tmp_path / "runtime.json", + "model": tmp_path / "model.json", + "parquet": tmp_path / "parquet.json", + } + paths["identity"].write_bytes(identity_bytes) + paths["source"].write_bytes(source_bytes) + paths["runtime"].write_bytes(runtime_bytes) + paths["model"].write_bytes(model_bytes) + paths["parquet"].write_bytes(parquet_bytes) + imported: list[bool] = [] + monkeypatch.setattr(runner, "_load_adapter", lambda *args, **kwargs: imported.append(True)) + + with pytest.raises(runner.CalibrationRunError, match="bootstrap binding"): + runner._official_main( + [ + "--frozen-identity", + str(paths["identity"]), + "--repository-source-manifest", + str(paths["source"]), + "--runtime-manifest", + str(paths["runtime"]), + "--expected-runtime-manifest-sha256", + digest(runtime_bytes), + "--model-file-manifest", + str(paths["model"]), + "--expected-model-file-manifest-sha256", + digest(model_bytes), + "--parquet-materialization-manifest", + str(paths["parquet"]), + "--expected-parquet-materialization-manifest-sha256", + digest(parquet_bytes), + "--model-root", + str(tmp_path / "unopened-model"), + "--cache-root", + str(tmp_path / "unopened-cache"), + "--ruler-root", + str(tmp_path / "unopened-ruler"), + "--repository-root", + str(tmp_path / "unopened-repository"), + "--source-commit", + "1" * 40, + "--output-dir", + str(tmp_path / "output"), + ], + runtime_context=runner.SealedRuntimeContext( + manifest_file_sha256=digest(runtime_bytes), + base_runtime_root=tmp_path / "base", + package_roots={"packages": tmp_path / "packages"}, + package_import_paths={"packages": "Lib/site-packages"}, + pycache_prefix=tmp_path / "pycache", + ), + interpreter_path=tmp_path / "base" / "python.exe", + ) + + assert imported == [] + assert not (tmp_path / "unopened-model").exists() + + +def test_public_main_rejects_unsealed_official_execution() -> None: + with pytest.raises(runner.CalibrationRunError, match="launch_static_q468"): + runner.main(["--frozen-identity", "identity.json"]) + + +def test_official_loader_rejects_generic_and_preloaded_adapters() -> None: + kwargs = { + "repository_root": SCRIPT.parents[1], + "source_entry": {}, + "calibration_api": api, + "context": object(), + } + with pytest.raises(runner.CalibrationRunError, match="requires exactly"): + runner._load_adapter("untrusted.module:factory", **kwargs) + + sys.modules[runner.CANONICAL_ADAPTER_MODULE] = object() # type: ignore[assignment] + try: + with pytest.raises(runner.CalibrationRunError, match="preloaded reviewed adapter"): + runner._load_adapter(runner.CANONICAL_ADAPTER_SPEC, **kwargs) + finally: + sys.modules.pop(runner.CANONICAL_ADAPTER_MODULE, None) + + +def test_default_services_do_not_eagerly_import_static_calibration_modules() -> None: + code = f""" +import importlib.util +import sys +from pathlib import Path +script = Path({str(SCRIPT)!r}) +spec = importlib.util.spec_from_file_location('isolated_runner', script) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) +assert 'torch' not in sys.modules +assert 'recurquant.static_q468' not in sys.modules +api_script = script.parents[1] / 'src' / 'recurquant' / 'experiment013_calibration_api.py' +api_spec = importlib.util.spec_from_file_location('isolated_calibration_api', api_script) +api_module = importlib.util.module_from_spec(api_spec) +sys.modules[api_spec.name] = api_module +api_spec.loader.exec_module(api_module) +module.default_services( + script.parents[1], + base_runtime_root=script.parents[1] / 'base', + calibration_api=api_module, + interpreter_path=script.parents[1] / 'base' / 'python.exe', + package_roots={{'packages': script.parents[1] / 'packages'}}, +) +assert 'recurquant.static_q468' not in sys.modules +assert 'recurquant.static_q468_calibration' not in sys.modules +""" + subprocess.run( + [str(SCRIPT.parents[1] / ".venv" / "Scripts" / "python.exe"), "-c", code], + cwd=SCRIPT.parents[1], + check=True, + capture_output=True, + text=True, + timeout=60, + ) diff --git a/tests/test_static_q468.py b/tests/test_static_q468.py index 009a968..fd049be 100644 --- a/tests/test_static_q468.py +++ b/tests/test_static_q468.py @@ -18,6 +18,7 @@ STATIC_Q468_ABLATION_METHOD, STATIC_Q468_PRIMARY_METHOD, StaticPackedRhtQ48State, + StaticPackedRhtQ468State, StaticRhtQ468Geometry, allocate_exact_q48_mask, build_static_rht_q48_policy, @@ -752,6 +753,68 @@ def test_real_geometry_primary_policy_has_exact_k_counts_and_uint16_offsets() -> replace(policy, model_id="Qwen/other-model") +def test_real_q468_policy_has_a_physical_exact_3454664_byte_state() -> None: + """Construct every real Q4/Q6/Q8 pool without allocating dense model state.""" + + geometry = FROZEN_QWEN35_STATIC_Q468_GEOMETRY + row = torch.arange(geometry.total_rows, dtype=torch.float64) + distortions = ( + (((17 * row + 13) % 1009) / 1009).reshape(geometry.layers, -1), + (((29 * row + 7) % 1013) / 1013).reshape(geometry.layers, -1), + (((43 * row + 3) % 1019) / 1019).reshape(geometry.layers, -1), + ) + policy = build_static_rht_q468_policy( + *distortions, + geometry=geometry, + marginal_steps=FROZEN_STATIC_Q468_PRIMARY_STEPS, + method_id=STATIC_Q468_PRIMARY_METHOD, + calibration_manifest_sha256=MANIFEST_SHA256, + **BINDINGS, + ) + q4_count, q6_count, q8_count = policy.pool_counts + packed = StaticPackedRhtQ468State( + policy=policy, + int4_payload=torch.zeros( + (q4_count, geometry.value_width * 4 // 8), dtype=torch.uint8 + ), + int6_payload=torch.zeros( + (q6_count, geometry.value_width * 6 // 8), dtype=torch.uint8 + ), + int8_payload=torch.zeros((q8_count, geometry.value_width), dtype=torch.int8), + scales=torch.ones(geometry.total_rows, dtype=torch.float16), + padding=torch.zeros(8, dtype=torch.uint8), + ) + + evidence = verify_static_packed_rht_q468(packed) + assert q4_count + q6_count + q8_count == 36_864 + assert ( + packed.int4_payload.numel() + + packed.int6_payload.numel() + + packed.int8_payload.numel() + == 3_297_984 + ) + assert policy.packed_precision_codes.numel() == 9_216 + assert policy.pool_offsets.numel() * policy.pool_offsets.element_size() == 73_728 + assert packed.scales.numel() * packed.scales.element_size() == 73_728 + assert packed.data_bytes == 3_454_656 + assert packed.resident_bytes == 3_454_664 + assert evidence["physical_resident_bytes"] == 3_454_664 + + source_states = { + layer_index: torch.zeros( + (1, geometry.heads, geometry.key_rows, geometry.value_width), + dtype=torch.float32, + ) + for layer_index in geometry.layer_indices + } + with torch.no_grad(): + physically_packed = pack_static_rht_q468(source_states, policy) + physical_evidence = verify_static_packed_rht_q468(physically_packed) + assert physically_packed.policy.pool_counts == packed.policy.pool_counts + assert physically_packed.resident_bytes == 3_454_664 + assert physical_evidence["physical_resident_bytes"] == 3_454_664 + + def test_real_q48_policy_has_a_physical_exact_3454664_byte_state() -> None: geometry = FROZEN_QWEN35_STATIC_Q468_GEOMETRY row = torch.arange(geometry.total_rows, dtype=torch.float64) From ceab2ce3001725d5f48544a2a98f9e64e759ca09 Mon Sep 17 00:00:00 2001 From: HiddenTrojan Date: Sun, 2 Aug 2026 17:10:52 +0800 Subject: [PATCH 12/24] Fix Experiment 013 cross-platform CI closure --- scripts/capture_statelease_stage0.py | 4 ++++ scripts/screen_statelease_stage_a.py | 4 ++++ scripts/verify_statelease_stage0.py | 4 ++++ tests/test_experiment013_calibration_api.py | 3 ++- tests/test_run_static_q468_calibration.py | 2 +- 5 files changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/capture_statelease_stage0.py b/scripts/capture_statelease_stage0.py index ff88df4..5d8714f 100644 --- a/scripts/capture_statelease_stage0.py +++ b/scripts/capture_statelease_stage0.py @@ -112,6 +112,10 @@ "src/recurquant/confirmation.py", "src/recurquant/evaluation.py", "src/recurquant/evidence.py", + "src/recurquant/experiment013_calibration_api.py", + "src/recurquant/experiment013_parquet.py", + "src/recurquant/experiment013_qwen35_adapter.py", + "src/recurquant/experiment013_source.py", "src/recurquant/finite_difference.py", "src/recurquant/fisher_sensitivity.py", "src/recurquant/horizon.py", diff --git a/scripts/screen_statelease_stage_a.py b/scripts/screen_statelease_stage_a.py index 626e423..5297abe 100644 --- a/scripts/screen_statelease_stage_a.py +++ b/scripts/screen_statelease_stage_a.py @@ -381,6 +381,10 @@ "src/recurquant/confirmation.py", "src/recurquant/evaluation.py", "src/recurquant/evidence.py", + "src/recurquant/experiment013_calibration_api.py", + "src/recurquant/experiment013_parquet.py", + "src/recurquant/experiment013_qwen35_adapter.py", + "src/recurquant/experiment013_source.py", "src/recurquant/finite_difference.py", "src/recurquant/fisher_sensitivity.py", "src/recurquant/horizon.py", diff --git a/scripts/verify_statelease_stage0.py b/scripts/verify_statelease_stage0.py index c41be40..fdbe380 100644 --- a/scripts/verify_statelease_stage0.py +++ b/scripts/verify_statelease_stage0.py @@ -157,6 +157,10 @@ "src/recurquant/confirmation.py", "src/recurquant/evaluation.py", "src/recurquant/evidence.py", + "src/recurquant/experiment013_calibration_api.py", + "src/recurquant/experiment013_parquet.py", + "src/recurquant/experiment013_qwen35_adapter.py", + "src/recurquant/experiment013_source.py", "src/recurquant/finite_difference.py", "src/recurquant/fisher_sensitivity.py", "src/recurquant/horizon.py", diff --git a/tests/test_experiment013_calibration_api.py b/tests/test_experiment013_calibration_api.py index 1053ca6..0e585c7 100644 --- a/tests/test_experiment013_calibration_api.py +++ b/tests/test_experiment013_calibration_api.py @@ -1,6 +1,7 @@ from __future__ import annotations import subprocess +import sys from pathlib import Path import pytest @@ -123,7 +124,7 @@ def test_api_import_is_stdlib_only_even_when_loaded_under_external_name() -> Non assert module.AuthenticatedSequence.__module__ == 'external_experiment013_api' """ subprocess.run( - [str(ROOT / ".venv" / "Scripts" / "python.exe"), "-I", "-c", code], + [sys.executable, "-I", "-c", code], cwd=ROOT, check=True, capture_output=True, diff --git a/tests/test_run_static_q468_calibration.py b/tests/test_run_static_q468_calibration.py index 82cd9c5..e217735 100644 --- a/tests/test_run_static_q468_calibration.py +++ b/tests/test_run_static_q468_calibration.py @@ -1773,7 +1773,7 @@ def test_default_services_do_not_eagerly_import_static_calibration_modules() -> assert 'recurquant.static_q468_calibration' not in sys.modules """ subprocess.run( - [str(SCRIPT.parents[1] / ".venv" / "Scripts" / "python.exe"), "-c", code], + [sys.executable, "-c", code], cwd=SCRIPT.parents[1], check=True, capture_output=True, From 4bf5e267c080cb64d9e6bed8dae1cc7ee4c8cbf6 Mon Sep 17 00:00:00 2001 From: HiddenTrojan Date: Fri, 14 Aug 2026 22:05:11 +0800 Subject: [PATCH 13/24] Freeze Experiment 013 Stage A protocol --- ...EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md | 569 +- scripts/capture_statelease_stage0.py | 1 + scripts/capture_static_q468_identity_input.py | 1874 ++++++- .../generate_static_q468_ruler_receipts.py | 60 +- scripts/launch_static_q468_calibration.py | 461 +- scripts/launch_static_q468_stage_a.py | 822 +++ scripts/resolve_static_q468_identity.py | 766 ++- scripts/run_static_q468_calibration.py | 2002 ++++++- scripts/screen_statelease_stage_a.py | 1 + scripts/screen_static_q468_stage_a.py | 4956 +++++++++++++++++ scripts/verify_statelease_stage0.py | 1 + src/recurquant/__init__.py | 14 + .../experiment013_calibration_api.py | 33 + src/recurquant/experiment013_parquet.py | 91 +- .../experiment013_qwen35_adapter.py | 407 +- src/recurquant/experiment013_source.py | 309 +- src/recurquant/experiment013_stage_a.py | 1020 ++++ src/recurquant/static_q468.py | 78 +- src/recurquant/static_q468_cache.py | 44 +- src/recurquant/static_q468_calibration.py | 1570 +++++- ...test_capture_static_q468_identity_input.py | 917 ++- tests/test_experiment013_calibration_api.py | 8 + tests/test_experiment013_parquet.py | 71 +- tests/test_experiment013_qwen35_adapter.py | 344 +- tests/test_experiment013_source.py | 101 + tests/test_experiment013_stage_a.py | 560 ++ ...est_generate_static_q468_ruler_receipts.py | 50 + tests/test_launch_static_q468_calibration.py | 348 +- tests/test_launch_static_q468_stage_a.py | 731 +++ tests/test_resolve_static_q468_identity.py | 891 ++- tests/test_run_static_q468_calibration.py | 1027 +++- tests/test_screen_static_q468_stage_a.py | 1860 +++++++ tests/test_static_q468.py | 202 +- tests/test_static_q468_cache.py | 104 +- tests/test_static_q468_calibration.py | 734 ++- 35 files changed, 22084 insertions(+), 943 deletions(-) create mode 100644 scripts/launch_static_q468_stage_a.py create mode 100644 scripts/screen_static_q468_stage_a.py create mode 100644 src/recurquant/experiment013_stage_a.py create mode 100644 tests/test_experiment013_stage_a.py create mode 100644 tests/test_launch_static_q468_stage_a.py create mode 100644 tests/test_screen_static_q468_stage_a.py diff --git a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md index 2fc1087..12226c1 100644 --- a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md +++ b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md @@ -1,14 +1,22 @@ # Experiment 013: static RHT-Q468 packed-native adoption protocol -> **Status: amended and frozen before Experiment 013 identity resolution, policy fitting, -> protected-set access, model-weight loading, or quality measurement.** +> **Status: pre-freeze H0 candidate; not yet preregistered.** > -> Upstream revisions are frozen below. Canonical row identities, token spans, -> tokenizer-file hashes, and content hashes remain unresolved placeholders -> until a separate identity candidate is resolved and explicitly promoted. An -> identity candidate is not authorization to load model weights. +> This working copy becomes the frozen Experiment 013 preregistration only when +> its exact bytes and dependencies are committed in a clean source commit H0 and +> that H0 is bound by the pre-access seal before identity resolution, policy +> fitting, any protected materialization, model-weight loading, or quality +> measurement. A dirty or unbound working copy is not a frozen protocol. +> +> The amendment history is retained below, but its rules remain candidate rules +> until that H0 binding. Canonical row identities, token spans, tokenizer-file +> hashes, and content hashes remain unresolved placeholders until a separate +> identity candidate is resolved and explicitly promoted. An identity candidate +> is not authorization to load model weights. + +Protocol draft initiated: 2026-08-02 -Protocol frozen: 2026-08-02 +Current pre-H0 candidate amended: 2026-08-14 Pre-resolution audit amendment: 2026-08-02. The amendment corrects a cache-exposed-span off-by-one, binds the Stage-A calibration chain by exact @@ -81,11 +89,117 @@ These are canonical projection hashes, not quality results. A clean source commit, sealed calibration runtime, fresh v6 receipt batch, and promoted v4 identity are still required before model access. +Seventh pre-resolution audit amendment: 2026-08-02. An independent scientific +and execution audit found four defects before any model-weight or quality +access: the static/dynamic gate was step-matched rather than equal-resident; +"quality oracle" overstated a local allocator; Stage C did not explicitly own +the confirmation decision; and the named 2B checkpoint had no executable +quality contract. It also found that the source verifier safely accepts an +unchanged identity-only descendant of source commit H0, while the runner +duplicated that policy with an incompatible raw `HEAD == H0` check. These are +corrected below. Identity schemas must advance together to v5 to bind a causal +one-step diagonal empirical-Fisher comparator and its boundary commitments. +Every v4 candidate or identity is superseded. A new clean H0, sealed runtime, +fresh receipts, v5 identity, and identity-bound model staging are required. No +model weights, calibration scores, stability values, or quality results had +been opened. + +Eighth pre-resolution audit amendment: 2026-08-02. The first runtime-preparation +attempt correctly rejected a previously compiled environment because an +installed distribution's RECORD inventory owned forbidden bytecode. A fresh +no-compile environment passed that check, but its first preparation attempt +stopped before publication when the staged interpreter legitimately reported +the runtime root as `.` in `sys.path` and the generic relative-path validator +indexed the sentinel as though it had path components. The runtime builder now +accepts exactly `.` only in the base-`sys.path` contract; ordinary repository, +runtime-file, interpreter, and import paths still reject it. The next +no-overwrite preparation completed with 38 distributions, 831 base-runtime +files, 20,201 package files, base paths `python311.zip`, `DLLs`, `Lib`, and `.`, +and manifest file SHA-256 +`6cb19144bc373a38001f2349e8e3f9317a7809a41fd415457434bb2bc5cd74ee`. +An isolated import smoke confirmed Python 3.11.15, Torch 2.13.0+cu130, CUDA +13.0, BF16 support, Transformers 5.14.1, and PyArrow 25.0.0 on the RTX 5070. +These facts establish runtime readiness only. No model weights, calibration +scores, stability values, or quality results were opened. + +Ninth pre-resolution audit amendment: 2026-08-02. The Stage-A evaluator +contract candidate was specified before the final H0 source capture and becomes +frozen only through the clean H0 and pre-access binding described above. Its +method order is +exactly `fp32_reference`, `rht_q468_uniform_q4`, `rht_q468_uniform_q8`, +`rht_q48_static_p14739`, `rht_q468_static_k27030`, +`rht_q468_dynamic_k27030`, `rht_q468_static_mse_k29334`, +`rht_q468_static_diag_empirical_fisher_h1_k29334`, and +`rht_q468_static_k29334`. The two uniform anchors are deterministic Q468 +policies reconstructed from the authenticated candidate-score arrays at +`K=0` and `K=73728`. Both use the same RHT, per-row FP16 scales, two-bit code +stream, uint16 pool offsets, and packed Q4/Q6/Q8 pool implementation as the +mixed policies. Their natural resident sizes are `2,515,968` and `4,875,264` +bytes, respectively; they are descriptive lower- and upper-precision anchors, +not equal-byte comparators and never decide passage. + +For every method and example, excess NLL is the ordinary mean over only the +identity-bound cache-exposed transitions. Family values first average the four +example means inside PG19, RULER, and HumanEval+, and the task macro is the +unweighted mean of those three family values. Top-1 agreement uses the same +example-then-family-then-three-family macro. Token micro is reported but never +gates. `cvar95_kl` means the FP32 mean of the largest +`max(1, ceil(0.05 * token_count))` finite per-token KL values, matching the +reviewed `fidelity_summary` implementation. + +Each paired non-inferiority bootstrap independently resamples four examples +with replacement inside each family, averages inside family, then averages the +three family means equally. The generator is NumPy `PCG64`, initialized once +with seed 2,339; it draws one `10000 x 4` integer-index matrix for each family +in PG19, RULER, HumanEval+ order. The sample count is 10,000. The one-sided +95th-percentile upper bound is the nearest-rank order +statistic: sort the 10,000 replicate contrasts ascending and select zero-based +index `ceil(0.95 * 10000) - 1 = 9499`, with no interpolation. The primary is +checked separately against dynamic K27030, unweighted-MSE K29334, and diagonal +empirical-Fisher H1 K29334 under the prespecified three conjunctive margins. +Static K27030 remains diagnostic and cannot alter passage. "Beats Q48 on every +Stage-A workload" means strictly lower family-mean excess NLL in each of PG19, +RULER, and HumanEval+; equality in any family fails. No model weights, Stage-A +rows, calibration scores, or quality results were opened before this amendment. + +Tenth pre-resolution audit amendment: 2026-08-02. Before the complete +calibration, the sealed runner must execute `--fisher-h1-smoke` against exactly +the first canonical record of the promoted calibration identity. It still +materializes and authenticates the complete 160-record identity before model +access, then loads the pinned model once and runs the selected record's full +token sequence through the ordinary causal capture path. Fisher calls replace +ordinary forwards at every identity-bound `B(T)` input position. Passage +requires exact token, anchor, Fisher-boundary, kernel-receipt, and adapter +Fisher-step counts; finite CPU-FP64 Q4/Q6/Q8 endpoint scores; frozen model +parameters with no gradients; successful cache detachment; model/source/runtime +reauthentication; and a clean close. The no-overwrite receipt records elapsed +time, peak CUDA allocated and reserved bytes, device identity, token count, +anchor count, and Fisher-boundary count. It publishes no score aggregate, +policy, binding, stability value, or quality result and uses the distinct +`FISHER_H1_SMOKE_COMPLETE` marker. A pass authorizes only attempting the full +calibration. Failure stops the run; there is no automatic retry or weakened +smoke input. + +Eleventh pre-H0 scientific amendment candidate: 2026-08-14. A read-only +methodology audit found that an independent Stage-C HumanEval+ hash domain did +not by itself guarantee disjointness from Stage A/B, and that the Stage-C +bootstrap and relative-improvement gates were not yet executable-grade. The +rules below now exclude and bind every Stage-A/B HumanEval+ ID before Stage-C +selection; fail closed on any overlap; define the exact example, family, task, +contrast, bootstrap, bound, effect-size, top-1, and multiplicity equations; and +limit stability and fidelity language to what is actually tested. No protected +identity or row content, model weights, calibration score, stability value, or +quality result was opened, and no evaluator was run for this amendment. This +working copy remains unpreregistered until its exact bytes are committed and +bound as H0 before protected materialization. + ## Question -Can a calibration-frozen, static Q4/Q6/Q8 recurrent-state layout retain the -quality of an exact dynamic mixed-bit oracle while using one immutable code map -that a packed GPU kernel can execute efficiently? +Can a calibration-frozen, static Q4/Q6/Q8 recurrent-state layout satisfy the +prespecified cache-exposed teacher-forced target-NLL and top-1 non-inferiority +gates against an online local-distortion allocator at the same complete +recurrent-resident byte budget while using one immutable code map that a packed +GPU kernel can execute efficiently? The candidate is: @@ -108,7 +222,7 @@ revision dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68 Transformers 5.14.1 ``` -The conditional scale check is: +The conditional scale resource probe is: ```text Qwen/Qwen3.5-2B-Base @@ -116,16 +230,23 @@ revision b1485b2fa6dfa1287294f269f5fb618e03d52d7c Transformers 5.14.1 ``` -The 2B check is authorized only if a cold-start language-model load measures no -more than `7.5 GiB` peak device memory on the RTX 5070 8GB. Failure or inability -to satisfy that gate is recorded as a resource stop; it cannot be replaced by -an unreported offload configuration. +The 2B checkpoint is feasibility-only in Experiment 013. The only authorized +operation is an otherwise idle BF16/eager cold-start load with no prompt, +generation, cache-quality measurement, or offload. Record both peak allocated +and peak reserved device memory. A value above `7.5 GiB`, or inability to make +the measurement, is a resource stop. + +A passing cold start authorizes only a separately frozen, model-specific +protocol defining geometry, complete byte budgets, calibration refit or transfer, +identities, workloads, thresholds, and implementation. It contributes no +quality, generalization, scaling, or confirmation evidence to Experiment 013. -Both checkpoints use batch one, eager evaluation, no sampling, BF16 model +The primary 0.8B run uses batch one, eager evaluation, no sampling, BF16 model weights, and FP32 reference recurrent state. Model architecture, recurrent layer indices, state geometry, tokenizer class, tokenizer files, and every runtime package are identity-bound before weights are opened. A mismatch stops -the run. +the run. The 2B resource probe shares only the frozen BF16/eager load contract; +it does not enter recurrent-state evaluation under this protocol. The metadata-only upstream identities are frozen without opening example contents: @@ -176,10 +297,15 @@ use Q8 and the rest Q4. Its payload is `3,302,592` bytes, its one-bit code map is `4,608` bytes, and its scales, uint16 offsets, and eight alignment bytes make the same `3,454,664`-byte total. -The `K27030` layout is a prespecified diagnostic budget used only to compare -static and dynamic selection without conflating that comparison with the -primary exact-byte contrast. Its physical allocation is reported exactly and -is never rounded up to the primary budget. +Static `K27030` is a prespecified selection-step diagnostic. Its payload is +`3,224,256` bytes; scales, two-bit codes, and uint16 offsets add `73,728`, +`9,216`, and `73,728` bytes, respectively, for `3,380,928` resident bytes with +no alignment padding. It is never rounded up to the primary budget. + +The online `rht_q468_dynamic_k27030` baseline owns the same `K27030` payload, +scales, and code bytes, plus `147,456` bytes of persistent FP32 query EMA and +eight alignment bytes, for exactly `3,454,664` recurrent-resident bytes. It has +the same complete resident budget as static `K29334`; static `K27030` does not. ## Calibration and static-policy freeze @@ -343,10 +469,61 @@ lexicographically greatest flattened code vector, so the lower flat row gets higher precision first. No task loss, Stage-A value, Stage-B value, or Stage-C value participates in -the code-map fit. The primary map uses `K29334`. The diagnostic static and -dynamic layouts both use `K27030`. Once an identity and code map are committed, -no seed, score, quota, tie rule, token span, group size, or bit budget may be -changed under Experiment 013. +the candidate's query-energy code-map fit. The primary map uses `K29334`. The +diagnostic static and dynamic layouts both use `K27030`. Once an identity and +code map are committed, no seed, score, quota, tie rule, token span, group size, +or bit budget may be changed under Experiment 013. + +## Prespecified sensitivity comparators + +Two same-format `K29334` static maps separate the mixed-format question from the +candidate's query-energy selector. Both use the identical RHT, Q4/Q6/Q8 codec, +scales, codes, offsets, tie rule, and `3,454,664` resident-byte ledger. + +The unweighted comparator is `rht_q468_static_mse_k29334`. It replaces the +query-energy factor by one and otherwise uses the same per-sequence and +family-balanced reduction. + +The loss-sensitive comparator is +`rht_q468_static_diag_empirical_fisher_h1_k29334`. It is an adapted diagonal +empirical-Fisher baseline, not RateQuant itself and not an exact Fisher matrix +or Hessian. Let `x[0:T]` denote one frozen token sequence and let `S_b` be the +persistent FP32 recurrent state after consuming token `x_b`. Eligible stored +boundaries are `b = 0, ..., T-3`, selected exactly as + +```text +B(T) = frozen_anchor_positions(T - 2). +``` + +At boundary `b`, the measured recurrent step consumes `x_(b+1)`. Its logits +`z_(b+1)` predict target `x_(b+2)`. The causal gradient is + +```text +g_b = d CE(z_(b+1), x_(b+2)) / d S_b. +``` + +`z_b` is forbidden because it was produced before `S_b` was stored. For row +`r`, transform both state and gradient into the codec basis: + +```text +Z_b(r) = RHT(S_b(r)) +G_b(r) = RHT(g_b(r)) +risk_q(b,r) = 0.5 * sum_value(G_b(r)^2 * (Q_q(Z_b(r)) - Z_b(r))^2) +``` + +for `q` in `{4, 6, 8}`. Compute the risk in deterministic FP64 reduction order. +Mean boundaries within sequence, then use the same MBPP/PG19/RULER and RULER +category balancing as the candidate. Feed the three endpoint risks directly to +the exact allocator at `K=29334` with the existing flattening and tie rule. +There is no query-energy multiplication, gradient normalization, clipping, +layer quota, protected-set loss, or post-result tuning. + +Identity v5 domain-separates and binds, for every calibration sequence, the +horizon `H=1`, boundary positions, input positions, target positions, and their +token-ID hashes. The implementation must prove one successful recurrent kernel +call per layer, no model-parameter gradients, exact rollback on a failed Fisher +step, and a fully detached continuing FP32 cache. A real RTX 5070 peak-memory +and runtime smoke receipt is required before the complete calibration run. ## Policy-stability gates @@ -363,6 +540,15 @@ split halves A and B. All three gates are conjunctive: Failure stops the static candidate. The map may not be stabilized by changing the data, metric, seed, threshold, or aggregation after the failure is known. +These split-half gates apply only to the query-energy candidate map. The +unweighted-MSE and diagonal empirical-Fisher maps are fixed comparator +instantiations fitted on the complete calibration identity; Experiment 013 does +not test their split-half stability. Therefore any Stage-A or Stage-C comparison +is only against those exact frozen maps. It cannot establish that query-energy +selection is generally more stable, robust, or effective than MSE or Fisher +sensitivity. Such a selector-principle claim requires a new pre-access amendment +with symmetric comparator-stability tests or a new experiment. + ## Frozen evaluation identities Identity resolution is staged. A resolver may create only a quarantined @@ -371,7 +557,7 @@ creates the identity that must be committed before model weights are loaded. Stage-B and Stage-C content is protected and requires separate authorization; ordinary resolver tests and dry runs must not read it. -Identity schema v4 also binds four exact pre-model evidence files under +Identity schema v5 also binds four exact pre-model evidence files under `execution_bindings`: ```text @@ -391,9 +577,30 @@ promotion may the runner hash local model files and compare them with the frozen model manifest. A missing, malformed, or byte-different dependency stops before adapter data access or model loading. +The source manifest binds implementation commit H0. Committing the promoted +identity creates H1. H1 is authorized only when H0 is its Git ancestor, the +authenticated source verifier proves every frozen source path has identical H0 +tree, H1/index, and worktree bytes, and the worktree is otherwise clean. Reports +and policy artifacts continue to record H0 as implementation provenance; H1 is +the identity authorization commit and may not be relabelled as source commit. + +Model payload staging begins only after the frozen identity is tracked with +identical H1, index, and worktree bytes. The identity-bound stager downloads +only the exact sorted root files in the frozen model manifest at the exact +40-hex public Hub revision, using an external cache and no token. Returned cache +paths are untrusted: every source must resolve inside that cache, then be +stream-copied into a fresh sibling staging directory. Ordinary files are checked +by Git blob OID and size; LFS payloads are checked by payload SHA-256 and size. +The staged tree must have exact case-insensitive-unique inventory and contain no +links, reparse points, cache metadata, marker, or extra file. Reauthenticate the +identity, source, and manifest immediately before an atomic no-replace directory +rename, then independently authenticate the published model root. Failure may +clean only the owned staging directory; it never overwrites the output or +deletes the shared Hub cache. + Stage-A resolution additionally consumes one strictly decoded -`experiment-013-stage-a-calibration-binding-v2` artifact. The resolved Stage-A -identity binds these five dependency files directly, not merely semantic IDs +`experiment-013-stage-a-calibration-binding-v3` artifact. The resolved Stage-A +identity binds these eight dependency files directly, not merely semantic IDs copied from a caller: ```text @@ -402,8 +609,25 @@ calibration_score_artifact_file_sha256 split_half_stability_artifact_file_sha256 static_k27030_policy_file_sha256 static_k29334_policy_file_sha256 +comparator_score_artifact_file_sha256 +static_fisher_k29334_policy_file_sha256 +static_mse_k29334_policy_file_sha256 ``` +The comparator-score dependency is one strict canonical artifact containing +exactly the unweighted-MSE and diagonal empirical-Fisher H=1 aggregates, their +selector-specific sequence manifests, and their exact K29334 allocations. It +is combined only to keep the dependency inventory at eight files; the two +profiles retain separate score hashes, position manifests, and policy +bindings. A policy file without its matching embedded comparator scores is not +verifiable and fails closed. + +The static Q4/Q8 comparator is deterministically reconstructed inside the +authenticated Stage-A evaluator from the bound candidate score artifact at +the frozen `P=14739` promotion count. A separately published convenience copy +is not a ninth trusted dependency and may not be accepted without exact +reconstruction equality. + Changing any byte in any dependency requires a new binding artifact and a new Stage-A identity candidate. @@ -443,12 +667,39 @@ content hashes, prompt-token hashes, target-token hashes, and half-open token spans must be committed before weights are opened. No example may be replaced because its result is inconvenient. -The exact dynamic Q468 allocator is the Stage-A quality oracle. Static K27030 -must have an excess-NLL upper confidence bound no more than `0.01` nats/token -above dynamic K27030. No Stage-A workload may have a static-candidate -disadvantage above `0.015` nats/token. Static K29334 must beat exact-byte -`rht_q48_static_p14739` on every Stage-A workload. If the multi-workload advantage -does not reproduce, stop; do not proceed by reframing the oracle as optional. +`rht_q468_dynamic_k27030` is the prespecified online local-distortion allocator +baseline. "Exact" describes only its discrete `K27030` allocation under the +frozen local objective and tie rule; it is not an oracle for sequence NLL, +downstream accuracy, generation, or globally optimal recurrent trajectories. + +The primary equal-resident Stage-A contrast is static `K29334` versus dynamic +`K27030`; both own exactly `3,454,664` recurrent-resident bytes. For paired +example `e`, define + +```text +d_e = excess_NLL(static_K29334, e) - excess_NLL(dynamic_K27030, e). +``` + +Using seed 2,339, perform 10,000 paired stratified bootstrap resamples by +resampling examples with replacement inside PG19, RULER, and HumanEval+, then +averaging the three family means equally. The one-sided 95% upper percentile +bound for `d` must be no more than `0.010` nats/token; no family point estimate +may exceed `0.015`; and static `K29334` top-1 agreement may trail dynamic +`K27030` by at most `0.005`. All conditions are conjunctive. + +Static `K27030` versus dynamic `K27030` remains a selection-step-matched +diagnostic and cannot decide the equal-resident adoption claim. Static `K29334` +must also beat exact-byte `rht_q48_static_p14739` on every Stage-A workload. If +either prespecified multi-workload contrast fails, stop rather than changing the +baseline or budget after observing the result. + +The candidate must separately be non-inferior to each of +`rht_q468_static_diag_empirical_fisher_h1_k29334` and +`rht_q468_static_mse_k29334` under the same `0.010` upper-bound, `0.015` +per-family, and `0.005` top-1 margins. These are selector comparisons at the +same `K29334` format and byte ledger. If either comparator wins, the frozen +query-energy candidate fails Experiment 013; the winning comparator may seed a +new experiment but may not replace the candidate post hoc. ### Stage B: development @@ -472,8 +723,151 @@ contains: - 32 SHA-ranked eligible PG19 test books; - all 52 combinations of the 13 exact RULER configs at configured length 4,096 and seeds 3,339 through 3,342; and -- the next 32 HumanEval+ canonical IDs under the separate Stage-C confirmation - hash domain. +- 32 HumanEval+ canonical IDs selected from the exact eligible remainder by the + disjoint procedure below. + +Let `U_HE` be the exact 164-ID HumanEval+ canonical-ID projection bound by the +immutable projection manifest. Rank `U_HE` under +`recurquant.experiment013.humaneval-plus.stage-a-b.v1\0`, breaking a hash tie by +canonical ID. Let `H_A` be ranks 0 through 3, `H_B` be ranks 4 through 31, and +`H_AB = H_A union H_B`. Thus `H_AB` must contain exactly the 32 distinct IDs +used by Stage A/B. Define the Stage-C eligible remainder before looking up any +HumanEval+ row content: + +```text +U_C = U_HE setminus H_AB +``` + +`U_C` must contain exactly 132 distinct IDs. Rank only `U_C`, never the complete +`U_HE`, by lowercase SHA-256 of the UTF-8 domain-separated canonical ID under +`recurquant.experiment013.humaneval-plus.stage-c.v1\0`, again breaking a hash +tie by canonical ID. Let `H_C` be ranks 0 through 31 of that remainder. + +Before any Stage-C HumanEval+ row, prompt, canonical solution, token, or token +span is requested or materialized, the Stage-C identity candidate must bind: + +1. the exact 164-ID projection-manifest hash; +2. the 32-entry `H_AB` exclusion manifest ordered by Stage-A/B rank, including + each canonical ID, rank, and Stage-A/B selection hash, plus the canonical + SHA-256 of that complete manifest; +3. the 132-entry `U_C` remainder manifest ordered by Stage-C remainder rank and + its canonical SHA-256; and +4. the 32-entry `H_C` selection manifest ordered by Stage-C remainder rank, + including canonical ID, rank, and Stage-C selection hash, plus its canonical + SHA-256. + +The resolver must recompute all four objects from the authenticated ID-only +projection and prove exact counts, distinctness, membership, ordering, hashes, +`H_AB union U_C = U_HE`, `H_AB intersection U_C = empty`, and +`H_AB intersection H_C = empty`. Any mismatch or overlap fails closed before +content access. There is no fallback row, replacement, or reranking. + +Stage C alone decides confirmation. Stage-A and Stage-B observations may not be +pooled into a Stage-C point estimate, confidence interval, method choice, +comparator choice, effect threshold, or claim. + +The Stage-C family order is PG19, RULER, HumanEval+, with exact example counts +`n_f = 32, 52, 32`. Within each family, examples use their authenticated +identity-file order. For method `j`, example `e`, and its exact `m_e` finite +cache-exposed transitions, define: + +```text +x_(j,e) = (1 / m_e) * sum_t [NLL_(j,e,t) - NLL_(FP32,e,t)] +a_(j,e) = top1_agreement_count_(j,e) / m_e +X_(j,f) = (1 / n_f) * sum_(e in f) x_(j,e) +A_(j,f) = (1 / n_f) * sum_(e in f) a_(j,e) +X_j = (1 / 3) * sum_f X_(j,f) +A_j = (1 / 3) * sum_f A_(j,f) +``` + +`top1_agreement_count_(j,e)` is the exact number of transitions for which +method `j` and the matched FP32 reference have the same argmax token. +`a_(j,e)`, `A_(j,f)`, and `A_j` are exact rational values derived from integer +agreement and transition counts until the threshold comparison. Each method +must contain exactly the same identity-bound transitions and FP32 reference NLL +for an example. Missing, duplicate, reordered, non-finite, or reference-drifted +rows fail closed. Examples are equal-weighted within family and the three family +means are equal-weighted regardless of token or example counts. Token-micro NLL, +mean KL, tail KL, and maximum KL are diagnostic only and never gate. + +All four primary contrasts use one shared deterministic bootstrap schedule. +Initialize exactly one NumPy `Generator(PCG64(2339))`. In PG19, RULER, +HumanEval+ order, call +`integers(0, n_f, size=(10000, n_f), dtype=np.int64, endpoint=False)` once per +family, producing matrices of shapes `10000 x 32`, `10000 x 52`, and +`10000 x 32`. Reuse those same three matrices, without reinitialization or any +additional random draw, for every contrast in this fixed order: + +1. `rht_q468_static_k29334` versus + `rht_q468_dynamic_k27030` non-inferiority; +2. superiority to `rht_q48_static_p14739`; +3. superiority to + `rht_q468_static_diag_empirical_fisher_h1_k29334`; and +4. superiority to `rht_q468_static_mse_k29334`. + +For any example-level contrast vector `c_e`, bootstrap replicate `b` first +averages the `n_f` indexed values inside each family, then averages the three +bootstrap family means equally. Sort the 10,000 replicate task-macro contrasts +ascending with a stable sort. The one-sided 98.75% upper percentile bound is +zero-based element `ceil(0.9875 * 10000) - 1 = 9874`; the one-sided 98.75% +lower percentile bound is zero-based element +`ceil(0.0125 * 10000) - 1 = 124`. Both are nearest-rank order statistics with +no interpolation. + +For candidate `p = rht_q468_static_k29334` and equal-resident dynamic baseline +`d = rht_q468_dynamic_k27030`, define the non-inferiority contrast with positive +values meaning that the candidate is worse: + +```text +d_e = x_(p,e) - x_(d,e) +D_f = X_(p,f) - X_(d,f) +D = X_p - X_d +``` + +The upper bound is computed from bootstrap replicates of `d_e`. This contrast +passes only if that upper bound is at most `0.010` nats/token, every `D_f` is at +most `0.015`, and the exact task-macro top-1 trail `A_d - A_p` is at most +`0.005`. The bound threshold applies to the bootstrap bound; the family and +top-1 thresholds apply to their observed Stage-C point estimates. + +For each superiority comparator `c` in the fixed Q48, Fisher, MSE order above, +define positive values to mean that the candidate is better: + +```text +g_(c,e) = x_(c,e) - x_(p,e) +G_(c,f) = X_(c,f) - X_(p,f) +G_c = X_c - X_p +R_c = G_c / X_c +``` + +The lower bound is computed from bootstrap replicates of `g_(c,e)`. A +superiority contrast passes only if all of the following conjunctive conditions +hold: `X_c > 0`; observed point improvement `G_c >= 0.002` nats/token; +observed relative point improvement `R_c >= 0.10`; the 98.75% lower bound is +strictly greater than zero; every observed family point `G_(c,f) > 0`; and the +exact task-macro top-1 trail `A_c - A_p <= 0.005`. `X_c` is the sole denominator +of the 10% condition. If `X_c <= 0`, `R_c` is undefined and the contrast fails; +there is no absolute value, epsilon, clipping, sign reversal, or alternate +denominator. The `0.002` and `10%` thresholds apply to observed task-macro point +estimates, not bootstrap bounds. Equality passes the inclusive `0.002`, `0.10`, +and `0.005` checks but fails the strict lower-bound and every-family positivity +checks. + +The dynamic non-inferiority contrast and three superiority contrasts are the +four primary hypotheses. Each uses one one-sided alpha of `0.0125`, implemented +by its 98.75% bound; Bonferroni therefore limits the family-wise alpha to +`4 * 0.0125 = 0.05`. All four hypotheses and every associated deterministic +effect-size, family-point, and top-1 gate must pass. These additional gates are +not separate confidence claims. No post-result comparator selection, alpha +reallocation, pooling, or strongest-comparator claim is permitted. + +Passing Stage C supports only teacher-forced cache-exposed target-NLL and FP32 +top-1-agreement statements for these exact frozen identities, spans, methods, +checkpoint, and budgets. Because KL is diagnostic rather than gated, passage +does not establish FP32 logit-distribution fidelity. It also does not establish +free generation, generated-code execution or correctness, downstream task +accuracy, selector-principle superiority, deployment, speed, novelty, or a +breakthrough. The generated RULER IDs, auxiliary-source hashes, formatter hashes, actual lengths, and token spans remain unresolved until a separate protected identity @@ -482,11 +876,24 @@ amendment is frozen. Stage C may not be partially previewed. ## Methods and measurements Every accepted quality run includes FP32 recurrent state, uniform RHT Q4 and -Q8 anchors, `rht_q48_static_p14739`, static K27030, exact dynamic K27030, and -`rht_q468_static_k29334`. A closest eligible published comparator is added only -through a pre-result identity amendment with its exact implementation and byte -accounting; an incompatible or unavailable comparator is documented rather -than imitated under its name. +Q8 anchors, `rht_q48_static_p14739`, static K27030, online dynamic K27030, +`rht_q468_static_mse_k29334`, +`rht_q468_static_diag_empirical_fisher_h1_k29334`, and +`rht_q468_static_k29334`. A closest eligible published implementation is added +only through a pre-result identity amendment with its exact implementation and +byte accounting; an incompatible or unavailable implementation is documented +rather than imitated under its name. + +[When Good Enough Is Optimal](https://arxiv.org/abs/2606.06034) already applies +low-bit integer arithmetic to the chunkwise matrix-inversion path of Gated +DeltaNet on Qwen3.5-family models. It targets multiplication-only inverse +approximation and kernel overhead, not mixed-precision storage of the +persistent recurrent state, but it means that low-bit Gated DeltaNet execution +itself is not a novelty claim. [SAW-INT4](https://arxiv.org/abs/2604.19157) +likewise demonstrates block-diagonal Hadamard rotation in a fused INT4 KV-cache +path under serving constraints. Its cache object differs from the fixed-size +Gated DeltaNet state, but it reinforces that rotation plus four-bit packing is +prior art and that deployment evidence must come from an integrated kernel. Primary quality is task-macro aligned excess next-token NLL over only the identity-bound cache-exposed transitions relative to the matched FP32 @@ -495,12 +902,20 @@ tail KL, top-1 agreement, local codec SSE, trajectory error, result by workload family, resident bytes, transient bytes, peak HBM, and latency. Statistical intervals are paired task bootstraps with 10,000 resamples and seed 2,339. -For the full evaluation, the candidate must improve on the strongest eligible -comparator by both at least `10%` and at least `0.002` nats/token, with the -paired 95% lower confidence bound above zero. The point improvement must be -positive in each of PG19, RULER, and HumanEval+. Candidate top-1 agreement may -trail by at most `0.005`. A non-positive comparator excess NLL makes the -relative gate fail closed; it is not redefined. +Stage-A resource fields are diagnostic rather than like-for-like deployment +measurements. Per-transition latency and CUDA allocator peaks cover the scored +one-token decode forward only; prefill latency and allocator peaks are reported +separately per method. Logical recurrent-resident bytes exclude model weights, +ordinary attention caches, allocator reservation, and temporary workspaces. +The cache-reported workspace value is a cumulative high-water sum since the +method began, including prefill, while CUDA reserved-byte peaks can retain +allocator history and method order. None of these fields may substitute for +the packed-native end-to-end deployment gate below. + +Stage-A results are a falsification screen, not confirmation or selector +superiority evidence. Only the separately frozen Stage-C decision above can +support those claims. A non-positive comparator excess NLL makes a relative +superiority gate fail closed; it is not redefined. ## Packed-native deployment gate @@ -524,6 +939,53 @@ driver, CUDA, kernel version, compiler flags, batch, prompt/decode lengths, and all raw repeats. The existing uniform-kernel microbenchmark is an isolated diagnostic only and cannot be used as end-to-end evidence. +### Opaque pre-staging boundary + +The official Stage-A launcher uses two cold authenticated child processes. +Before the one-run seal, a credential-stripped network child copies and hashes +the exact pinned tokenizer, Parquet, RULER-generator, generation-manifest, +receipt, and model-metadata bytes into a content-addressed bundle without +decoding protected rows or receipt bodies. A new child then starts with +`HF_HUB_OFFLINE=1`, `HF_DATASETS_OFFLINE=1`, and `TRANSFORMERS_OFFLINE=1`, +authenticates that bundle, binds its manifest SHA-256 into the seal and durable +evidence chain, and materializes Stage-A content only after reservation from +the local bundle. A Python socket audit guard makes attempted network access in +that offline child fatal. + +"Opaque" describes the authenticated program's procedure, not encryption or +access control. The bundle necessarily contains raw Parquet and RULER bytes. A +human who controls the filesystem could inspect them before the seal, so this +is an honest-process one-run boundary rather than a claim of human blindness. +Stronger blindness would require independent custody, encryption, and a +post-seal key or data release outside this local evaluator. + +### One-run evidence boundary + +The Stage-A empty-diff seal commit, Git refs and reflogs, and the +identity-scoped lock in the repository's Git common directory make accidental +re-execution and ordinary local history changes detectable. They provide a +durable honest-process audit trail on the machine where the run is performed; +they are not a cryptographic proof that only one execution was ever possible. + +The source and runtime manifests authenticate the exact canonical Git +executable bytes, file size, and normalized absolute-path digest used by the +runner. They do not recursively authenticate Git-for-Windows loaded DLLs, +helper executables, the Windows kernel, or the underlying operating system; +those remain part of the external trusted computing base. Public Hub, +GitHub, certificate, and TLS availability are also external, although every +accepted revision, manifest, and downloaded object is checked against its +frozen identity before protected execution. Reports must not describe this as +cryptographic attestation of the complete OS or toolchain. + +A person with filesystem control can deliberately delete the lock and reflog, +rewrite or remove refs, or start from a fresh clone. The pre-run seal also +cannot authenticate a result that does not yet exist. Stronger public proof +requires an external append-only anchor for the seal before protected access +and a second external anchor or signature over the completed result bundle. +External anchoring is outside this evaluator. Accordingly, reports must call +the local controls one-run auditability or tamper evidence, never tamperproof +or cryptographically non-bypassable enforcement. + ## Advancement and claim boundary Every integrity, stability, quality, and deployment gate is conjunctive for an @@ -533,9 +995,12 @@ supports only an implementation result. The prior-art boundary is narrow. [RateQuant](https://arxiv.org/abs/2605.06675v2) already fits calibration-based mixed-precision rate-distortion policies for KV -caches. [Q-Mamba](https://aclanthology.org/2025.findings-acl.551/) quantizes -Mamba state caches, while [Quamba2](https://arxiv.org/abs/2503.22879v4) provides -quantized SSM deployment and kernels. [Gated DeltaNet-2](https://arxiv.org/abs/2605.22791v1) +caches and makes loss-gradient sensitivity central. Its published KV +head/token implementation is not a direct Gated DeltaNet matrix-row baseline, +but the sensitivity principle transfers; the prespecified H1 comparator above +tests it without claiming to be RateQuant. [Q-Mamba](https://aclanthology.org/2025.findings-acl.551/) +quantizes Mamba state caches, while [Quamba2](https://arxiv.org/abs/2503.22879v4) +provides quantized SSM deployment and kernels. [Gated DeltaNet-2](https://arxiv.org/abs/2605.22791v1) motivates the architecture family. Its official repository currently provides training code but no tagged release or pretrained checkpoint for the reported 1.3B run. Therefore only a confirmed exact-byte static Q4/Q6/Q8 packed-native @@ -543,11 +1008,13 @@ Gated DeltaNet path plus end-to-end adoption benefit could be differentiated; this protocol makes no novelty claim. Even a complete pass would establish only that the frozen static packed layout -was useful on the pinned checkpoints, workloads, budgets, and hardware. It +was useful on the pinned 0.8B checkpoint, workloads, budgets, and hardware. It would not establish that RHT, mixed precision, loss sensitivity, Q4/Q6/Q8, static allocation, or packed kernels are new. It would not make RecurQuant a new base model, prove generated-code correctness, eliminate contamination, or justify "breakthrough," "state of the art," "lossless," or universal language. +It also would not prove a closed-loop StateLease controller; Experiment 013's +map is immutable after calibration. Failure is a publishable result. Any change after a gate is observed creates a new experiment number with new protected data. diff --git a/scripts/capture_statelease_stage0.py b/scripts/capture_statelease_stage0.py index 5d8714f..a7155dc 100644 --- a/scripts/capture_statelease_stage0.py +++ b/scripts/capture_statelease_stage0.py @@ -116,6 +116,7 @@ "src/recurquant/experiment013_parquet.py", "src/recurquant/experiment013_qwen35_adapter.py", "src/recurquant/experiment013_source.py", + "src/recurquant/experiment013_stage_a.py", "src/recurquant/finite_difference.py", "src/recurquant/fisher_sensitivity.py", "src/recurquant/horizon.py", diff --git a/scripts/capture_static_q468_identity_input.py b/scripts/capture_static_q468_identity_input.py index dcee8cf..7c3ec63 100644 --- a/scripts/capture_static_q468_identity_input.py +++ b/scripts/capture_static_q468_identity_input.py @@ -60,10 +60,11 @@ class or requests a weight file. sys.modules.pop(_RESOLVER_MODULE_NAME, None) raise -CAPTURE_VERSION: Final = 4 +CAPTURE_VERSION: Final = 5 RUNTIME_AUTHENTICATION_CONTEXT_FIELDS: Final = frozenset( { "base_runtime_root", + "git_executable", "staged_interpreter", "package_runtime_roots", "package_import_paths", @@ -167,6 +168,7 @@ class or requests a weight file. RULER_NIAH_CONFIGS: Final = frozenset( {"niah_multiquery", "niah_multikey_2", "niah_single_1", "niah_multivalue"} ) +_STAGE_A_MATERIALIZATION_AUTHENTICATION_SEAL: Final = object() @dataclass(frozen=True, slots=True) @@ -361,64 +363,64 @@ class RulerTaskInvariant: # and task-specific flags, without trusting claims embedded in the receipt set. RULER_COMMAND_MANIFEST_SHA256_BY_FILENAME: Final = { "aggregation__cwe__l2048__s12340.json": ( - "de342cdbdfd2876e9a63d1201f00ebf1ab539ba030311dd4d6b8c9e7539cc38b" + "2b0bf4952b9fd1f4447947ff0e6257da3a32552ec28bf850b7d992761a90c76c" ), "aggregation__cwe__l4096__s12340.json": ( - "dba911e2ccd64251688a34c23974c14b38f9b6f56cee1dabff92b772e4da9684" + "47e06031fdeea0041e007526e18b0f68902b8e248fb6a4325ac0ca1f3e179071" ), "aggregation__fwe__l2048__s12339.json": ( - "60396cfcf9a5528fd6c2f9be4bf8727adde8f3794fae478962c6535abb950eb3" + "39f939a6cd7041a7922f2a3f87c5c1af047e948040ae39969468934993dfca54" ), "aggregation__fwe__l4096__s12339.json": ( - "b33f0f12630b8b94db46fd2c7ba9cfcef3097d6161e28d6fec6d332548e3d9af" + "d4e47efd8b709a6e27d5969128d2869bea3a99e1093606e89f31720ccb650448" ), "aggregation__fwe__l4096__s2339.json": ( - "747d0a4b3af13a91a0aed560023bd5da11546e06eb6471ee9d964612f3cf64cc" + "b889115e728fe5eb1e59c3a6f276d464969ae696011d8bfb99a1330bf5ced6f4" ), "multi_hop_tracing__vt__l2048__s12339.json": ( - "edd4e2e387bd48bd7fe26211d9bd6fa56e23d187b33dba2ceb522d211b853888" + "c362d4c629dd96a94ac01eb24d3eb92b8cae0adaffec49dc8de931a881a68335" ), "multi_hop_tracing__vt__l2048__s12340.json": ( - "ab2cc8b7b60fa459ea9b592ad890e37b39991bbe61163426daa39152970f1c92" + "82bd4e0157a8ac52ad1f7ab1c001bce38a73a584e7a557aeffb4d8672f75556a" ), "multi_hop_tracing__vt__l4096__s12339.json": ( - "15f9b2c1e1d79ebb466e771febddc0640ee4e3b21894591df70f0f2b7b750bec" + "0b039028f6888c790979c521d679b334ea512b41fc0bbf021a7037afefddf0f4" ), "multi_hop_tracing__vt__l4096__s12340.json": ( - "612759085fdf165a665281e43bcec060d86e6e0b900e8f3ab25bc9d4d3aa5eac" + "d2a5b85b5bd563374a492f45f4578e6f6e04c6addf7c8c62d7f34fd5ab2e0035" ), "multi_hop_tracing__vt__l4096__s2339.json": ( - "5a5ebdf3ab054a825bc9e18176a8513f28964a555536e5a2138dd71a4a53a042" + "ed61a589b8c288a7499d8a26ff1dc91af4418b3ac5f61eef7e4f77c7c3c2741a" ), "question_answering__qa_1__l2048__s12339.json": ( - "574119d0a8fe4d8ec4518c2665a5ecbc9dc0f0d97c040c7e07cfc7c7a3b6e59e" + "c5d7533b7aa53ff7a6e0efc67af88947cfc5c1c76672024dd7daf0e6ad4a4018" ), "question_answering__qa_1__l4096__s12339.json": ( - "c9da9b5c0a2b4a8e691a4df84ca1f23bcf8d4427bc6b29e940c065715e31b3a2" + "e57351321e38e98129a7784915333ac08079dcc01a80724e1dc78d068f8101ff" ), "question_answering__qa_1__l4096__s2339.json": ( - "fd5935bc5ba0da67015995fb08df1288d33b81cc0b4f29b0fbb5b40f8e43a690" + "77b57f21537d318c7100055f1198823f68ebfc4c0deccbe528e881f952e08239" ), "question_answering__qa_2__l2048__s12340.json": ( - "16f9804bdd9705a068dafa3746936501f8d01120930f180a81511e02eba50e3b" + "c210afae817638883b1b07da358fc726c7d12575a55772078ddb9091443b595c" ), "question_answering__qa_2__l4096__s12340.json": ( - "abe15545a61b50fc06ad77d9262cffdbfe8480f84ff468e30cf038678736a591" + "a25b93adc8b8f0e4bb6f9faeaa319f58942349ec9c7c8c163f376883353982d3" ), "retrieval__niah_multikey_2__l2048__s12340.json": ( - "f12039ecaf89bbf7fe7180255ad36aa304148d84f2fdae551648e04c5d25c428" + "bc71b720edfa491e5fe0cad882cd0e361829f3c10ed4781b66a71c28b9075731" ), "retrieval__niah_multiquery__l2048__s12339.json": ( - "099c511108e1b8d4431d03b530b772298e510b6dbefb6d9ff2f51b72b059641d" + "2d57d83f619d8e462beb67ac629badde65b661bbbc771ff39e5cfdb9c9eaa97f" ), "retrieval__niah_multiquery__l4096__s2339.json": ( - "f36ead6ac85202ae799598dc2d9086442686aed2cc918951cc46166ad3780918" + "481c41baed008fc121049ca0b43308c79f3e85839c0e941ea8eb8d7294a0037c" ), "retrieval__niah_multivalue__l4096__s12340.json": ( - "03cd0aa9b7e2d16d23ab157a6138cfbf344fd884863cb32723f31131de8ecde2" + "252be364c6fbc93f21e004f71b38468da195f9acd0aeaf2d0357e85ddfe6c1c8" ), "retrieval__niah_single_1__l4096__s12339.json": ( - "4edd45298317225a7f6cd3ae22c93f05b33d47902b62d2dea7425bd7d86d19c2" + "aa7d34fb3a5176fccaa58a925635c667fcfb0fea92dbf6ba3d900316faa68e60" ), } @@ -569,6 +571,7 @@ class TokenizerMaterial: @dataclass(frozen=True, slots=True) class _RuntimeAuthenticationContext: base_runtime_root: Path + git_executable: Path staged_interpreter: Path package_runtime_roots: Mapping[str, Path] package_import_paths: Mapping[str, str] @@ -646,6 +649,8 @@ def __post_init__(self) -> None: or record.get("prompt_token_ids_sha256") != _token_hash(self.prompt_token_ids) or record.get("target_token_ids_sha256") != _token_hash(self.target_token_ids) or record.get("sequence_token_ids_sha256") != _token_hash(sequence_ids) + or record.get("fisher_boundary") + != resolver.build_fisher_boundary_contract(sequence_ids) or record.get("sequence_length") != len(sequence_ids) or span.get("prefill_start") != 0 or span.get("prefill_stop") != len(self.prompt_token_ids) @@ -754,6 +759,247 @@ def token_sequence_manifest_sha256(self) -> str: return sha256_bytes(canonical_json_bytes(commitments)) +@dataclass(frozen=True, slots=True) +class MaterializedStageASequence: + """One authenticated Stage-A identity record and its exact token sequence. + + The record is the content-redacted capture projection authenticated by a + promoted resolver-v5 Stage-A artifact. Raw source rows, formatted prompts, + targets, and receipt bodies are never retained on this object. + """ + + _identity_record_bytes: bytes = dataclass_field(repr=False) + prompt_token_ids: tuple[int, ...] = dataclass_field(repr=False) + target_token_ids: tuple[int, ...] = dataclass_field(repr=False) + _authentication_seal: object = dataclass_field(repr=False, compare=False) + + def __post_init__(self) -> None: + if self._authentication_seal is not _STAGE_A_MATERIALIZATION_AUTHENTICATION_SEAL: + raise ValueError( + "Stage-A sequences may be created only by authenticated v5 materialization" + ) + record = _strict_json( + self._identity_record_bytes, + context="materialized Stage-A identity record", + ) + if canonical_json_bytes(record) != self._identity_record_bytes: + raise ValueError("materialized Stage-A identity record is not canonical") + expected_fields = set(resolver.IDENTITY_RECORD_PAYLOAD_FIELDS) | {"identity_record_sha256"} + if set(record) != expected_fields: + raise ValueError("materialized Stage-A identity record fields drifted") + if not isinstance(self.prompt_token_ids, tuple) or not isinstance( + self.target_token_ids, tuple + ): + raise TypeError("materialized Stage-A token IDs must be tuples") + for side, token_ids in ( + ("prompt", self.prompt_token_ids), + ("target", self.target_token_ids), + ): + if any( + isinstance(token_id, bool) or not isinstance(token_id, int) or token_id < 0 + for token_id in token_ids + ): + raise ValueError( + f"materialized Stage-A {side} token IDs must be non-negative integers" + ) + if not self.prompt_token_ids: + raise ValueError("materialized Stage-A prompt cannot be empty") + if len(self.target_token_ids) < 2: + raise ValueError("materialized Stage-A target must contain at least two tokens") + + for name in ( + "source_content_sha256", + "formatted_content_sha256", + "prompt_token_ids_sha256", + "target_token_ids_sha256", + "sequence_token_ids_sha256", + "tokenizer_manifest_sha256", + "anchor_manifest_sha256", + "identity_record_sha256", + ): + _require_sha256(record.get(name), context=f"materialized Stage-A {name}") + receipt_hash = record.get("generator_receipt_sha256") + if receipt_hash is not None: + _require_sha256( + receipt_hash, + context="materialized Stage-A generator receipt", + ) + + sequence_ids = self.sequence_token_ids + span = record.get("token_span") + if not isinstance(span, Mapping) or set(span) != resolver.TOKEN_SPAN_FIELDS: + raise ValueError("materialized Stage-A identity token span is malformed") + expected_span = { + "prefill_start": 0, + "prefill_stop": len(self.prompt_token_ids), + "scored_start": len(self.prompt_token_ids), + "scored_stop": len(sequence_ids), + "cache_exposed_start": len(self.prompt_token_ids) + 1, + "cache_exposed_stop": len(sequence_ids), + } + if dict(span) != expected_span: + raise ValueError("materialized Stage-A full token span differs from its tokens") + continuation_count = expected_span["scored_stop"] - expected_span["scored_start"] + exposed_count = expected_span["cache_exposed_stop"] - expected_span["cache_exposed_start"] + if continuation_count != len(self.target_token_ids) or exposed_count != ( + continuation_count - 1 + ): + raise ValueError("materialized Stage-A cache-exposed transition count drifted") + + family = record.get("family") + if family == "pg19": + if len(self.prompt_token_ids) != 4_096 or len(self.target_token_ids) != 128: + raise ValueError("materialized Stage-A PG19 must contain 4096+128 tokens") + if exposed_count != 127: + raise ValueError("materialized Stage-A PG19 must expose 127 transitions") + elif family == "ruler": + if record.get("configured_length") != 4_096: + raise ValueError("materialized Stage-A RULER length must be configured at 4096") + elif family == "humaneval_plus": + if len(self.target_token_ids) > 128: + raise ValueError("materialized Stage-A HumanEval+ target exceeds 128 tokens") + else: + raise ValueError("materialized Stage-A identity family is not in the frozen inventory") + + if ( + record.get("identity_record_sha256") != resolver.identity_record_sha256(record) + or record.get("prompt_token_ids_sha256") != _token_hash(self.prompt_token_ids) + or record.get("target_token_ids_sha256") != _token_hash(self.target_token_ids) + or record.get("sequence_token_ids_sha256") != _token_hash(sequence_ids) + or record.get("sequence_length") != len(sequence_ids) + or record.get("anchor_manifest_sha256") + != _anchor_manifest_hash( + canonical_id=str(record.get("canonical_id")), + sequence_ids=sequence_ids, + token_span=expected_span, + ) + or record.get("fisher_boundary") + != resolver.build_fisher_boundary_contract(sequence_ids) + ): + raise ValueError("materialized Stage-A tokens differ from their identity record") + + forbidden = { + "answer_prefix", + "canonical_solution", + "code", + "formatted_payload", + "input", + "outputs", + "prompt", + "source_payload", + "text", + } + if forbidden & set(record): + raise ValueError("materialized Stage-A identity record contains forbidden raw content") + + @property + def identity_record(self) -> dict[str, Any]: + """Return a fresh content-redacted copy of the authenticated record.""" + + return _strict_json( + self._identity_record_bytes, + context="materialized Stage-A identity record", + ) + + @property + def identity_record_sha256(self) -> str: + return str(self.identity_record["identity_record_sha256"]) + + @property + def sequence_token_ids(self) -> tuple[int, ...]: + return self.prompt_token_ids + self.target_token_ids + + @property + def cache_exposed_transition_count(self) -> int: + return len(self.target_token_ids) - 1 + + +@dataclass(frozen=True, slots=True) +class StageAIdentityMaterialization: + """Twelve content-redacted sequences authenticated by a frozen v5 identity.""" + + sequences: tuple[MaterializedStageASequence, ...] + tokenizer_manifest_sha256: str + capture_input_sha256: str + frozen_identity_file_sha256: str + frozen_identity_canonical_evidence_sha256: str + calibration_binding_file_sha256: str + + def __post_init__(self) -> None: + if not isinstance(self.sequences, tuple) or len(self.sequences) != 12: + raise ValueError("Stage-A materialization must contain exactly 12 sequences") + tokenizer_hash = _require_sha256( + self.tokenizer_manifest_sha256, + context="Stage-A materialization tokenizer manifest", + ) + for name, value in ( + ("capture input", self.capture_input_sha256), + ("frozen identity file", self.frozen_identity_file_sha256), + ( + "frozen identity canonical evidence", + self.frozen_identity_canonical_evidence_sha256, + ), + ("calibration binding file", self.calibration_binding_file_sha256), + ): + _require_sha256(value, context=f"Stage-A materialization {name}") + + records = [sequence.identity_record for sequence in self.sequences] + expected_order = [ + (family, rank) for family in ("pg19", "ruler", "humaneval_plus") for rank in range(4) + ] + identities = [sequence.identity_record_sha256 for sequence in self.sequences] + if len(set(identities)) != len(identities): + raise ValueError("Stage-A materialization contains duplicate identities") + actual_order = [ + (str(record["family"]), int(record["selection_rank"])) for record in records + ] + if actual_order != expected_order: + raise ValueError("Stage-A materialization is not ordered by family then rank") + if any(record["tokenizer_manifest_sha256"] != tokenizer_hash for record in records): + raise ValueError("Stage-A materialization tokenizer commitments differ") + if records != sorted(records, key=resolver._record_sort_key): + raise ValueError("Stage-A materialization records are not in canonical order") + + @property + def identity_records(self) -> tuple[dict[str, Any], ...]: + """Return fresh content-redacted record copies in frozen identity order.""" + + return tuple(sequence.identity_record for sequence in self.sequences) + + @property + def by_identity_record_sha256(self) -> Mapping[str, MaterializedStageASequence]: + """Return an immutable digest lookup for the twelve frozen records.""" + + return MappingProxyType( + {sequence.identity_record_sha256: sequence for sequence in self.sequences} + ) + + def lookup(self, identity_record_sha256: str) -> MaterializedStageASequence: + digest = _require_sha256( + identity_record_sha256, + context="Stage-A materialization identity lookup", + ) + try: + return self.by_identity_record_sha256[digest] + except KeyError as error: + raise KeyError(f"unknown Stage-A identity record: {digest}") from error + + @property + def token_sequence_manifest_sha256(self) -> str: + commitments = [ + { + "identity_record_sha256": sequence.identity_record_sha256, + "prompt_token_ids_sha256": _token_hash(sequence.prompt_token_ids), + "target_token_ids_sha256": _token_hash(sequence.target_token_ids), + "sequence_token_ids_sha256": _token_hash(sequence.sequence_token_ids), + "sequence_length": len(sequence.sequence_token_ids), + "cache_exposed_transition_count": sequence.cache_exposed_transition_count, + } + for sequence in self.sequences + ] + return sha256_bytes(canonical_json_bytes(commitments)) + + TokenCaptureSink = dict[tuple[str, str], tuple[tuple[int, ...], tuple[int, ...]]] @@ -967,6 +1213,7 @@ def _base_record( sequence_ids=sequence_ids, token_span=token_span, ), + "fisher_boundary": resolver.build_fisher_boundary_contract(sequence_ids), } if token_sink is not None: key = (family, canonical_id) @@ -1619,9 +1866,8 @@ def _replay_ruler_task_invariants( or not input_text.endswith(invariant.input_suffix) ): raise ValueError(f"RULER {config} input task markers drifted") - if ( - not answer_prefix.startswith(invariant.answer_prefix_marker) - or not answer_prefix.endswith(invariant.answer_prefix_suffix) + if not answer_prefix.startswith(invariant.answer_prefix_marker) or not answer_prefix.endswith( + invariant.answer_prefix_suffix ): raise ValueError(f"RULER {config} answer-prefix boundaries drifted") if ( @@ -1664,9 +1910,7 @@ def _verify_ruler_raw_row( } | ({"token_position_answer"} if niah else set()) if set(row) != expected_fields: raise ValueError("RULER raw validation fields drifted") - row_length = _require_int( - row["length"], context="RULER raw validation length", minimum=1 - ) + row_length = _require_int(row["length"], context="RULER raw validation length", minimum=1) row_length_with_template = _require_int( row["length_w_model_temp"], context="RULER raw validation length_w_model_temp", @@ -1718,9 +1962,7 @@ def _verify_ruler_raw_row( index = input_text.find(first_output) if index < 0 or row_index != index: raise ValueError("RULER NIAH raw answer position drifted") - expected_position = len( - _encode(tokenizer, input_text[:index], add_special_tokens=False) - ) + expected_position = len(_encode(tokenizer, input_text[:index], add_special_tokens=False)) token_position = _require_int( row["token_position_answer"], context="RULER raw validation token_position_answer", @@ -1995,10 +2237,11 @@ def _normalize_auxiliary_files(value: object, *, context: str) -> list[dict[str, def _ruler_canonical_id(*, category: str, config: str, configured_length: int, seed: int) -> str: - return ( - f"{RULER_SEQUENCE_NAMESPACE}:" - f"{resolver.RULER_REVISION}:{category}:{config}:" - f"length={configured_length}:seed={seed}:sample=0" + return resolver.ruler_canonical_id( + category=category, + config=config, + configured_length=configured_length, + seed=seed, ) @@ -2359,9 +2602,16 @@ def _normalize_runtime_authentication_context( context="runtime_authentication_context", ) base_runtime_root = value["base_runtime_root"] + git_executable = value["git_executable"] staged_interpreter = value["staged_interpreter"] - if not isinstance(base_runtime_root, Path) or not isinstance(staged_interpreter, Path): - raise ValueError("runtime authentication roots and interpreter must be Path values") + if ( + not isinstance(base_runtime_root, Path) + or not isinstance(git_executable, Path) + or not isinstance(staged_interpreter, Path) + ): + raise ValueError( + "runtime authentication roots, Git executable, and interpreter must be Path values" + ) raw_roots = value["package_runtime_roots"] raw_import_paths = value["package_import_paths"] if not isinstance(raw_roots, Mapping) or not raw_roots: @@ -2394,6 +2644,7 @@ def _normalize_runtime_authentication_context( raise ValueError("package runtime-root and import-path names differ") return _RuntimeAuthenticationContext( base_runtime_root=base_runtime_root, + git_executable=git_executable, staged_interpreter=staged_interpreter, package_runtime_roots=MappingProxyType(dict(sorted(roots.items()))), package_import_paths=MappingProxyType(dict(sorted(import_paths.items()))), @@ -2465,8 +2716,7 @@ def _decode_execution_binding_artifacts( != PARQUET_MATERIALIZATION_MANIFEST_PATH.resolve(strict=True) or experiment013_parquet.EXPERIMENT013_PARQUET_MANIFEST_SHA256 != resolver.PARQUET_MATERIALIZATION_MANIFEST_FILE_SHA256 - or sha256_bytes(parquet_bytes) - != resolver.PARQUET_MATERIALIZATION_MANIFEST_FILE_SHA256 + or sha256_bytes(parquet_bytes) != resolver.PARQUET_MATERIALIZATION_MANIFEST_FILE_SHA256 or parquet_bytes != parquet_path.read_bytes() ): raise ValueError("Parquet materialization manifest file identity drifted") @@ -2528,9 +2778,7 @@ def _validate_runtime_context_for_manifest( context: _RuntimeAuthenticationContext, runtime_manifest: Any, ) -> None: - expected_import_paths = { - item.name: item.import_path for item in runtime_manifest.package_roots - } + expected_import_paths = {item.name: item.import_path for item in runtime_manifest.package_roots} if dict(context.package_import_paths) != expected_import_paths: raise ValueError("runtime package import paths differ from the frozen manifest") if set(context.package_runtime_roots) != set(expected_import_paths): @@ -2552,6 +2800,7 @@ def _authenticate_execution_binding_artifacts( artifacts: Mapping[str, bytes], *, runtime_context: _RuntimeAuthenticationContext, + model_file_manifest_attestation: bytes | None = None, previous: _AuthenticatedExecutionBindings | None = None, ) -> _AuthenticatedExecutionBindings: """Reauthenticate source, runtime, model metadata, and Parquet at point of use.""" @@ -2572,6 +2821,7 @@ def _authenticate_execution_binding_artifacts( verified_source = decoded.source_module.verify_experiment013_source_manifest( decoded.source_manifest, repo_root=REPOSITORY_ROOT, + git_executable=runtime_context.git_executable, ) if verified_source != dict(decoded.source_manifest): raise RuntimeError("repository source verifier returned a different manifest") @@ -2590,17 +2840,24 @@ def _authenticate_execution_binding_artifacts( base_runtime_root=runtime_context.base_runtime_root, package_roots=runtime_context.package_runtime_roots, interpreter_path=runtime_context.staged_interpreter, + git_executable_path=runtime_context.git_executable, ) if ( authenticated_runtime.manifest_file_sha256 != decoded.bindings["calibration_runtime_manifest_file_sha256"] ): raise RuntimeError("runtime authenticator returned a different manifest identity") - live_model_manifest = runner.capture_model_file_manifest_from_hub( - resolver.PRIMARY_MODEL_ID, - resolver.PRIMARY_MODEL_REVISION, - transformers_version=resolver.TRANSFORMERS_VERSION, - ) + if model_file_manifest_attestation is None: + live_model_manifest = runner.capture_model_file_manifest_from_hub( + resolver.PRIMARY_MODEL_ID, + resolver.PRIMARY_MODEL_REVISION, + transformers_version=resolver.TRANSFORMERS_VERSION, + token=False, + ) + else: + if not isinstance(model_file_manifest_attestation, bytes): + raise TypeError("model file manifest attestation must be bytes") + live_model_manifest = model_file_manifest_attestation if live_model_manifest != artifacts["model_file_manifest_file_sha256"]: raise ValueError("pinned model Hub metadata differs from the frozen manifest") return _AuthenticatedExecutionBindings( @@ -2629,7 +2886,7 @@ def _capture_identity_input_with_tokens( """Run the sole capture flow, optionally retaining formatter token IDs.""" if phase in resolver.PROTECTED_STAGES: - raise PermissionError(f"{phase} is protected; capture v4 refuses it before source access") + raise PermissionError(f"{phase} is protected; capture v5 refuses it before source access") if phase not in resolver.ALLOWED_PHASES: raise ValueError(f"unsupported identity phase: {phase!r}") if phase == "stage_a" and calibration_binding is None: @@ -2642,17 +2899,25 @@ def _capture_identity_input_with_tokens( if runtime_provider is None or not callable(runtime_provider): raise ValueError("capture requires an explicit sealed runtime authentication context") runtime_authentication_context = runtime_provider() - runtime_context = _normalize_runtime_authentication_context( - runtime_authentication_context - ) + runtime_context = _normalize_runtime_authentication_context(runtime_authentication_context) if execution_binding_artifacts is None: fixture_provider = getattr(source, "execution_binding_artifacts", None) if fixture_provider is None or not callable(fixture_provider): raise ValueError("capture requires all four verified execution-binding artifacts") execution_binding_artifacts = fixture_provider() + model_attestation_provider = getattr(source, "model_file_manifest_attestation", None) + model_attestation = ( + model_attestation_provider() + if model_attestation_provider is not None and callable(model_attestation_provider) + else None + ) + authentication_kwargs: dict[str, bytes] = {} + if model_attestation is not None: + authentication_kwargs["model_file_manifest_attestation"] = model_attestation authentication = _authenticate_execution_binding_artifacts( execution_binding_artifacts, runtime_context=runtime_context, + **authentication_kwargs, ) try: before = _validate_heads(source.source_heads(), context="pre-capture") @@ -2695,6 +2960,7 @@ def _capture_identity_input_with_tokens( execution_binding_artifacts, runtime_context=runtime_context, previous=authentication, + **authentication_kwargs, ) result: dict[str, Any] = { @@ -2799,6 +3065,112 @@ def materialize_calibration_identity_sequences( ) +def materialize_stage_a_identity_sequences( + *, + source: CaptureSource, + frozen_stage_a_identity_artifact: bytes, + calibration_binding_artifact: bytes, + expected_frozen_stage_a_identity_file_sha256: str | None = None, + execution_binding_artifacts: Mapping[str, bytes] | None = None, + runtime_authentication_context: Mapping[str, object] | None = None, +) -> StageAIdentityMaterialization: + """Authenticate and materialize the exact twelve frozen Stage-A sequences. + + The promoted resolver-v5 artifact and its complete calibration binding are + authenticated before any data source is touched. The canonical capture is + then replayed once with token retention, and every content-redacted record + must equal the corresponding authenticated frozen record byte for byte. + Candidate-only, unpromoted, altered, missing, duplicate, or reordered + identity inputs therefore cannot enter Stage-A evaluation through this API. + """ + + if not isinstance(frozen_stage_a_identity_artifact, bytes): + raise TypeError("frozen Stage-A identity artifact must be bytes") + if not isinstance(calibration_binding_artifact, bytes): + raise TypeError("Stage-A calibration binding artifact must be bytes") + if ( + CAPTURE_VERSION != 5 + or resolver.RESOLVER_VERSION != 5 + or resolver.INPUT_SCHEMA != "recurquant.experiment013.identity-input.v5" + or resolver.FROZEN_SCHEMA != "recurquant.experiment013.identity-frozen.v5" + ): + raise RuntimeError("Stage-A materialization requires the resolver-v5 identity contract") + + frozen = resolver.deserialize_frozen_stage_a_identity_artifact( + frozen_stage_a_identity_artifact, + calibration_binding_artifact=calibration_binding_artifact, + expected_file_sha256=expected_frozen_stage_a_identity_file_sha256, + ) + result, token_sink = _capture_identity_input_with_tokens( + phase="stage_a", + source=source, + calibration_binding=calibration_binding_artifact, + execution_binding_artifacts=execution_binding_artifacts, + runtime_authentication_context=runtime_authentication_context, + collect_tokens=True, + ) + if result.get("schema") != resolver.INPUT_SCHEMA or result.get("phase") != "stage_a": + raise RuntimeError("Stage-A capture did not return the resolver-v5 input contract") + + replayed_candidate = resolver.build_candidate( + result, + expected_revisions=resolver.FROZEN_DATASET_REVISIONS, + calibration_binding_artifact=calibration_binding_artifact, + ) + frozen_document = _strict_json( + frozen_stage_a_identity_artifact, + context="authenticated frozen Stage-A identity artifact", + ) + frozen_promotion = frozen_document["evidence"]["promotion"] + if ( + sha256_bytes(canonical_json_bytes(replayed_candidate)) + != frozen_promotion["candidate_file_sha256"] + ): + raise ValueError("Stage-A capture lineage differs from the authenticated identity") + + frozen_records = tuple( + {name: record[name] for name in resolver.RECORD_FIELDS} for record in frozen.records + ) + captured_records = tuple(result["records"]) + if canonical_json_bytes(captured_records) != canonical_json_bytes(frozen_records): + raise ValueError("materialized Stage-A records differ from the authenticated identity") + if len(captured_records) != 12: + raise ValueError("authenticated Stage-A identity must contain exactly 12 records") + + sequences: list[MaterializedStageASequence] = [] + remaining = dict(token_sink) + for record in frozen_records: + key = (str(record["family"]), str(record["canonical_id"])) + try: + prompt_ids, target_ids = remaining.pop(key) + except KeyError as error: + raise RuntimeError( + f"missing materialized Stage-A tokens for {key[0]}/{key[1]}" + ) from error + sequences.append( + MaterializedStageASequence( + _identity_record_bytes=canonical_json_bytes(record), + prompt_token_ids=prompt_ids, + target_token_ids=target_ids, + _authentication_seal=_STAGE_A_MATERIALIZATION_AUTHENTICATION_SEAL, + ) + ) + if remaining: + raise RuntimeError("Stage-A token sink contains identities absent from frozen capture") + + tokenizer_manifest_sha256 = sha256_bytes(canonical_json_bytes(result["tokenizer"]["files"])) + if tokenizer_manifest_sha256 != frozen.tokenizer_manifest_sha256: + raise ValueError("Stage-A tokenizer differs from the authenticated identity") + return StageAIdentityMaterialization( + sequences=tuple(sequences), + tokenizer_manifest_sha256=tokenizer_manifest_sha256, + capture_input_sha256=sha256_bytes(canonical_json_bytes(result)), + frozen_identity_file_sha256=frozen.file_sha256, + frozen_identity_canonical_evidence_sha256=frozen.canonical_evidence_sha256, + calibration_binding_file_sha256=sha256_bytes(calibration_binding_artifact), + ) + + def atomic_write_no_overwrite(path: Path, payload: bytes) -> None: """Atomically publish *payload* while refusing an existing destination.""" @@ -2855,24 +3227,35 @@ def source_heads(self) -> Mapping[str, str]: from huggingface_hub import HfApi except ModuleNotFoundError as error: # pragma: no cover - dependency guard raise RuntimeError("live capture requires huggingface-hub") from error - api = HfApi() + api = HfApi(token=False, endpoint="https://huggingface.co") return { "primary_model": str( api.model_info( - resolver.PRIMARY_MODEL_ID, revision=resolver.PRIMARY_MODEL_REVISION + resolver.PRIMARY_MODEL_ID, + revision=resolver.PRIMARY_MODEL_REVISION, + token=False, ).sha ), "mbpp": str( - api.dataset_info(resolver.MBPP_DATASET_ID, revision=resolver.MBPP_REVISION).sha + api.dataset_info( + resolver.MBPP_DATASET_ID, + revision=resolver.MBPP_REVISION, + token=False, + ).sha ), "pg19": str( - api.dataset_info(resolver.PG19_DATASET_ID, revision=resolver.PG19_REVISION).sha + api.dataset_info( + resolver.PG19_DATASET_ID, + revision=resolver.PG19_REVISION, + token=False, + ).sha ), "ruler": self._github_revision(resolver.RULER_SOURCE_ID, resolver.RULER_REVISION), "humaneval_plus": str( api.dataset_info( resolver.HUMANEVAL_PLUS_DATASET_ID, revision=resolver.HUMANEVAL_PLUS_REVISION, + token=False, ).sha ), "evalplus": self._github_revision( @@ -2886,9 +3269,13 @@ def tokenizer_material(self) -> TokenizerMaterial: from transformers import AutoTokenizer except ModuleNotFoundError as error: # pragma: no cover - dependency guard raise RuntimeError("live capture requires huggingface-hub and Transformers") from error - api = HfApi() + api = HfApi(token=False, endpoint="https://huggingface.co") available = set( - api.list_repo_files(resolver.PRIMARY_MODEL_ID, revision=resolver.PRIMARY_MODEL_REVISION) + api.list_repo_files( + resolver.PRIMARY_MODEL_ID, + revision=resolver.PRIMARY_MODEL_REVISION, + token=False, + ) ) selected = [name for name in TOKENIZER_ASSET_NAMES if name in available] if "tokenizer.json" not in selected or "tokenizer_config.json" not in selected: @@ -2902,6 +3289,8 @@ def tokenizer_material(self) -> TokenizerMaterial: filename=name, revision=resolver.PRIMARY_MODEL_REVISION, cache_dir=self.cache_dir, + token=False, + endpoint="https://huggingface.co", ) ) for name in selected @@ -2920,6 +3309,7 @@ def tokenizer_material(self) -> TokenizerMaterial: isolated, local_files_only=True, trust_remote_code=False, + token=False, ) isolated_inventory = { path.relative_to(isolated).as_posix() @@ -2947,6 +3337,7 @@ def mbpp_train_rows(self) -> Sequence[Mapping[str, Any]]: revision=resolver.MBPP_REVISION, split="train", streaming=True, + token=False, ) return tuple(dict(row) for row in rows) @@ -3075,53 +3466,1336 @@ def humaneval_row(self, *, offset: int, expected_task_id: str) -> Mapping[str, A return row -def _parse_named_cli_values( - values: Sequence[str], - *, - context: str, - paths: bool, -) -> dict[str, object]: - parsed: dict[str, object] = {} - for raw in values: - if not isinstance(raw, str) or "=" not in raw: - raise ValueError(f"{context} must use NAME=VALUE") - name, rendered = raw.split("=", 1) - if _RUNTIME_ROOT_NAME_RE.fullmatch(name) is None or not rendered: - raise ValueError(f"{context} contains a non-canonical name or empty value") - if name in parsed: - raise ValueError(f"{context} contains a duplicate name") - parsed[name] = Path(rendered) if paths else rendered - if not parsed: - raise ValueError(f"at least one {context} is required") - return parsed +STAGE_A_INPUT_BUNDLE_SCHEMA: Final = "recurquant.experiment013.stage-a-input-bundle.v1" +STAGE_A_INPUT_BUNDLE_FILENAME: Final = "stage-a-input-bundle.json" +STAGE_A_INPUT_BUNDLE_PROFILE: Final = "opaque-byte-copy-no-semantic-decode-v1" +_STAGE_A_INPUT_BUNDLE_AUTHENTICATION_SEAL: Final = object() +_STAGE_A_OFFLINE_ENVIRONMENT: Final = { + "HF_DATASETS_OFFLINE": "1", + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", +} +_STAGE_A_FORBIDDEN_CREDENTIAL_ENVIRONMENT: Final = frozenset( + { + "GITHUB_TOKEN", + "HF_API_TOKEN", + "HF_TOKEN", + "HUGGINGFACE_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + } +) +_STAGE_A_INPUT_BUNDLE_MANIFEST_FIELDS: Final = frozenset( + { + "schema", + "phase", + "capture_version", + "staging_profile", + "frozen_identity_file_sha256", + "calibration_binding_file_sha256", + "execution_bindings", + "source_heads", + "model_hub_manifest_file_sha256", + "parquet_hub_snapshots", + "objects", + } +) +_STAGE_A_INPUT_BUNDLE_OBJECT_FIELDS: Final = frozenset( + { + "role", + "source_id", + "revision", + "logical_path", + "relative_path", + "sha256", + "size_bytes", + "git_blob_oid", + "lfs_sha256", + } +) +_STAGE_A_INPUT_BUNDLE_ROLES: Final = frozenset( + { + "model_hub_manifest", + "tokenizer", + "parquet", + "ruler_generator", + "ruler_generation_manifest", + "ruler_receipt", + } +) +_WINDOWS_REPARSE_POINT: Final = 0x400 -def _runtime_context_from_cli(args: argparse.Namespace) -> _RuntimeAuthenticationContext: - if args.base_runtime_root is None or args.staged_interpreter is None: - raise ValueError("capture requires --base-runtime-root and --staged-interpreter") - roots = _parse_named_cli_values( - args.package_root, - context="--package-root", - paths=True, +def _bundle_is_link_or_reparse(path: Path) -> bool: + try: + status = path.lstat() + except OSError as error: + raise ValueError("cannot authenticate Stage-A bundle path status") from error + return path.is_symlink() or bool( + int(getattr(status, "st_file_attributes", 0)) & _WINDOWS_REPARSE_POINT ) - import_paths = _parse_named_cli_values( - args.package_import_path, - context="--package-import-path", - paths=False, + + +def _bundle_safe_relative_path(value: object, *, context: str) -> str: + if not isinstance(value, str) or not value or value != value.strip(): + raise ValueError(f"{context} must be a non-empty canonical relative path") + if any(character in value for character in ("\\", "\0", "\n", "\r", ":")): + raise ValueError(f"{context} is unsafe") + path = PurePosixPath(value) + if ( + path.is_absolute() + or path.as_posix() != value + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise ValueError(f"{context} is not a canonical relative path") + return value + + +def _bundle_safe_directory(path: Path, *, create: bool) -> Path: + absolute = Path(os.path.abspath(path)) + if not absolute.is_absolute() or not absolute.anchor: + raise ValueError("Stage-A input bundle directory must be absolute") + current = Path(absolute.anchor) + for component in absolute.parts[1:]: + current /= component + if not os.path.lexists(current): + if not create: + raise FileNotFoundError(f"Stage-A input bundle directory is absent: {current}") + current.mkdir() + if _bundle_is_link_or_reparse(current) or not current.is_dir(): + raise ValueError("Stage-A input bundle path traverses a link or non-directory") + return absolute + + +def _bundle_stable_file_bytes(path: Path, *, context: str) -> bytes: + absolute = Path(os.path.abspath(path)) + if _bundle_is_link_or_reparse(absolute) or not absolute.is_file(): + raise ValueError(f"{context} must be a regular non-link file") + before = absolute.stat() + payload = absolute.read_bytes() + after = absolute.stat() + identity_before = ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, ) - return _normalize_runtime_authentication_context( - { - "base_runtime_root": args.base_runtime_root, - "staged_interpreter": args.staged_interpreter, - "package_runtime_roots": roots, - "package_import_paths": import_paths, - } + identity_after = ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, ) + if identity_before != identity_after or len(payload) != before.st_size: + raise ValueError(f"{context} changed while it was read") + return payload -def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: - parser = argparse.ArgumentParser( - description=( +def _bundle_stable_descendant_bytes( + root: Path, + relative_path: str, + *, + context: str, +) -> bytes: + """Read one bundle file while authenticating every lexical path component.""" + + safe_relative = _bundle_safe_relative_path(relative_path, context=f"{context} path") + absolute_root = _bundle_safe_directory(root, create=False) + components = PurePosixPath(safe_relative).parts + current = absolute_root + for index, component in enumerate(components): + current /= component + if _bundle_is_link_or_reparse(current): + raise ValueError(f"{context} path traverses a link or reparse point") + if index < len(components) - 1: + if not current.is_dir(): + raise ValueError(f"{context} parent is not a directory") + elif not current.is_file(): + raise ValueError(f"{context} is not a regular file") + payload = _bundle_stable_file_bytes(current, context=context) + # Rewalk after the read so a swapped parent cannot silently survive the + # point-of-use authentication boundary. + repeated = absolute_root + for index, component in enumerate(components): + repeated /= component + if _bundle_is_link_or_reparse(repeated): + raise ValueError(f"{context} path changed to a link or reparse point") + if index < len(components) - 1: + if not repeated.is_dir(): + raise ValueError(f"{context} parent changed during authentication") + elif not repeated.is_file(): + raise ValueError(f"{context} changed during authentication") + return payload + + +def _bundle_deep_freeze(value: Any) -> Any: + if isinstance(value, Mapping): + return MappingProxyType({key: _bundle_deep_freeze(item) for key, item in value.items()}) + if isinstance(value, (list, tuple)): + return tuple(_bundle_deep_freeze(item) for item in value) + return value + + +def _require_stage_a_offline_environment() -> None: + inherited = {name.upper(): value for name, value in os.environ.items()} + if any( + inherited.get(name) != expected for name, expected in _STAGE_A_OFFLINE_ENVIRONMENT.items() + ): + raise RuntimeError("offline Stage-A capture requires all frozen offline-mode flags") + present_credentials = sorted(_STAGE_A_FORBIDDEN_CREDENTIAL_ENVIRONMENT & set(inherited)) + if present_credentials: + raise RuntimeError( + "offline Stage-A capture environment contains forbidden credential variables: " + + ", ".join(present_credentials) + ) + + +def _bundle_directory_entries(path: Path, *, context: str) -> dict[str, Path]: + if _bundle_is_link_or_reparse(path) or not path.is_dir(): + raise ValueError(f"{context} must be a non-link directory") + try: + entries = tuple(path.iterdir()) + except OSError as error: + raise ValueError(f"cannot enumerate {context}") from error + names = [entry.name for entry in entries] + if len({name.casefold() for name in names}) != len(names): + raise ValueError(f"{context} contains case-colliding names") + return {entry.name: entry for entry in entries} + + +def _bundle_authenticate_filesystem_inventory(root: Path, *, digests: set[str]) -> None: + """Reject anything outside the exact manifest-plus-content-addressed tree.""" + + absolute_root = _bundle_safe_directory(root, create=False) + root_entries = _bundle_directory_entries(absolute_root, context="Stage-A input bundle root") + if set(root_entries) != {STAGE_A_INPUT_BUNDLE_FILENAME, "objects"}: + raise ValueError("Stage-A input bundle root filesystem inventory drifted") + manifest_path = root_entries[STAGE_A_INPUT_BUNDLE_FILENAME] + if _bundle_is_link_or_reparse(manifest_path) or not manifest_path.is_file(): + raise ValueError("Stage-A input bundle manifest must be a regular non-link file") + object_root = root_entries["objects"] + object_entries = _bundle_directory_entries( + object_root, + context="Stage-A input bundle object directory", + ) + if set(object_entries) != digests: + raise ValueError("Stage-A input bundle object filesystem inventory drifted") + for digest, path in object_entries.items(): + _require_sha256(digest, context="Stage-A input bundle object filename") + if _bundle_is_link_or_reparse(path) or not path.is_file(): + raise ValueError("Stage-A input bundle object must be a regular non-link file") + + +def _bundle_path_within(path: Path, root: Path) -> bool: + try: + path.relative_to(root) + except ValueError: + return False + return True + + +def _bundle_cache_payload_bytes(cache_dir: Path, returned_path: object, *, context: str) -> bytes: + if not isinstance(returned_path, (str, os.PathLike)): + raise ValueError(f"{context} downloader returned no filesystem path") + cache = _bundle_safe_directory(cache_dir, create=True).resolve(strict=True) + returned = Path(os.path.abspath(returned_path)) + if not _bundle_path_within(returned, cache): + raise ValueError(f"{context} downloader returned a path outside the explicit cache") + try: + resolved = returned.resolve(strict=True) + resolved.relative_to(cache) + except (OSError, ValueError) as error: + raise ValueError(f"{context} cache pointer escapes the explicit cache") from error + candidate = cache + for component in resolved.relative_to(cache).parts: + candidate /= component + if _bundle_is_link_or_reparse(candidate): + raise ValueError(f"{context} resolved cache path traverses a link or reparse point") + return _bundle_stable_file_bytes(resolved, context=context) + + +def _bundle_object_key(record: Mapping[str, Any]) -> tuple[str, str]: + return str(record["role"]), str(record["logical_path"]) + + +def _bundle_lfs_pointer_bytes(*, sha256: str, size_bytes: int) -> bytes: + digest = _require_sha256(sha256, context="Stage-A bundle LFS pointer SHA-256") + size = _require_int(size_bytes, context="Stage-A bundle LFS pointer size", minimum=1) + return ( + f"version https://git-lfs.github.com/spec/v1\noid sha256:{digest}\nsize {size}\n" + ).encode("ascii") + + +def _bundle_expected_object_record( + *, + role: str, + source_id: str, + revision: str, + logical_path: str, + payload: bytes, + git_blob_oid: str | None = None, + lfs_sha256: str | None = None, +) -> dict[str, Any]: + digest = sha256_bytes(payload) + return { + "role": role, + "source_id": source_id, + "revision": revision, + "logical_path": logical_path, + "relative_path": f"objects/{digest}", + "sha256": digest, + "size_bytes": len(payload), + "git_blob_oid": git_blob_oid, + "lfs_sha256": lfs_sha256, + } + + +def _bundle_add_object( + staging_root: Path, + records: list[dict[str, Any]], + *, + role: str, + source_id: str, + revision: str, + logical_path: str, + payload: bytes, + git_blob_oid: str | None = None, + lfs_sha256: str | None = None, +) -> None: + if role not in _STAGE_A_INPUT_BUNDLE_ROLES: + raise ValueError("Stage-A input bundle object role is unsupported") + _bundle_safe_relative_path(logical_path, context=f"{role} logical path") + if not isinstance(payload, bytes) or not payload: + raise ValueError(f"{role} object {logical_path!r} must contain bytes") + digest = sha256_bytes(payload) + if lfs_sha256 is not None and digest != _require_sha256( + lfs_sha256, context=f"{role} {logical_path} LFS SHA-256" + ): + raise ValueError(f"{role} object {logical_path!r} differs from its LFS identity") + if git_blob_oid is not None: + if not isinstance(git_blob_oid, str) or re.fullmatch(r"[0-9a-f]{40}", git_blob_oid) is None: + raise ValueError(f"{role} object {logical_path!r} has an invalid Git blob OID") + git_payload = ( + _bundle_lfs_pointer_bytes(sha256=digest, size_bytes=len(payload)) + if lfs_sha256 is not None + else payload + ) + if _git_blob_sha1(git_payload) != git_blob_oid: + raise ValueError(f"{role} object {logical_path!r} differs from its Git blob") + relative_path = f"objects/{digest}" + destination = staging_root / PurePosixPath(relative_path) + destination.parent.mkdir(exist_ok=True) + if destination.exists(): + if _bundle_stable_file_bytes(destination, context="deduplicated bundle object") != payload: + raise ValueError("content-addressed Stage-A bundle object collided") + else: + destination.write_bytes(payload) + record = { + "role": role, + "source_id": source_id, + "revision": revision, + "logical_path": logical_path, + "relative_path": relative_path, + "sha256": digest, + "size_bytes": len(payload), + "git_blob_oid": git_blob_oid, + "lfs_sha256": lfs_sha256, + } + if _bundle_object_key(record) in {_bundle_object_key(item) for item in records}: + raise ValueError(f"duplicate Stage-A bundle object identity: {role}/{logical_path}") + records.append(record) + + +def _bundle_public_github_revision( + repo_id: str, + revision: str, + *, + opener: Any, +) -> str: + request = urllib.request.Request( + f"https://api.github.com/repos/{repo_id}/commits/{urllib.parse.quote(revision, safe='')}", + headers={ + "Accept": "application/vnd.github+json", + "User-Agent": "RecurQuant-Experiment-013-stage-a-stager", + }, + ) + try: + with opener(request, timeout=30) as response: + value = json.load(response) + except (OSError, urllib.error.HTTPError, json.JSONDecodeError) as error: + raise RuntimeError(f"cannot resolve public GitHub revision for {repo_id}") from error + resolved = _require_string(value.get("sha"), context=f"{repo_id} public revision") + if resolved != revision: + raise ValueError(f"GitHub returned a different object for pinned {repo_id} revision") + return resolved + + +def _bundle_public_source_heads(*, api: Any, opener: Any) -> dict[str, str]: + return _validate_heads( + { + "primary_model": str( + api.model_info( + resolver.PRIMARY_MODEL_ID, + revision=resolver.PRIMARY_MODEL_REVISION, + token=False, + ).sha + ), + "mbpp": str( + api.dataset_info( + resolver.MBPP_DATASET_ID, + revision=resolver.MBPP_REVISION, + token=False, + ).sha + ), + "pg19": str( + api.dataset_info( + resolver.PG19_DATASET_ID, + revision=resolver.PG19_REVISION, + token=False, + ).sha + ), + "ruler": _bundle_public_github_revision( + resolver.RULER_SOURCE_ID, + resolver.RULER_REVISION, + opener=opener, + ), + "humaneval_plus": str( + api.dataset_info( + resolver.HUMANEVAL_PLUS_DATASET_ID, + revision=resolver.HUMANEVAL_PLUS_REVISION, + token=False, + ).sha + ), + "evalplus": _bundle_public_github_revision( + resolver.EVALPLUS_SOURCE_ID, + resolver.EVALPLUS_SOURCE_REVISION, + opener=opener, + ), + }, + context="opaque Stage-A staging", + ) + + +def _bundle_tokenizer_files(frozen_identity_bytes: bytes) -> dict[str, dict[str, Any]]: + document = _strict_json(frozen_identity_bytes, context="frozen Stage-A identity") + evidence = document.get("evidence") + if not isinstance(evidence, Mapping): + raise ValueError("frozen Stage-A identity evidence is unavailable") + tokenizer = evidence.get("tokenizer") + if not isinstance(tokenizer, Mapping): + raise ValueError("frozen Stage-A tokenizer contract is unavailable") + raw_files = tokenizer.get("files") + if not isinstance(raw_files, list): + raise ValueError("frozen Stage-A tokenizer files are unavailable") + result: dict[str, dict[str, Any]] = {} + for index, raw in enumerate(raw_files): + if not isinstance(raw, Mapping) or set(raw) != {"name", "sha256", "size_bytes"}: + raise ValueError(f"frozen Stage-A tokenizer files[{index}] is malformed") + name = _bundle_safe_relative_path(raw["name"], context="tokenizer file name") + if PurePosixPath(name).name != name or name in result: + raise ValueError("frozen Stage-A tokenizer file inventory is unsafe or duplicated") + digest = _require_sha256(raw["sha256"], context=f"tokenizer {name} SHA-256") + size = _require_int(raw["size_bytes"], context=f"tokenizer {name} size", minimum=1) + result[name] = {"name": name, "sha256": digest, "size_bytes": size} + expected = { + name: {"name": name, "size_bytes": size, "sha256": digest} + for name, (size, digest) in RULER_EXPECTED_TOKENIZER_ASSETS.items() + } + if result != expected: + raise ValueError("frozen Stage-A tokenizer inventory differs from the exact four-file set") + return result + + +def _bundle_expected_parquet_files() -> tuple[tuple[Any, Any], ...]: + from recurquant import experiment013_parquet + + manifest = experiment013_parquet.load_experiment013_parquet_manifest( + PARQUET_MATERIALIZATION_MANIFEST_PATH + ) + selected: list[tuple[Any, Any]] = [] + for dataset_key, logical_split in (("pg19", "validation"), ("humaneval_plus", "test")): + dataset = manifest.dataset(dataset_key) + files = tuple(file for file in dataset.files if file.logical_split == logical_split) + if len(files) != 1: + raise RuntimeError( + "Stage-A Parquet inventory must contain exactly two single-file splits" + ) + selected.append((dataset, files[0])) + return tuple(selected) + + +@dataclass(frozen=True, slots=True) +class AuthenticatedStageAInputBundle: + root: Path + manifest: Mapping[str, Any] + manifest_file_sha256: str + objects: Mapping[tuple[str, str], Mapping[str, Any]] + _authentication_seal: object = dataclass_field(repr=False, compare=False) + + def __post_init__(self) -> None: + if self._authentication_seal is not _STAGE_A_INPUT_BUNDLE_AUTHENTICATION_SEAL: + raise ValueError( + "Stage-A input bundles may be created only by the authenticated loader" + ) + + def object_bytes(self, role: str, logical_path: str) -> bytes: + try: + record = self.objects[(role, logical_path)] + except KeyError as error: + raise KeyError(f"unknown Stage-A input bundle object: {role}/{logical_path}") from error + payload = _bundle_stable_descendant_bytes( + self.root, + str(record["relative_path"]), + context=f"Stage-A bundle {role}/{logical_path}", + ) + if len(payload) != record["size_bytes"] or sha256_bytes(payload) != record["sha256"]: + raise ValueError(f"Stage-A bundle object changed: {role}/{logical_path}") + return payload + + +def authenticate_stage_a_input_bundle( + bundle_root: Path, + *, + frozen_stage_a_identity_artifact: bytes, + calibration_binding_artifact: bytes, + execution_binding_artifacts: Mapping[str, bytes], +) -> AuthenticatedStageAInputBundle: + """Authenticate an opaque Stage-A byte bundle without decoding protected rows.""" + + if not isinstance(frozen_stage_a_identity_artifact, bytes): + raise TypeError("frozen Stage-A identity artifact must be bytes") + if not isinstance(calibration_binding_artifact, bytes): + raise TypeError("Stage-A calibration binding artifact must be bytes") + frozen = resolver.deserialize_frozen_stage_a_identity_artifact( + frozen_stage_a_identity_artifact, + calibration_binding_artifact=calibration_binding_artifact, + ) + expected_bindings = _validate_execution_binding_artifacts(execution_binding_artifacts) + if dict(frozen.execution_bindings) != expected_bindings: + raise ValueError("Stage-A input bundle execution bindings differ from the frozen identity") + root = _bundle_safe_directory(bundle_root, create=False) + raw_manifest = _bundle_stable_descendant_bytes( + root, + STAGE_A_INPUT_BUNDLE_FILENAME, + context="Stage-A input bundle manifest", + ) + manifest = _strict_json(raw_manifest, context="Stage-A input bundle manifest") + if canonical_json_bytes(manifest) != raw_manifest: + raise ValueError("Stage-A input bundle manifest is not canonical JSON") + _require_exact_fields( + manifest, + _STAGE_A_INPUT_BUNDLE_MANIFEST_FIELDS, + context="Stage-A input bundle manifest", + ) + if ( + manifest["schema"] != STAGE_A_INPUT_BUNDLE_SCHEMA + or manifest["phase"] != "stage_a" + or manifest["capture_version"] != CAPTURE_VERSION + or manifest["staging_profile"] != STAGE_A_INPUT_BUNDLE_PROFILE + or manifest["frozen_identity_file_sha256"] != frozen.file_sha256 + or manifest["calibration_binding_file_sha256"] != sha256_bytes(calibration_binding_artifact) + or manifest["execution_bindings"] != expected_bindings + or manifest["source_heads"] != EXPECTED_SOURCE_HEADS + or manifest["model_hub_manifest_file_sha256"] + != expected_bindings["model_file_manifest_file_sha256"] + ): + raise ValueError("Stage-A input bundle identity or staging contract drifted") + snapshots = manifest["parquet_hub_snapshots"] + if not isinstance(snapshots, list) or len(snapshots) != 2: + raise ValueError("Stage-A input bundle must bind two Parquet snapshots") + raw_objects = manifest["objects"] + if not isinstance(raw_objects, list) or not raw_objects: + raise ValueError("Stage-A input bundle contains no objects") + objects: dict[tuple[str, str], Mapping[str, Any]] = {} + normalized_records: list[dict[str, Any]] = [] + for index, raw in enumerate(raw_objects): + if not isinstance(raw, Mapping): + raise ValueError(f"Stage-A input bundle objects[{index}] must be an object") + _require_exact_fields( + raw, + _STAGE_A_INPUT_BUNDLE_OBJECT_FIELDS, + context=f"Stage-A input bundle objects[{index}]", + ) + role = _require_string(raw["role"], context=f"bundle objects[{index}].role") + if role not in _STAGE_A_INPUT_BUNDLE_ROLES: + raise ValueError("Stage-A input bundle contains an unknown object role") + source_id = _require_string(raw["source_id"], context=f"bundle objects[{index}].source_id") + revision = _require_string(raw["revision"], context=f"bundle objects[{index}].revision") + logical_path = _bundle_safe_relative_path( + raw["logical_path"], context=f"bundle objects[{index}].logical_path" + ) + relative_path = _bundle_safe_relative_path( + raw["relative_path"], context=f"bundle objects[{index}].relative_path" + ) + digest = _require_sha256(raw["sha256"], context=f"bundle objects[{index}].sha256") + size = _require_int(raw["size_bytes"], context=f"bundle objects[{index}].size", minimum=1) + if relative_path != f"objects/{digest}": + raise ValueError("Stage-A input bundle object is not content-addressed") + git_blob_oid = raw["git_blob_oid"] + if git_blob_oid is not None and ( + not isinstance(git_blob_oid, str) or re.fullmatch(r"[0-9a-f]{40}", git_blob_oid) is None + ): + raise ValueError("Stage-A input bundle object Git blob OID is invalid") + lfs_sha256 = raw["lfs_sha256"] + if lfs_sha256 is not None: + _require_sha256(lfs_sha256, context="Stage-A input bundle object LFS SHA-256") + record = { + "role": role, + "source_id": source_id, + "revision": revision, + "logical_path": logical_path, + "relative_path": relative_path, + "sha256": digest, + "size_bytes": size, + "git_blob_oid": git_blob_oid, + "lfs_sha256": lfs_sha256, + } + key = _bundle_object_key(record) + if key in objects: + raise ValueError("Stage-A input bundle object identities are duplicated") + objects[key] = MappingProxyType(record) + normalized_records.append(record) + if normalized_records != sorted( + normalized_records, + key=lambda item: (item["role"], item["source_id"], item["logical_path"]), + ): + raise ValueError("Stage-A input bundle object inventory is not canonical") + object_digests = {str(record["sha256"]) for record in normalized_records} + _bundle_authenticate_filesystem_inventory(root, digests=object_digests) + bundle = AuthenticatedStageAInputBundle( + root=root, + manifest=_bundle_deep_freeze(manifest), + manifest_file_sha256=sha256_bytes(raw_manifest), + objects=_bundle_deep_freeze(objects), + _authentication_seal=_STAGE_A_INPUT_BUNDLE_AUTHENTICATION_SEAL, + ) + + expected_records: dict[tuple[str, str], dict[str, Any]] = {} + model_bytes = bundle.object_bytes("model_hub_manifest", "model-file-manifest.json") + if model_bytes != execution_binding_artifacts["model_file_manifest_file_sha256"]: + raise ValueError("Stage-A bundle model Hub attestation differs from the frozen manifest") + expected_records[("model_hub_manifest", "model-file-manifest.json")] = ( + _bundle_expected_object_record( + role="model_hub_manifest", + source_id=resolver.PRIMARY_MODEL_ID, + revision=resolver.PRIMARY_MODEL_REVISION, + logical_path="model-file-manifest.json", + payload=model_bytes, + ) + ) + tokenizer_files = _bundle_tokenizer_files(frozen_stage_a_identity_artifact) + for name, expected in tokenizer_files.items(): + payload = bundle.object_bytes("tokenizer", name) + if len(payload) != expected["size_bytes"] or sha256_bytes(payload) != expected["sha256"]: + raise ValueError(f"Stage-A bundle tokenizer object drifted: {name}") + expected_records[("tokenizer", name)] = _bundle_expected_object_record( + role="tokenizer", + source_id=resolver.PRIMARY_MODEL_ID, + revision=resolver.PRIMARY_MODEL_REVISION, + logical_path=name, + payload=payload, + ) + for path, git_blob_oid in RULER_GENERATOR_GIT_BLOBS.items(): + payload = bundle.object_bytes("ruler_generator", path) + if _git_blob_sha1(payload) != git_blob_oid: + raise ValueError(f"Stage-A bundle RULER generator object drifted: {path}") + expected_records[("ruler_generator", path)] = _bundle_expected_object_record( + role="ruler_generator", + source_id=resolver.RULER_SOURCE_ID, + revision=resolver.RULER_REVISION, + logical_path=path, + payload=payload, + git_blob_oid=git_blob_oid, + ) + generation_manifest = bundle.object_bytes( + "ruler_generation_manifest", + "generation-manifest.json", + ) + expected_records[("ruler_generation_manifest", "generation-manifest.json")] = ( + _bundle_expected_object_record( + role="ruler_generation_manifest", + source_id=resolver.RULER_SOURCE_ID, + revision=resolver.RULER_REVISION, + logical_path="generation-manifest.json", + payload=generation_manifest, + ) + ) + for item in required_ruler_receipts(): + filename = str(item["filename"]) + payload = bundle.object_bytes("ruler_receipt", filename) + expected_records[("ruler_receipt", filename)] = _bundle_expected_object_record( + role="ruler_receipt", + source_id=resolver.RULER_SOURCE_ID, + revision=resolver.RULER_REVISION, + logical_path=filename, + payload=payload, + ) + expected_snapshots: list[dict[str, Any]] = [] + for dataset, file in _bundle_expected_parquet_files(): + logical = f"{dataset.key}/{file.logical_split}/{file.immutable_path}" + payload = bundle.object_bytes("parquet", logical) + if ( + len(payload) != file.size_bytes + or sha256_bytes(payload) != file.lfs_sha256 + or _git_blob_sha1( + _bundle_lfs_pointer_bytes( + sha256=file.lfs_sha256, + size_bytes=file.size_bytes, + ) + ) + != file.git_blob_oid + ): + raise ValueError(f"Stage-A bundle Parquet object drifted: {logical}") + expected_records[("parquet", logical)] = _bundle_expected_object_record( + role="parquet", + source_id=dataset.dataset_id, + revision=dataset.conversion_revision, + logical_path=logical, + payload=payload, + git_blob_oid=file.git_blob_oid, + lfs_sha256=file.lfs_sha256, + ) + expected_snapshots.append( + { + "dataset_key": dataset.key, + "dataset_id": dataset.dataset_id, + "source_revision": dataset.source_revision, + "conversion_revision": dataset.conversion_revision, + "files": [ + { + "path": file.immutable_path, + "git_blob_oid": file.git_blob_oid, + "lfs_sha256": file.lfs_sha256, + "size_bytes": file.size_bytes, + } + ], + } + ) + if snapshots != expected_snapshots: + raise ValueError("Stage-A input bundle Parquet Hub snapshots drifted") + if {key: dict(record) for key, record in objects.items()} != expected_records: + raise ValueError("Stage-A input bundle object records differ from the frozen semantics") + return bundle + + +def stage_stage_a_input_bundle( + *, + bundle_root: Path, + cache_dir: Path, + ruler_receipt_dir: Path, + frozen_stage_a_identity_artifact: bytes, + calibration_binding_artifact: bytes, + execution_binding_artifacts: Mapping[str, bytes], + runtime_authentication_context: Mapping[str, object], +) -> AuthenticatedStageAInputBundle: + """Stage exact public bytes without decoding any protected Stage-A content.""" + + destination = Path(os.path.abspath(bundle_root)) + repository = REPOSITORY_ROOT.resolve(strict=True) + if _bundle_path_within(destination, repository): + raise ValueError("Stage-A input bundle must be outside the repository") + parent = _bundle_safe_directory(destination.parent, create=True) + cache = _bundle_safe_directory(cache_dir, create=True) + if _bundle_path_within(destination, cache) or _bundle_path_within(cache, destination): + raise ValueError("Stage-A input bundle and shared Hub cache must not be nested") + ruler_root = _bundle_safe_directory(ruler_receipt_dir, create=False) + frozen = resolver.deserialize_frozen_stage_a_identity_artifact( + frozen_stage_a_identity_artifact, + calibration_binding_artifact=calibration_binding_artifact, + ) + expected_execution_bindings = _validate_execution_binding_artifacts(execution_binding_artifacts) + if dict(frozen.execution_bindings) != expected_execution_bindings: + raise ValueError("opaque stager inputs differ from the frozen Stage-A identity") + tokenizer_files = _bundle_tokenizer_files(frozen_stage_a_identity_artifact) + runtime_context = _normalize_runtime_authentication_context(runtime_authentication_context) + authentication = _authenticate_execution_binding_artifacts( + execution_binding_artifacts, + runtime_context=runtime_context, + ) + try: + if os.path.lexists(destination): + return authenticate_stage_a_input_bundle( + destination, + frozen_stage_a_identity_artifact=frozen_stage_a_identity_artifact, + calibration_binding_artifact=calibration_binding_artifact, + execution_binding_artifacts=execution_binding_artifacts, + ) + try: + from huggingface_hub import HfApi, hf_hub_download + except ModuleNotFoundError as error: # pragma: no cover - dependency guard + raise RuntimeError("opaque Stage-A staging requires huggingface-hub") from error + api = HfApi(endpoint="https://huggingface.co", token=False) + opener = urllib.request.urlopen + before = _bundle_public_source_heads(api=api, opener=opener) + staging_root = Path(tempfile.mkdtemp(prefix=f".{destination.name}.staging-", dir=parent)) + records: list[dict[str, Any]] = [] + try: + _bundle_add_object( + staging_root, + records, + role="model_hub_manifest", + source_id=resolver.PRIMARY_MODEL_ID, + revision=resolver.PRIMARY_MODEL_REVISION, + logical_path="model-file-manifest.json", + payload=execution_binding_artifacts["model_file_manifest_file_sha256"], + ) + for name, expected in sorted(tokenizer_files.items()): + returned = hf_hub_download( + repo_id=resolver.PRIMARY_MODEL_ID, + filename=name, + repo_type="model", + revision=resolver.PRIMARY_MODEL_REVISION, + cache_dir=cache, + token=False, + endpoint="https://huggingface.co", + ) + payload = _bundle_cache_payload_bytes(cache, returned, context=f"tokenizer {name}") + if ( + len(payload) != expected["size_bytes"] + or sha256_bytes(payload) != expected["sha256"] + ): + raise ValueError(f"downloaded tokenizer object drifted: {name}") + _bundle_add_object( + staging_root, + records, + role="tokenizer", + source_id=resolver.PRIMARY_MODEL_ID, + revision=resolver.PRIMARY_MODEL_REVISION, + logical_path=name, + payload=payload, + ) + + parquet_snapshots: list[dict[str, Any]] = [] + from recurquant import experiment013_parquet + + metadata = experiment013_parquet.HuggingFaceHubMetadataBackend(token=False) + for dataset, file in _bundle_expected_parquet_files(): + if ( + metadata.resolve_dataset_revision( + repo_id=dataset.dataset_id, + revision=dataset.source_revision, + ) + != dataset.source_revision + ): + raise ValueError("Stage-A Parquet source revision drifted during staging") + snapshot = metadata.snapshot_parquet_files( + repo_id=dataset.dataset_id, + revision=dataset.conversion_revision, + paths=(file.immutable_path,), + ) + observed = snapshot.files[0] if len(snapshot.files) == 1 else None + if ( + snapshot.commit_hash != dataset.conversion_revision + or observed is None + or observed.path != file.immutable_path + or observed.git_blob_oid != file.git_blob_oid + or observed.lfs_sha256 != file.lfs_sha256 + or observed.size_bytes != file.size_bytes + or observed.lfs_size_bytes != file.lfs_size_bytes + or observed.etag != file.lfs_sha256 + ): + raise ValueError("Stage-A Parquet Hub metadata drifted during staging") + returned = hf_hub_download( + repo_id=dataset.dataset_id, + filename=file.immutable_path, + repo_type="dataset", + revision=dataset.conversion_revision, + cache_dir=cache, + token=False, + endpoint="https://huggingface.co", + ) + payload = _bundle_cache_payload_bytes( + cache, + returned, + context=f"Parquet {dataset.key}/{file.logical_split}", + ) + if len(payload) != file.size_bytes or sha256_bytes(payload) != file.lfs_sha256: + raise ValueError("downloaded Stage-A Parquet object differs from frozen LFS") + logical = f"{dataset.key}/{file.logical_split}/{file.immutable_path}" + _bundle_add_object( + staging_root, + records, + role="parquet", + source_id=dataset.dataset_id, + revision=dataset.conversion_revision, + logical_path=logical, + payload=payload, + git_blob_oid=file.git_blob_oid, + lfs_sha256=file.lfs_sha256, + ) + parquet_snapshots.append( + { + "dataset_key": dataset.key, + "dataset_id": dataset.dataset_id, + "source_revision": dataset.source_revision, + "conversion_revision": dataset.conversion_revision, + "files": [ + { + "path": file.immutable_path, + "git_blob_oid": file.git_blob_oid, + "lfs_sha256": file.lfs_sha256, + "size_bytes": file.size_bytes, + } + ], + } + ) + + raw_base = ( + "https://raw.githubusercontent.com/" + f"{resolver.RULER_SOURCE_ID}/{resolver.RULER_REVISION}/" + ) + for path, git_blob_oid in sorted(RULER_GENERATOR_GIT_BLOBS.items()): + request = urllib.request.Request( + raw_base + path, + headers={"User-Agent": "RecurQuant-Experiment-013-stage-a-stager"}, + ) + try: + with opener(request, timeout=30) as response: + payload = response.read() + except (OSError, urllib.error.HTTPError) as error: + raise RuntimeError(f"cannot stage pinned RULER source file {path}") from error + _bundle_add_object( + staging_root, + records, + role="ruler_generator", + source_id=resolver.RULER_SOURCE_ID, + revision=resolver.RULER_REVISION, + logical_path=path, + payload=payload, + git_blob_oid=git_blob_oid, + ) + generation_manifest = _bundle_stable_file_bytes( + ruler_root / "generation-manifest.json", + context="RULER generation manifest for opaque staging", + ) + _bundle_add_object( + staging_root, + records, + role="ruler_generation_manifest", + source_id=resolver.RULER_SOURCE_ID, + revision=resolver.RULER_REVISION, + logical_path="generation-manifest.json", + payload=generation_manifest, + ) + for item in required_ruler_receipts(): + filename = str(item["filename"]) + payload = _bundle_stable_file_bytes( + ruler_root / filename, + context=f"RULER receipt {filename} for opaque staging", + ) + _bundle_add_object( + staging_root, + records, + role="ruler_receipt", + source_id=resolver.RULER_SOURCE_ID, + revision=resolver.RULER_REVISION, + logical_path=filename, + payload=payload, + ) + after = _bundle_public_source_heads(api=api, opener=opener) + if after != before: + raise ValueError("public source heads changed during opaque Stage-A staging") + records.sort(key=lambda item: (item["role"], item["source_id"], item["logical_path"])) + manifest = { + "schema": STAGE_A_INPUT_BUNDLE_SCHEMA, + "phase": "stage_a", + "capture_version": CAPTURE_VERSION, + "staging_profile": STAGE_A_INPUT_BUNDLE_PROFILE, + "frozen_identity_file_sha256": frozen.file_sha256, + "calibration_binding_file_sha256": sha256_bytes(calibration_binding_artifact), + "execution_bindings": dict(frozen.execution_bindings), + "source_heads": before, + "model_hub_manifest_file_sha256": sha256_bytes( + execution_binding_artifacts["model_file_manifest_file_sha256"] + ), + "parquet_hub_snapshots": parquet_snapshots, + "objects": records, + } + (staging_root / STAGE_A_INPUT_BUNDLE_FILENAME).write_bytes( + canonical_json_bytes(manifest) + ) + authenticate_stage_a_input_bundle( + staging_root, + frozen_stage_a_identity_artifact=frozen_stage_a_identity_artifact, + calibration_binding_artifact=calibration_binding_artifact, + execution_binding_artifacts=execution_binding_artifacts, + ) + lock_path = parent / f".{destination.name}.publish.lock" + lock_payload = canonical_json_bytes( + { + "bundle_manifest_sha256": sha256_bytes(canonical_json_bytes(manifest)), + "owner_nonce": os.urandom(32).hex(), + "staging_directory": staging_root.name, + } + ) + lock_owned = False + try: + descriptor = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + lock_owned = True + except FileExistsError as error: + raise FileExistsError( + "Stage-A input bundle publication is already owned" + ) from error + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(lock_payload) + handle.flush() + os.fsync(handle.fileno()) + os.rename(staging_root, destination) + finally: + if lock_owned and os.path.lexists(lock_path): + observed_lock = _bundle_stable_file_bytes( + lock_path, + context="Stage-A input bundle publication lock", + ) + if observed_lock != lock_payload: + raise ValueError("Stage-A input bundle publication lock ownership changed") + lock_path.unlink() + if os.path.lexists(lock_path): + raise ValueError( + "Stage-A input bundle publication lock survived owned cleanup" + ) + except BaseException: + # The uniquely-owned temporary tree is intentionally retained for + # forensic inspection. It is never treated as an authenticated bundle. + raise + return authenticate_stage_a_input_bundle( + destination, + frozen_stage_a_identity_artifact=frozen_stage_a_identity_artifact, + calibration_binding_artifact=calibration_binding_artifact, + execution_binding_artifacts=execution_binding_artifacts, + ) + finally: + if sys.modules.get(_CALIBRATION_RUNNER_MODULE_NAME) is authentication.runner: + sys.modules.pop(_CALIBRATION_RUNNER_MODULE_NAME, None) + + +class _StagedParquetHubBackend: + def __init__(self, bundle: AuthenticatedStageAInputBundle) -> None: + from recurquant import experiment013_parquet + + self._module = experiment013_parquet + self._snapshots = { + str(item["dataset_id"]): item for item in bundle.manifest["parquet_hub_snapshots"] + } + + def resolve_dataset_revision(self, *, repo_id: str, revision: str) -> str: + snapshot = self._snapshots.get(repo_id) + if snapshot is None or snapshot["source_revision"] != revision: + raise ValueError("offline Parquet source revision is absent or different") + return revision + + def snapshot_parquet_files( + self, + *, + repo_id: str, + revision: str, + paths: tuple[str, ...], + ) -> Any: + snapshot = self._snapshots.get(repo_id) + if snapshot is None or snapshot["conversion_revision"] != revision: + raise ValueError("offline Parquet conversion revision is absent or different") + files_by_path = {str(item["path"]): item for item in snapshot["files"]} + files = [] + for path in paths: + item = files_by_path.get(path) + if item is None: + raise ValueError("offline Parquet snapshot omitted a requested path") + files.append( + self._module.HubFileMetadata( + path=path, + commit_hash=revision, + size_bytes=int(item["size_bytes"]), + git_blob_oid=str(item["git_blob_oid"]), + lfs_sha256=str(item["lfs_sha256"]), + lfs_size_bytes=int(item["size_bytes"]), + etag=str(item["lfs_sha256"]), + ) + ) + return self._module.HubDatasetMetadata(commit_hash=revision, files=tuple(files)) + + +class _StagedParquetBackend: + def __init__(self, bundle: AuthenticatedStageAInputBundle) -> None: + self._bundle = bundle + self._uris: dict[str, str] = {} + for snapshot in bundle.manifest["parquet_hub_snapshots"]: + dataset_key = str(snapshot["dataset_key"]) + dataset_id = str(snapshot["dataset_id"]) + revision = str(snapshot["conversion_revision"]) + for file in snapshot["files"]: + path = str(file["path"]) + split = "validation" if dataset_key == "pg19" else "test" + logical = f"{dataset_key}/{split}/{path}" + self._uris[f"hf://datasets/{dataset_id}@{revision}/{path}"] = logical + + def _parquet_file(self, uri: str) -> Any: + from io import BytesIO + + import pyarrow.parquet as parquet + + try: + logical_path = self._uris[uri] + except KeyError as error: + raise ValueError("offline Parquet URI is outside the staged inventory") from error + payload = self._bundle.object_bytes("parquet", logical_path) + return parquet.ParquetFile(BytesIO(payload)) + + def inspect(self, uri: str) -> Any: + from recurquant import experiment013_parquet + + parquet_file = self._parquet_file(uri) + metadata = parquet_file.metadata + return experiment013_parquet.ParquetFileLayout( + row_group_rows=tuple( + metadata.row_group(index).num_rows for index in range(metadata.num_row_groups) + ), + columns=tuple(parquet_file.schema_arrow.names), + ) + + def read_row( + self, + uri: str, + *, + row_group_index: int, + row_index_in_group: int, + columns: tuple[str, ...], + ) -> Mapping[str, object]: + parquet_file = self._parquet_file(uri) + table = parquet_file.read_row_group(row_group_index, columns=list(columns)) + rows = table.slice(row_index_in_group, 1).to_pylist() + if len(rows) != 1 or not isinstance(rows[0], Mapping): + raise ValueError("offline Parquet backend did not return exactly one row") + return rows[0] + + def read_row_group_projection( + self, + uri: str, + *, + row_group_index: int, + columns: tuple[str, ...], + ) -> Sequence[Mapping[str, object]]: + return ( + self._parquet_file(uri) + .read_row_group( + row_group_index, + columns=list(columns), + ) + .to_pylist() + ) + + +class StagedCaptureSource: + """Strictly local CaptureSource backed by one authenticated opaque bundle.""" + + def __init__(self, bundle: AuthenticatedStageAInputBundle) -> None: + if not isinstance(bundle, AuthenticatedStageAInputBundle): + raise TypeError("StagedCaptureSource requires an authenticated Stage-A input bundle") + _require_stage_a_offline_environment() + self.bundle = bundle + self._hub = _StagedParquetHubBackend(bundle) + self._parquet = _StagedParquetBackend(bundle) + + def model_file_manifest_attestation(self) -> bytes: + return self.bundle.object_bytes("model_hub_manifest", "model-file-manifest.json") + + def source_heads(self) -> Mapping[str, str]: + return MappingProxyType(dict(self.bundle.manifest["source_heads"])) + + def tokenizer_material(self) -> TokenizerMaterial: + try: + from transformers import AutoTokenizer + except ModuleNotFoundError as error: # pragma: no cover - dependency guard + raise RuntimeError("offline Stage-A capture requires Transformers") from error + records = { + logical_path: record + for (role, logical_path), record in self.bundle.objects.items() + if role == "tokenizer" + } + files = {name: self.bundle.object_bytes("tokenizer", name) for name in sorted(records)} + with tempfile.TemporaryDirectory(prefix="recurquant-exp013-staged-tokenizer-") as temporary: + isolated = Path(temporary) + for name, payload in files.items(): + (isolated / name).write_bytes(payload) + tokenizer = AutoTokenizer.from_pretrained( + isolated, + local_files_only=True, + trust_remote_code=False, + token=False, + ) + inventory = { + path.relative_to(isolated).as_posix() + for path in isolated.rglob("*") + if path.is_file() + } + if inventory != set(files): + raise ValueError("offline tokenizer construction changed the staged inventory") + return TokenizerMaterial( + tokenizer=tokenizer, + tokenizer_class=tokenizer.__class__.__name__, + transformers_version=importlib.metadata.version("transformers"), + files=MappingProxyType(files), + model_weights_loaded=False, + ) + + def mbpp_train_rows(self) -> Sequence[Mapping[str, Any]]: + raise RuntimeError("Stage-A offline source forbids MBPP payload access") + + def pg19_projection(self, split: str) -> Sequence[ProjectionRow]: + if split != "validation": + raise ValueError("Stage-A offline source permits only PG19 validation") + from recurquant import experiment013_parquet + + projection = experiment013_parquet.project_experiment013_parquet_columns( + "pg19", + split, + columns=("url",), + expected_count=50, + hub_backend=self._hub, + parquet_backend=self._parquet, + ) + return tuple( + ProjectionRow( + _require_string(row.values[0], context="PG19 validation url"), + row.global_offset, + ) + for row in projection.rows + ) + + def pg19_row(self, split: str, *, offset: int, expected_url: str) -> Mapping[str, Any]: + if split != "validation": + raise ValueError("Stage-A offline source permits only PG19 validation") + from recurquant import experiment013_parquet + + selected = experiment013_parquet.read_experiment013_parquet_row( + "pg19", + split, + offset, + columns=("url", "text"), + hub_backend=self._hub, + parquet_backend=self._parquet, + ) + row = dict(selected.values) + if row.get("url") != expected_url: + raise ValueError("offline PG19 row URL differs from its projection") + return row + + def ruler_generator_files(self) -> Mapping[str, bytes]: + return MappingProxyType( + { + path: self.bundle.object_bytes("ruler_generator", path) + for path in sorted(RULER_GENERATOR_GIT_BLOBS) + } + ) + + def ruler_generation_manifest_bytes(self) -> bytes: + return self.bundle.object_bytes("ruler_generation_manifest", "generation-manifest.json") + + def ruler_receipt_bytes( + self, + *, + category: str, + config: str, + configured_length: int, + seed: int, + ) -> bytes: + filename = ruler_receipt_filename( + category=category, + config=config, + configured_length=configured_length, + seed=seed, + ) + return self.bundle.object_bytes("ruler_receipt", filename) + + def humaneval_projection(self) -> Sequence[ProjectionRow]: + from recurquant import experiment013_parquet + + projection = experiment013_parquet.project_experiment013_parquet_columns( + "humaneval_plus", + "test", + columns=("task_id",), + expected_count=164, + hub_backend=self._hub, + parquet_backend=self._parquet, + ) + return tuple( + ProjectionRow( + _require_string(row.values[0], context="HumanEval+ task_id"), + row.global_offset, + ) + for row in projection.rows + ) + + def humaneval_row(self, *, offset: int, expected_task_id: str) -> Mapping[str, Any]: + from recurquant import experiment013_parquet + + selected = experiment013_parquet.read_experiment013_parquet_row( + "humaneval_plus", + "test", + offset, + columns=("task_id", "prompt", "canonical_solution"), + hub_backend=self._hub, + parquet_backend=self._parquet, + ) + row = dict(selected.values) + if row.get("task_id") != expected_task_id: + raise ValueError("offline HumanEval+ row differs from its projection") + return row + + +def _parse_named_cli_values( + values: Sequence[str], + *, + context: str, + paths: bool, +) -> dict[str, object]: + parsed: dict[str, object] = {} + for raw in values: + if not isinstance(raw, str) or "=" not in raw: + raise ValueError(f"{context} must use NAME=VALUE") + name, rendered = raw.split("=", 1) + if _RUNTIME_ROOT_NAME_RE.fullmatch(name) is None or not rendered: + raise ValueError(f"{context} contains a non-canonical name or empty value") + if name in parsed: + raise ValueError(f"{context} contains a duplicate name") + parsed[name] = Path(rendered) if paths else rendered + if not parsed: + raise ValueError(f"at least one {context} is required") + return parsed + + +def _runtime_context_from_cli(args: argparse.Namespace) -> _RuntimeAuthenticationContext: + if ( + args.base_runtime_root is None + or args.git_executable is None + or args.staged_interpreter is None + ): + raise ValueError( + "capture requires --base-runtime-root, --git-executable, and --staged-interpreter" + ) + roots = _parse_named_cli_values( + args.package_root, + context="--package-root", + paths=True, + ) + import_paths = _parse_named_cli_values( + args.package_import_path, + context="--package-import-path", + paths=False, + ) + return _normalize_runtime_authentication_context( + { + "base_runtime_root": args.base_runtime_root, + "git_executable": args.git_executable, + "staged_interpreter": args.staged_interpreter, + "package_runtime_roots": roots, + "package_import_paths": import_paths, + } + ) + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( "Capture a calibration or Stage-A Experiment 013 identity input. " "No model weights are requested or loaded." ) @@ -3141,6 +4815,7 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.add_argument("--model-file-manifest", type=Path) parser.add_argument("--parquet-materialization-manifest", type=Path) parser.add_argument("--base-runtime-root", type=Path) + parser.add_argument("--git-executable", type=Path) parser.add_argument("--staged-interpreter", type=Path) parser.add_argument("--package-root", action="append", default=[]) parser.add_argument("--package-import-path", action="append", default=[]) @@ -3151,7 +4826,7 @@ def main(argv: Sequence[str] | None = None) -> int: args = parse_args(argv) if args.phase in resolver.PROTECTED_STAGES: raise PermissionError( - f"{args.phase} is protected; capture v4 refuses it before file or source access" + f"{args.phase} is protected; capture v5 refuses it before file or source access" ) if args.ruler_receipt_dir is None: raise ValueError("--ruler-receipt-dir is required") @@ -3171,9 +4846,7 @@ def main(argv: Sequence[str] | None = None) -> int: "repository_source_manifest_file_sha256": args.repository_source_manifest, "calibration_runtime_manifest_file_sha256": args.calibration_runtime_manifest, "model_file_manifest_file_sha256": args.model_file_manifest, - "parquet_materialization_manifest_file_sha256": ( - args.parquet_materialization_manifest - ), + "parquet_materialization_manifest_file_sha256": (args.parquet_materialization_manifest), } if any(path is None for path in binding_paths.values()): raise ValueError( @@ -3195,6 +4868,7 @@ def main(argv: Sequence[str] | None = None) -> int: execution_binding_artifacts=execution_binding_artifacts, runtime_authentication_context={ "base_runtime_root": runtime_context.base_runtime_root, + "git_executable": runtime_context.git_executable, "staged_interpreter": runtime_context.staged_interpreter, "package_runtime_roots": runtime_context.package_runtime_roots, "package_import_paths": runtime_context.package_import_paths, diff --git a/scripts/generate_static_q468_ruler_receipts.py b/scripts/generate_static_q468_ruler_receipts.py index b84b3ba..423302f 100644 --- a/scripts/generate_static_q468_ruler_receipts.py +++ b/scripts/generate_static_q468_ruler_receipts.py @@ -550,13 +550,22 @@ def _subprocess_env(**updates: str) -> dict[str, str]: def _git_env() -> dict[str, str]: """Return a Git environment without caller or machine configuration.""" - env = {key: value for key, value in os.environ.items() if not key.upper().startswith("GIT_")} + inherited = {key.upper(): (key, value) for key, value in os.environ.items()} + env = { + inherited[name][0]: inherited[name][1] + for name in ("SYSTEMROOT", "WINDIR", "COMSPEC") + if name in inherited + } env.update( { "GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_SYSTEM": os.devnull, "GIT_CONFIG_GLOBAL": os.devnull, "GIT_NO_REPLACE_OBJECTS": "1", + "GIT_AUTHOR_NAME": "RecurQuant Experiment 013", + "GIT_AUTHOR_EMAIL": "experiment013@invalid", + "GIT_COMMITTER_NAME": "RecurQuant Experiment 013", + "GIT_COMMITTER_EMAIL": "experiment013@invalid", "LC_ALL": "C", "LANG": "C", } @@ -564,6 +573,36 @@ def _git_env() -> dict[str, str]: return env +def _authenticated_git_executable(path: Path | None) -> Path: + selected: str | os.PathLike[str] + if path is None: + discovered = shutil.which("git") + if discovered is None: + raise ValueError("Git executable is unavailable") + selected = discovered + else: + selected = path + try: + resolved = Path(selected).resolve(strict=True) + except OSError as error: + raise ValueError("Git executable is unavailable") from error + if resolved.name.casefold() == "git.exe" and resolved.parent.name.casefold() == "cmd": + try: + resolved = (resolved.parent.parent / "mingw64" / "bin" / "git.exe").resolve(strict=True) + except OSError as error: + raise ValueError( + "Git-for-Windows cmd shim has no canonical mingw64 executable" + ) from error + current = Path(resolved.anchor) + for part in resolved.parts[1:]: + current /= part + if _is_reparse_point(current): + raise ValueError("Git executable traverses a link or reparse point") + if not resolved.is_file() or _is_reparse_point(resolved) or resolved.stat().st_size <= 0: + raise ValueError("Git executable must be a non-empty regular non-link file") + return resolved + + def _file_entry(name: str, data: bytes) -> dict[str, object]: if not data: raise ValueError(f"bound file {name!r} is empty") @@ -958,10 +997,16 @@ def _launcher_source_entry() -> dict[str, object]: ) -def verify_ruler_checkout(ruler_root: Path, capture: Any) -> VerifiedRulerCheckout: +def verify_ruler_checkout( + ruler_root: Path, + capture: Any, + *, + git_executable_path: Path | None = None, +) -> VerifiedRulerCheckout: ruler_root = ruler_root.resolve() + git_executable = _authenticated_git_executable(git_executable_path) result = subprocess.run( - ["git", "-C", str(ruler_root), "rev-parse", "HEAD"], + [str(git_executable), "-C", str(ruler_root), "rev-parse", "HEAD"], check=True, capture_output=True, text=True, @@ -973,7 +1018,7 @@ def verify_ruler_checkout(ruler_root: Path, capture: Any) -> VerifiedRulerChecko files: dict[str, bytes] = {} for relative, expected_blob in capture.RULER_GENERATOR_GIT_BLOBS.items(): data = subprocess.run( - ["git", "-C", str(ruler_root), "cat-file", "blob", expected_blob], + [str(git_executable), "-C", str(ruler_root), "cat-file", "blob", expected_blob], check=True, capture_output=True, env=_git_env(), @@ -2210,6 +2255,7 @@ def finalize_generation_manifest_if_complete( def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--ruler-root", type=Path, required=True) + parser.add_argument("--git-executable", type=Path, required=True) parser.add_argument("--python", type=Path, required=True) parser.add_argument("--tokenizer-dir", type=Path, required=True) parser.add_argument("--nltk-data", type=Path, required=True) @@ -2230,7 +2276,11 @@ def main(argv: Sequence[str] | None = None) -> int: if args.timeout_seconds < 1: raise ValueError("--timeout-seconds must be positive") capture = _load_capture_module() - checkout = verify_ruler_checkout(args.ruler_root, capture) + checkout = verify_ruler_checkout( + args.ruler_root, + capture, + git_executable_path=args.git_executable, + ) verified_package_tree = verify_runtime_package_source(args.python) verified_python_runtime = verify_python_runtime_source(args.python) static_runtime_files = verify_static_runtime_input_source( diff --git a/scripts/launch_static_q468_calibration.py b/scripts/launch_static_q468_calibration.py index ef100f9..f4b3e87 100644 --- a/scripts/launch_static_q468_calibration.py +++ b/scripts/launch_static_q468_calibration.py @@ -26,20 +26,21 @@ from typing import Final RUNTIME_MANIFEST_KIND: Final = "recurquant_experiment013_calibration_runtime_manifest" -RUNTIME_MANIFEST_SCHEMA: Final = 3 -IDENTITY_SCHEMA: Final = 4 +RUNTIME_MANIFEST_SCHEMA: Final = 4 +IDENTITY_SCHEMA: Final = 5 BASE_RUNTIME_ROOT_NAME: Final = "base-runtime" RUNNER_SOURCE_PATH: Final = "scripts/run_static_q468_calibration.py" RUNNER_MODULE_NAME: Final = "_recurquant_experiment013_sealed_runner" +RUN_REPORT_KIND: Final = "recurquant_experiment013_calibration_run" +RUN_REPORT_SCHEMA: Final = 2 +FISHER_SMOKE_COMPLETE_BYTES: Final = b"recurquant-experiment013-fisher-h1-smoke-complete-v1\n" _WINDOWS_REPARSE_POINT: Final = 0x400 _SHA256_RE: Final = re.compile(r"[0-9a-f]{64}") _SHA1_RE: Final = re.compile(r"[0-9a-f]{40}") _ROOT_NAME_RE: Final = re.compile(r"[a-z][a-z0-9-]{0,63}") _FORBIDDEN_SUFFIXES: Final = frozenset({".egg-link", ".pth", ".pyc", ".pyo", "._pth"}) _FORBIDDEN_DIRECTORIES: Final = frozenset({"__pycache__"}) -_FORBIDDEN_FILENAMES: Final = frozenset( - {"pyvenv.cfg", "sitecustomize.py", "usercustomize.py"} -) +_FORBIDDEN_FILENAMES: Final = frozenset({"pyvenv.cfg", "sitecustomize.py", "usercustomize.py"}) _WINDOWS_RESERVED_NAMES: Final = frozenset( { "aux", @@ -87,6 +88,12 @@ *_EXPECTED_BOUND_DIGEST_OPTIONS.values(), } ) +_SMOKE_PREREQUISITE_OPTIONS: Final = frozenset( + { + "--prior-fisher-h1-smoke-report", + "--prior-fisher-h1-smoke-complete-marker", + } +) _SENSITIVE_MODULES: Final = frozenset( { "_virtualenv", @@ -119,9 +126,7 @@ def _canonical_json_bytes(value: object) -> bytes: def _pretty_json_bytes(value: object) -> bytes: - return (json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n").encode( - "utf-8" - ) + return (json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n").encode("utf-8") def _sha256_bytes(value: bytes) -> str: @@ -201,7 +206,8 @@ def _canonical_relative_path(value: object, *, context: str) -> str: raise SealedLaunchError(f"{context} is not a safe POSIX path") path = PurePosixPath(value) if ( - path.is_absolute() + value == "." + or path.is_absolute() or path.as_posix() != value or any(part in {"", ".", ".."} for part in path.parts) ): @@ -212,6 +218,15 @@ def _canonical_relative_path(value: object, *, context: str) -> str: return value +def _canonical_base_sys_path_entry(value: object, *, context: str) -> str: + # CPython's Windows runtime layout uses the exact ``.`` sentinel for the + # base-runtime root. Keep that exception local to base_sys_path: every + # other manifest path must remain a non-dot canonical relative path. + if value == ".": + return "." + return _canonical_relative_path(value, context=context) + + def _root_name(value: object, *, context: str) -> str: if not isinstance(value, str) or _ROOT_NAME_RE.fullmatch(value) is None: raise SealedLaunchError(f"{context} is not a canonical runtime-root name") @@ -293,6 +308,37 @@ def _stable_file_record(path: Path, *, relative: str, context: str) -> dict[str, return {"path": relative, "sha256": digest.hexdigest(), "size_bytes": size} +def _absolute_path_sha256(path: Path) -> str: + return _sha256_bytes(os.path.normcase(str(path.resolve(strict=True))).encode("utf-8")) + + +def _authenticated_git_executable(path: Path) -> tuple[Path, dict[str, object]]: + try: + resolved = Path(path).resolve(strict=True) + except OSError as exc: + raise SealedLaunchError("Git executable is unavailable") from exc + if resolved.name.casefold() == "git.exe" and resolved.parent.name.casefold() == "cmd": + try: + resolved = (resolved.parent.parent / "mingw64" / "bin" / "git.exe").resolve(strict=True) + except OSError as exc: + raise SealedLaunchError( + "Git-for-Windows cmd shim has no canonical mingw64 executable" + ) from exc + current = Path(resolved.anchor) + for part in resolved.parts[1:]: + current /= part + if _is_link_or_reparse(current): + raise SealedLaunchError("Git executable traverses a link or reparse point") + record = _stable_file_record(resolved, relative=resolved.name, context="Git executable") + if int(record["size_bytes"]) <= 0: + raise SealedLaunchError("Git executable is empty") + return resolved, { + "absolute_path_sha256": _absolute_path_sha256(resolved), + "sha256": record["sha256"], + "size_bytes": record["size_bytes"], + } + + def _tree_files(root: Path, *, context: str) -> tuple[dict[str, object], ...]: root = _absolute_directory(root, context=f"{context} root") stack: list[tuple[Path, tuple[str, ...]]] = [(root, ())] @@ -354,6 +400,7 @@ def _parse_runtime_manifest(data: bytes) -> dict[str, object]: "base_runtime_root", "base_sys_path", "distributions", + "git_executable", "interpreter", "launch_policy", "machine", @@ -378,6 +425,25 @@ def _parse_runtime_manifest(data: bytes) -> dict[str, object]: context="runtime manifest launch policy", ) + git_executable = root["git_executable"] + if not isinstance(git_executable, dict): + raise SealedLaunchError("runtime Git executable record must be an object") + _exact_fields( + git_executable, + {"absolute_path_sha256", "sha256", "size_bytes"}, + context="runtime Git executable", + ) + normalized_git_executable = { + "absolute_path_sha256": _sha256( + git_executable["absolute_path_sha256"], + context="runtime Git executable path SHA-256", + ), + "sha256": _sha256(git_executable["sha256"], context="runtime Git executable SHA-256"), + "size_bytes": _positive_int( + git_executable["size_bytes"], context="runtime Git executable size" + ), + } + python_record = root["python"] if not isinstance(python_record, dict): raise SealedLaunchError("runtime Python record must be an object") @@ -419,7 +485,7 @@ def _parse_runtime_manifest(data: bytes) -> dict[str, object]: if not isinstance(raw_base_sys_path, list) or not raw_base_sys_path: raise SealedLaunchError("base sys.path must be a non-empty list") base_sys_path = [ - _canonical_relative_path(item, context="base sys.path entry") + _canonical_base_sys_path_entry(item, context="base sys.path entry") for item in raw_base_sys_path ] if len({item.casefold() for item in base_sys_path}) != len(base_sys_path): @@ -442,11 +508,7 @@ def _parse_runtime_manifest(data: bytes) -> dict[str, object]: } ) names = [item["name"] for item in package_roots] - if ( - names != sorted(names) - or len(set(names)) != len(names) - or BASE_RUNTIME_ROOT_NAME in names - ): + if names != sorted(names) or len(set(names)) != len(names) or BASE_RUNTIME_ROOT_NAME in names: raise SealedLaunchError("package roots are not unique and sorted") raw_trees = root["runtime_trees"] @@ -475,9 +537,7 @@ def _parse_runtime_manifest(data: bytes) -> dict[str, object]: {"path", "sha256", "size_bytes"}, context="runtime file record", ) - relative = _canonical_relative_path( - raw_file["path"], context="runtime file path" - ) + relative = _canonical_relative_path(raw_file["path"], context="runtime file path") if _forbidden_runtime_path(relative): raise SealedLaunchError("runtime manifest contains a forbidden file") files.append( @@ -509,9 +569,7 @@ def _parse_runtime_manifest(data: bytes) -> dict[str, object]: "relative_path": interpreter_path, "root": interpreter["root"], "sha256": _sha256(interpreter["sha256"], context="runtime interpreter SHA-256"), - "size_bytes": _positive_int( - interpreter["size_bytes"], context="runtime interpreter size" - ), + "size_bytes": _positive_int(interpreter["size_bytes"], context="runtime interpreter size"), } if normalized_interpreter["root"] != BASE_RUNTIME_ROOT_NAME: raise SealedLaunchError("runtime interpreter is not in the base tree") @@ -524,8 +582,10 @@ def _parse_runtime_manifest(data: bytes) -> dict[str, object]: if base_files.get(interpreter_path) != expected_interpreter_file: raise SealedLaunchError("runtime interpreter differs from its base-tree record") for entry in base_sys_path: - present = entry in base_files or any( - str(path).startswith(f"{entry}/") for path in base_files + present = ( + entry == "." + or entry in base_files + or any(str(path).startswith(f"{entry}/") for path in base_files) ) optional_zip = re.fullmatch(r"python[0-9]+\.zip", entry) is not None if not present and not optional_zip: @@ -562,8 +622,7 @@ def _parse_runtime_manifest(data: bytes) -> dict[str, object]: if not isinstance(raw_files, list) or not raw_files: raise SealedLaunchError("runtime distribution has no RECORD files") files = [ - _canonical_relative_path(item, context="distribution RECORD path") - for item in raw_files + _canonical_relative_path(item, context="distribution RECORD path") for item in raw_files ] if files != sorted(files) or len({item.casefold() for item in files}) != len(files): raise SealedLaunchError("distribution RECORD paths are not unique and sorted") @@ -588,6 +647,7 @@ def _parse_runtime_manifest(data: bytes) -> dict[str, object]: "base_sys_path": base_sys_path, "distributions": distributions, "file_sha256": _sha256_bytes(data), + "git_executable": normalized_git_executable, "interpreter": normalized_interpreter, "machine": machine, "package_roots": package_roots, @@ -717,8 +777,9 @@ def _verify_runtime( *, base_runtime_root: Path, package_roots: Mapping[str, Path], + git_executable_path: Path, require_current_process: bool, -) -> tuple[Path, dict[str, Path], dict[str, str], Path]: +) -> tuple[Path, dict[str, Path], dict[str, str], Path, Path]: base, packages, import_paths = _runtime_roots( base_runtime_root, package_roots, @@ -732,6 +793,9 @@ def _verify_runtime( raise SealedLaunchError(f"runtime tree {name} differs from its frozen identity") if list(_distribution_inventory(packages, import_paths)) != manifest["distributions"]: raise SealedLaunchError("staged distribution identity differs from the runtime manifest") + git_executable, git_record = _authenticated_git_executable(git_executable_path) + if git_record != manifest["git_executable"]: + raise SealedLaunchError("Git executable differs from the runtime manifest") interpreter = _safe_join( base, str(manifest["interpreter"]["relative_path"]), # type: ignore[index] @@ -766,15 +830,16 @@ def _verify_runtime( or Path(sys.executable).resolve(strict=True) != interpreter ): raise SealedLaunchError("point-used Python or machine identity drifted") - return base, packages, import_paths, interpreter + return base, packages, import_paths, interpreter, git_executable def _extract_runner_options(arguments: Sequence[str]) -> dict[str, str]: result: dict[str, str] = {} + value_options = _REQUIRED_RUNNER_OPTIONS | _SMOKE_PREREQUISITE_OPTIONS index = 0 while index < len(arguments): option = arguments[index] - if option in _REQUIRED_RUNNER_OPTIONS: + if option in value_options: if option in result or index + 1 >= len(arguments): raise SealedLaunchError(f"runner option is duplicated or incomplete: {option}") raw_value = arguments[index + 1] @@ -787,9 +852,53 @@ def _extract_runner_options(arguments: Sequence[str]) -> dict[str, str]: missing = sorted(_REQUIRED_RUNNER_OPTIONS - set(result)) if missing: raise SealedLaunchError(f"runner arguments omit required sealed inputs: {missing}") + smoke_flag_count = sum(option == "--fisher-h1-smoke" for option in arguments) + if smoke_flag_count > 1: + raise SealedLaunchError("runner Fisher H=1 smoke flag is duplicated") + supplied_prerequisites = _SMOKE_PREREQUISITE_OPTIONS & set(result) + if smoke_flag_count == 1 and supplied_prerequisites: + raise SealedLaunchError("smoke mode forbids prior Fisher H=1 smoke prerequisites") + if smoke_flag_count == 0 and supplied_prerequisites != _SMOKE_PREREQUISITE_OPTIONS: + raise SealedLaunchError( + "full calibration requires both prior Fisher H=1 smoke prerequisite paths" + ) return result +def _verify_fisher_smoke_prerequisite_files(runner_options: Mapping[str, str]) -> None: + if "--prior-fisher-h1-smoke-report" not in runner_options: + return + try: + marker = Path(runner_options["--prior-fisher-h1-smoke-complete-marker"]).read_bytes() + report_bytes = Path(runner_options["--prior-fisher-h1-smoke-report"]).read_bytes() + except OSError as exc: + raise SealedLaunchError("prior Fisher H=1 smoke prerequisite is unavailable") from exc + if marker != FISHER_SMOKE_COMPLETE_BYTES: + raise SealedLaunchError("prior Fisher H=1 smoke completion marker drifted") + root = _strict_json(report_bytes, context="prior Fisher H=1 smoke report") + _exact_fields( + root, + {"artifact_kind", "canonical_evidence_sha256", "evidence", "schema_version"}, + context="prior Fisher H=1 smoke report", + ) + if _canonical_json_bytes(root) != report_bytes: + raise SealedLaunchError("prior Fisher H=1 smoke report is not canonical JSON") + evidence = root["evidence"] + if ( + root["artifact_kind"] != RUN_REPORT_KIND + or type(root["schema_version"]) is not int + or root["schema_version"] != RUN_REPORT_SCHEMA + or not isinstance(evidence, dict) + or evidence.get("status") != "fisher_h1_smoke_passed" + or _sha256( + root["canonical_evidence_sha256"], + context="prior Fisher H=1 smoke evidence SHA-256", + ) + != _sha256_bytes(_canonical_json_bytes(evidence)) + ): + raise SealedLaunchError("prior Fisher H=1 smoke report authentication failed") + + def _parse_identity(data: bytes) -> dict[str, str]: root = _strict_json(data, context="frozen identity") _exact_fields(root, {"canonical_evidence_sha256", "evidence"}, context="frozen identity") @@ -807,9 +916,7 @@ def _parse_identity(data: bytes) -> dict[str, str]: or evidence.get("promotion_required") is not False ): raise SealedLaunchError("frozen identity state or schema drifted") - claimed = _sha256( - root["canonical_evidence_sha256"], context="identity evidence SHA-256" - ) + claimed = _sha256(root["canonical_evidence_sha256"], context="identity evidence SHA-256") if claimed != _sha256_bytes(_canonical_json_bytes(evidence)): raise SealedLaunchError("frozen identity evidence hash drifted") bindings = evidence.get("execution_bindings") @@ -828,6 +935,7 @@ def _parse_source_manifest(data: bytes) -> dict[str, object]: root, { "canonical_manifest_sha256", + "git_executable", "object_format", "paths", "profile", @@ -839,14 +947,22 @@ def _parse_source_manifest(data: bytes) -> dict[str, object]: ) if _pretty_json_bytes(root) != data: raise SealedLaunchError("source manifest is not canonical JSON") - if root["schema"] != "recurquant.experiment013.source-manifest.v1": + if root["schema"] != "recurquant.experiment013.source-manifest.v2": raise SealedLaunchError("source manifest schema drifted") - if root["profile"] != "experiment-013-static-q468-frozen-source-v1": + if root["profile"] != "experiment-013-static-q468-frozen-source-v2": raise SealedLaunchError("source manifest profile drifted") if root["object_format"] != "sha1" or not isinstance(root["source_commit"], str): raise SealedLaunchError("source manifest Git identity drifted") if _SHA1_RE.fullmatch(str(root["source_commit"])) is None: raise SealedLaunchError("source manifest commit is invalid") + raw_git = root["git_executable"] + if not isinstance(raw_git, dict): + raise SealedLaunchError("source manifest Git executable record is missing") + _exact_fields(raw_git, {"sha256", "size_bytes"}, context="source Git executable") + git_executable = { + "sha256": _sha256(raw_git["sha256"], context="source Git executable SHA-256"), + "size_bytes": _positive_int(raw_git["size_bytes"], context="source Git executable size"), + } payload = dict(root) claimed = _sha256(payload.pop("canonical_manifest_sha256"), context="source self-hash") if claimed != _sha256_bytes(_pretty_json_bytes(payload)): @@ -865,13 +981,13 @@ def _parse_source_manifest(data: bytes) -> dict[str, object]: ) relative = _canonical_relative_path(raw_entry["path"], context="source path") for oid_field in ("git_blob_oid", "index_blob_oid", "worktree_blob_oid"): - if not isinstance(raw_entry[oid_field], str) or _SHA1_RE.fullmatch( - raw_entry[oid_field] - ) is None: + if ( + not isinstance(raw_entry[oid_field], str) + or _SHA1_RE.fullmatch(raw_entry[oid_field]) is None + ): raise SealedLaunchError("source Git object identity is invalid") git_object_ids = { - raw_entry[name] - for name in ("git_blob_oid", "index_blob_oid", "worktree_blob_oid") + raw_entry[name] for name in ("git_blob_oid", "index_blob_oid", "worktree_blob_oid") } if len(git_object_ids) != 1: raise SealedLaunchError("source Git object identities disagree") @@ -888,7 +1004,11 @@ def _parse_source_manifest(data: bytes) -> dict[str, object]: raise SealedLaunchError("source path inventory is not unique and sorted") if RUNNER_SOURCE_PATH not in rendered: raise SealedLaunchError("source manifest omits the calibration runner") - return {"file_sha256": _sha256_bytes(data), "paths": paths} + return { + "file_sha256": _sha256_bytes(data), + "git_executable": git_executable, + "paths": paths, + } def _verify_source(source_manifest: Mapping[str, object], repository_root: Path) -> Path: @@ -939,6 +1059,7 @@ def _verify_bound_artifacts( source_manifest, Path(runner_options["--repository-root"]), ) + _verify_fisher_smoke_prerequisite_files(runner_options) return bindings, source_manifest, runner_path @@ -952,13 +1073,69 @@ def _verify_empty_pycache(path: Path) -> Path: return root -def _sealed_environment() -> dict[str, str]: - return { - key: value - for key, value in os.environ.items() - if not key.upper().startswith("PYTHON") - and key.upper() not in {"VIRTUAL_ENV", "VIRTUAL_ENV_PROMPT"} +def _verify_empty_scratch(path: Path) -> Path: + root = _absolute_directory(path, context="sealed scratch directory") + try: + if any(os.scandir(root)): + raise SealedLaunchError("sealed scratch directory is not empty") + except OSError as exc: + raise SealedLaunchError("cannot enumerate sealed scratch directory") from exc + return root + + +def _assert_scratch_tree_has_no_reparse(path: Path) -> None: + stack = [path] + while stack: + directory = stack.pop() + for entry in os.scandir(directory): + status = entry.stat(follow_symlinks=False) + if entry.is_symlink() or bool( + getattr(status, "st_file_attributes", 0) & _WINDOWS_REPARSE_POINT + ): + raise SealedLaunchError("sealed scratch contains a link or reparse point") + if stat.S_ISDIR(status.st_mode): + stack.append(Path(entry.path)) + elif not stat.S_ISREG(status.st_mode): + raise SealedLaunchError("sealed scratch contains a non-regular path") + + +def _sealed_environment(*, scratch_directory: Path) -> dict[str, str]: + scratch = _absolute_directory(scratch_directory, context="sealed scratch directory") + inherited = {key.upper(): (key, value) for key, value in os.environ.items()} + environment = { + inherited[name][0]: inherited[name][1] + for name in ( + "SYSTEMROOT", + "WINDIR", + "COMSPEC", + "PROCESSOR_ARCHITECTURE", + "PROCESSOR_ARCHITEW6432", + ) + if name in inherited } + environment.update( + { + "LANG": "C", + "LC_ALL": "C", + "TEMP": str(scratch), + "TMP": str(scratch), + "TZ": "UTC", + } + ) + return environment + + +def _authenticated_stdin_loader(payload: bytes) -> str: + """Return a small ``-c`` loader bound to one exact stdin payload.""" + digest = hashlib.sha256(payload).hexdigest() + size = len(payload) + return ( + "import hashlib as _h,sys as _s\n" + f"_p=_s.stdin.buffer.read({size + 1})\n" + f"if len(_p)!={size} or _h.sha256(_p).hexdigest()!='{digest}':" + " raise RuntimeError('sealed bootstrap stdin authentication failed')\n" + "exec(compile(_p,'','exec',dont_inherit=True))" + ) def _sealed_argv( @@ -967,7 +1144,9 @@ def _sealed_argv( runtime_manifest: Path, base_runtime_root: Path, package_roots: Mapping[str, Path], + git_executable: Path, pycache_prefix: Path, + scratch_directory: Path, runner_arguments: Sequence[str], ) -> list[str]: serialized_roots = json.dumps( @@ -986,11 +1165,13 @@ def _sealed_argv( "-X", "utf8", "-c", - SEALED_BOOTSTRAP, + SEALED_STDIN_LOADER, str(runtime_manifest), str(base_runtime_root), serialized_roots, str(pycache_prefix), + str(git_executable), + str(scratch_directory), *runner_arguments, ] @@ -998,7 +1179,7 @@ def _sealed_argv( # This is intentionally standalone. The child starts with no repository or # package path and cannot import this host module safely. Keep its verification # semantics aligned with the host functions above and test both boundaries. -SEALED_BOOTSTRAP: Final = r''' +SEALED_BOOTSTRAP: Final = r""" import sys as _s _sensitive = { "site", "_virtualenv", "recurquant", @@ -1076,6 +1257,11 @@ def _sealed_argv( "parquet_materialization_manifest_file_sha256": "--expected-parquet-materialization-manifest-sha256", } +_smoke_options = { + "--prior-fisher-h1-smoke-report", + "--prior-fisher-h1-smoke-complete-marker", +} +_smoke_marker = b"recurquant-experiment013-fisher-h1-smoke-complete-v1\n" def _fail(message): raise RuntimeError(message) @@ -1138,7 +1324,7 @@ def _relative(value, context): if any(c in value for c in ("\\", "\0", "\n", "\r", ":")): _fail(context + " is not a safe POSIX path") path = _p.PurePosixPath(value) - if path.is_absolute() or path.as_posix() != value or any( + if value == "." or path.is_absolute() or path.as_posix() != value or any( part in {"", ".", ".."} for part in path.parts ): _fail(context + " is not a canonical relative path") @@ -1147,6 +1333,11 @@ def _relative(value, context): _fail(context + " is unsafe on Windows") return value +def _base_path(value, context): + if value == ".": + return "." + return _relative(value, context) + def _link(path): try: status = path.lstat() @@ -1208,6 +1399,28 @@ def _file(path, relative, context): _fail(context + " changed during authentication") return {"path": relative, "sha256": digest.hexdigest(), "size_bytes": size} +def _git(raw): + try: + path = _p.Path(raw).resolve(strict=True) + except OSError as error: + raise RuntimeError("Git executable is unavailable") from error + if path.name.casefold() == "git.exe" and path.parent.name.casefold() == "cmd": + try: + path = (path.parent.parent / "mingw64" / "bin" / "git.exe").resolve(strict=True) + except OSError as error: + raise RuntimeError("Git-for-Windows cmd shim has no implementation") from error + current = _p.Path(path.anchor) + for part in path.parts[1:]: + current /= part + if _link(current): + _fail("Git executable traverses a link or reparse point") + record = _file(path, path.name, "Git executable") + if record["size_bytes"] <= 0: + _fail("Git executable is empty") + return path, {"absolute_path_sha256": _h.sha256( + _o.path.normcase(str(path)).encode("utf-8") + ).hexdigest(), "sha256": record["sha256"], "size_bytes": record["size_bytes"]} + def _tree(root, context): root = _directory(root, context + " root") stack = [(root, ())] @@ -1249,11 +1462,12 @@ def _name(value): def _options(arguments): required = {"--frozen-identity", "--repository-root", *_binding_options.values(), *_expected_digest_options.values()} + value_options = required | _smoke_options result = {} index = 0 while index < len(arguments): item = arguments[index] - if item in required: + if item in value_options: if ( item in result or index + 1 >= len(arguments) @@ -1265,9 +1479,37 @@ def _options(arguments): else: index += 1 if set(result) != required: - _fail("runner arguments omit required sealed inputs") + if not required.issubset(result): + _fail("runner arguments omit required sealed inputs") + smoke_count = sum(item == "--fisher-h1-smoke" for item in arguments) + if smoke_count > 1: + _fail("runner Fisher H=1 smoke flag is duplicated") + supplied = _smoke_options.intersection(result) + if smoke_count == 1 and supplied: + _fail("smoke mode forbids prior Fisher H=1 smoke prerequisites") + if smoke_count == 0 and supplied != _smoke_options: + _fail("full calibration requires prior Fisher H=1 smoke prerequisites") return result +def _smoke(options): + if "--prior-fisher-h1-smoke-report" not in options: + return + if _p.Path(options["--prior-fisher-h1-smoke-complete-marker"]).read_bytes() != _smoke_marker: + _fail("prior Fisher H=1 smoke completion marker drifted") + data = _p.Path(options["--prior-fisher-h1-smoke-report"]).read_bytes() + root = _json(data, "prior Fisher H=1 smoke report") + _fields(root, {"artifact_kind", "canonical_evidence_sha256", "evidence", "schema_version"}, + "prior Fisher H=1 smoke report") + evidence = root["evidence"] + if (_canonical(root) != data + or root["artifact_kind"] != "recurquant_experiment013_calibration_run" + or type(root["schema_version"]) is not int or root["schema_version"] != 2 + or not isinstance(evidence, dict) + or evidence.get("status") != "fisher_h1_smoke_passed" + or _digest(root["canonical_evidence_sha256"], "smoke evidence hash") + != _h.sha256(_canonical(evidence)).hexdigest()): + _fail("prior Fisher H=1 smoke report authentication failed") + def _identity(data): root = _json(data, "frozen identity") _fields(root, {"canonical_evidence_sha256", "evidence"}, "frozen identity") @@ -1275,7 +1517,7 @@ def _identity(data): _fail("frozen identity is not canonical JSON") evidence = root["evidence"] if (not isinstance(evidence, dict) or type(evidence.get("schema_version")) is not int - or evidence.get("schema_version") != 4 + or evidence.get("schema_version") != 5 or evidence.get("status") != "frozen" or evidence.get("phase") != "calibration" or evidence.get("identity_only") is not True or evidence.get("promotion_required") is not False): @@ -1292,14 +1534,25 @@ def _identity(data): def _source(data): root = _json(data, "source manifest") - _fields(root, {"canonical_manifest_sha256", "object_format", "paths", "profile", - "repository_binding", "schema", "source_commit"}, "source manifest") + _fields(root, {"canonical_manifest_sha256", "git_executable", "object_format", + "paths", "profile", "repository_binding", "schema", + "source_commit"}, "source manifest") if _pretty(root) != data: _fail("source manifest is not canonical JSON") payload = dict(root) claimed = _digest(payload.pop("canonical_manifest_sha256"), "source self-hash") if claimed != _h.sha256(_pretty(payload)).hexdigest(): _fail("source manifest self-hash drifted") + if (root["schema"] != "recurquant.experiment013.source-manifest.v2" + or root["profile"] != "experiment-013-static-q468-frozen-source-v2" + or root["object_format"] != "sha1"): + _fail("source manifest profile drifted") + git = root["git_executable"] + if not isinstance(git, dict): + _fail("source Git executable record is invalid") + _fields(git, {"sha256", "size_bytes"}, "source Git executable") + git = {"sha256": _digest(git["sha256"], "source Git executable hash"), + "size_bytes": _positive(git["size_bytes"], "source Git executable size")} paths = [] for entry in root["paths"]: _fields(entry, {"git_blob_oid", "index_blob_oid", "mode", "path", "raw_sha256", @@ -1309,7 +1562,8 @@ def _source(data): rendered = [item["path"] for item in paths] if rendered != sorted(rendered) or "scripts/run_static_q468_calibration.py" not in rendered: _fail("source path inventory drifted") - return {"file_sha256": _h.sha256(data).hexdigest(), "paths": paths} + return {"file_sha256": _h.sha256(data).hexdigest(), "git_executable": git, + "paths": paths} def _verify_source(manifest, root): root = _directory(root, "repository root") @@ -1352,16 +1606,27 @@ def _distributions(packages, imports): def _manifest(data): root = _json(data, "runtime manifest") _fields(root, {"artifact_kind", "base_runtime_root", "base_sys_path", "distributions", + "git_executable", "interpreter", "launch_policy", "machine", "package_roots", "python", "runtime_trees", "schema_version"}, "runtime manifest") if _canonical(root) != data or root["artifact_kind"] != ( "recurquant_experiment013_calibration_runtime_manifest" - ) or type(root["schema_version"]) is not int or root["schema_version"] != 3: + ) or type(root["schema_version"]) is not int or root["schema_version"] != 4: _fail("runtime manifest identity or policy drifted") _typed(root["launch_policy"], _policy, "runtime launch policy") if root["base_runtime_root"] != "base-runtime": _fail("base runtime name drifted") + git = root["git_executable"] + if not isinstance(git, dict): + _fail("runtime Git executable record is invalid") + _fields(git, {"absolute_path_sha256", "sha256", "size_bytes"}, + "runtime Git executable") + git = {"absolute_path_sha256": _digest( + git["absolute_path_sha256"], "runtime Git executable path hash"), + "sha256": _digest(git["sha256"], "runtime Git executable hash"), + "size_bytes": _positive(git["size_bytes"], "runtime Git executable size")} + python = root["python"] if not isinstance(python, dict): _fail("runtime Python record is invalid") @@ -1406,7 +1671,7 @@ def _manifest(data): raw_base_paths = root["base_sys_path"] if not isinstance(raw_base_paths, list) or not raw_base_paths: _fail("base sys.path is invalid") - base_paths = [_relative(item, "base sys.path") for item in raw_base_paths] + base_paths = [_base_path(item, "base sys.path") for item in raw_base_paths] if len({item.casefold() for item in base_paths}) != len(base_paths): _fail("base sys.path entries collide") @@ -1461,7 +1726,8 @@ def _manifest(data): if base_files.get(normalized_interpreter["relative_path"]) != expected_interpreter: _fail("runtime interpreter differs from the base tree") for entry in base_paths: - present = entry in base_files or any(path.startswith(entry + "/") for path in base_files) + present = (entry == "." or entry in base_files + or any(path.startswith(entry + "/") for path in base_files)) if not present and _re.fullmatch(r"python[0-9]+\.zip", entry) is None: _fail("base sys.path entry is absent from the base tree") @@ -1504,11 +1770,12 @@ def _manifest(data): return {"base_sys_path": base_paths, "distributions": distributions, "file_sha256": _h.sha256(data).hexdigest(), + "git_executable": git, "interpreter": normalized_interpreter, "machine": machine, "package_roots": roots, "python": python, "runtime_trees": normalized_trees} -def _verify_runtime(manifest, base_raw, package_raw, packages_appended=False): +def _verify_runtime(manifest, base_raw, package_raw, git_raw, packages_appended=False): base = _directory(base_raw, "base runtime") declared = {item["name"]: item["import_path"] for item in manifest["package_roots"]} if set(package_raw) != set(declared): @@ -1520,6 +1787,9 @@ def _verify_runtime(manifest, base_raw, package_raw, packages_appended=False): _fail("complete runtime tree identity drifted") if _distributions(packages, declared) != manifest["distributions"]: _fail("distribution RECORD identity drifted") + git, git_record = _git(git_raw) + if git_record != manifest["git_executable"]: + _fail("Git executable identity drifted") interpreter = _join(base, manifest["interpreter"]["relative_path"], "interpreter") if _file( interpreter, @@ -1555,7 +1825,7 @@ def _verify_runtime(manifest, base_raw, package_raw, packages_appended=False): _o.path.normcase(_o.path.abspath(item)) for item in expected_path ]: _fail("authenticated sys.path differs from the frozen runtime") - return base, packages, declared, imports, interpreter + return base, packages, declared, imports, interpreter, git _runtime_path = _p.Path(_s.argv[1]) _base_raw = _p.Path(_s.argv[2]) @@ -1568,8 +1838,30 @@ def _verify_runtime(manifest, base_raw, package_raw, packages_appended=False): _s.pycache_prefix ).resolve(strict=True) != _pycache: _fail("sealed pycache prefix is not exact and empty") -_runner_args = list(_s.argv[5:]) +_git_raw = _p.Path(_s.argv[5]) +_scratch = _directory(_s.argv[6], "sealed scratch directory") +if any(_scratch.iterdir()): + _fail("sealed scratch directory is not initially empty") +_environment = {key.upper(): value for key, value in _o.environ.items()} +_required_environment = {"LANG", "LC_ALL", "TEMP", "TMP", "TZ"} +_os_environment = { + "SYSTEMROOT", "WINDIR", "COMSPEC", + "PROCESSOR_ARCHITECTURE", "PROCESSOR_ARCHITEW6432", +} +_allowed_environment = _required_environment | _os_environment +if (not _required_environment.issubset(_environment) + or not set(_environment).issubset(_allowed_environment) + or _environment["LANG"] != "C" or _environment["LC_ALL"] != "C" + or _environment["TZ"] != "UTC" + or _p.Path(_environment["TEMP"]).resolve(strict=True) != _scratch + or _p.Path(_environment["TMP"]).resolve(strict=True) != _scratch + or any(not value or "\0" in value or "\n" in value or "\r" in value + for key, value in _environment.items() + if key in _os_environment)): + _fail("sealed child environment differs from the minimal contract") +_runner_args = list(_s.argv[7:]) _runner_options = _options(_runner_args) +_smoke(_runner_options) _runtime_bytes = _runtime_path.read_bytes() _runtime = _manifest(_runtime_bytes) _bindings = _identity(_p.Path(_runner_options["--frozen-identity"]).read_bytes()) @@ -1586,10 +1878,15 @@ def _verify_runtime(manifest, base_raw, package_raw, packages_appended=False): _source_manifest = _source( _p.Path(_runner_options["--repository-source-manifest"]).read_bytes() ) +if _source_manifest["git_executable"] != { + "sha256": _runtime["git_executable"]["sha256"], + "size_bytes": _runtime["git_executable"]["size_bytes"], +}: + _fail("source and runtime manifests bind different Git bytes") _repository_root = _p.Path(_runner_options["--repository-root"]) _runner_path = _verify_source(_source_manifest, _repository_root) -_base, _packages, _import_rel, _imports, _interpreter = _verify_runtime( - _runtime, _base_raw, _package_raw +_base, _packages, _import_rel, _imports, _interpreter, _git_executable = _verify_runtime( + _runtime, _base_raw, _package_raw, _git_raw ) _s.path.extend(_imports[name] for name in sorted(_imports)) if _s.path != [str(_base / _p.PurePosixPath(item)) for item in _runtime["base_sys_path"]] + [ @@ -1625,19 +1922,27 @@ def _verify_runtime(manifest, base_raw, package_raw, packages_appended=False): package_roots=_packages, package_import_paths=_import_rel, interpreter_path=_interpreter, + git_executable_path=_git_executable, pycache_prefix=_pycache, ) finally: if any(_pycache.iterdir()): _fail("sealed pycache prefix changed during calibration") + if any(_scratch.iterdir()): + _fail("sealed scratch directory was not cleaned by the runner") _verify_source(_source_manifest, _repository_root) - _verify_runtime(_runtime, _base, _packages, packages_appended=True) + _verify_runtime( + _runtime, _base, _packages, _git_executable, packages_appended=True + ) finally: _s.modules.pop("_recurquant_experiment013_sealed_runner", None) if not isinstance(_result, int) or isinstance(_result, bool): _fail("sealed_main returned a non-integer status") raise SystemExit(_result) -'''.strip() +""".strip() + +SEALED_BOOTSTRAP_BYTES: Final = SEALED_BOOTSTRAP.encode("utf-8") +SEALED_STDIN_LOADER: Final = _authenticated_stdin_loader(SEALED_BOOTSTRAP_BYTES) def _split_host_and_runner_args(argv: Sequence[str]) -> tuple[list[str], list[str]]: @@ -1655,6 +1960,7 @@ def _split_host_and_runner_args(argv: Sequence[str]) -> tuple[list[str], list[st def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base-runtime-root", required=True, type=Path) + parser.add_argument("--git-executable", required=True, type=Path) parser.add_argument("--package-root", required=True, action="append", type=_parse_package_root) parser.add_argument("--runtime-manifest", required=True, type=Path) return parser @@ -1679,49 +1985,68 @@ def launch(argv: Sequence[str]) -> int: raise SealedLaunchError("runtime manifest is unavailable") from exc runtime_manifest = _parse_runtime_manifest(runtime_bytes) runner_options = _extract_runner_options(runner_arguments) - _verify_bound_artifacts( + _bindings, source_manifest, _runner_path = _verify_bound_artifacts( runner_options, runtime_manifest_path=args.runtime_manifest, ) - base, packages, _import_paths, interpreter = _verify_runtime( + if source_manifest["git_executable"] != { + "sha256": runtime_manifest["git_executable"]["sha256"], + "size_bytes": runtime_manifest["git_executable"]["size_bytes"], + }: + raise SealedLaunchError("source and runtime manifests bind different Git bytes") + base, packages, _import_paths, interpreter, git_executable = _verify_runtime( runtime_manifest, base_runtime_root=args.base_runtime_root, package_roots=package_roots, + git_executable_path=args.git_executable, require_current_process=False, ) pycache_parent = Path(tempfile.mkdtemp(prefix="recurquant-exp013-sealed-pycache-")) pycache = _verify_empty_pycache(pycache_parent) + scratch_parent = Path(tempfile.mkdtemp(prefix="recurquant-exp013-sealed-scratch-")) + scratch = _verify_empty_scratch(scratch_parent) try: command = _sealed_argv( interpreter=interpreter, runtime_manifest=args.runtime_manifest.resolve(strict=True), base_runtime_root=base, package_roots=packages, + git_executable=git_executable, pycache_prefix=pycache, + scratch_directory=scratch, runner_arguments=runner_arguments, ) completed = subprocess.run( command, check=False, cwd=base, - env=_sealed_environment(), + env=_sealed_environment(scratch_directory=scratch), + input=SEALED_BOOTSTRAP_BYTES, ) _verify_empty_pycache(pycache) - _verify_bound_artifacts( + _verify_empty_scratch(scratch) + _bindings, repeated_source, _runner_path = _verify_bound_artifacts( runner_options, runtime_manifest_path=args.runtime_manifest, ) + if repeated_source["git_executable"] != source_manifest["git_executable"]: + raise SealedLaunchError("source Git executable binding changed during execution") _verify_runtime( runtime_manifest, base_runtime_root=base, package_roots=packages, + git_executable_path=git_executable, require_current_process=False, ) return int(completed.returncode) finally: _verify_empty_pycache(pycache) shutil.rmtree(pycache, ignore_errors=False) + _assert_scratch_tree_has_no_reparse(scratch) + shutil.rmtree(scratch, ignore_errors=False) + if os.path.lexists(scratch): + raise SealedLaunchError("sealed scratch directory survived cleanup") def main(argv: Sequence[str] | None = None) -> int: diff --git a/scripts/launch_static_q468_stage_a.py b/scripts/launch_static_q468_stage_a.py new file mode 100644 index 0000000..54bbbd2 --- /dev/null +++ b/scripts/launch_static_q468_stage_a.py @@ -0,0 +1,822 @@ +#!/usr/bin/env python3 +"""Launch Experiment 013 Stage A in the authenticated sealed runtime. + +The host process is metadata-only. It verifies the promoted v5 Stage-A +identity bindings, exact H0 source files, and complete sealed runtime before +starting the staged interpreter. The isolated child repeats those checks and +loads exactly ``screen_static_q468_stage_a.py`` from authenticated bytes. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from collections.abc import Mapping, Sequence +from pathlib import Path, PurePosixPath +from types import ModuleType +from typing import Final + +RUNNER_SOURCE_PATH: Final = "scripts/screen_static_q468_stage_a.py" +LAUNCHER_SOURCE_PATH: Final = "scripts/launch_static_q468_stage_a.py" +CALIBRATION_LAUNCHER_SOURCE_PATH: Final = "scripts/launch_static_q468_calibration.py" +RUNNER_MODULE_NAME: Final = "_recurquant_experiment013_sealed_stage_a_runner" +CALIBRATION_LAUNCHER_MODULE_NAME: Final = ( + "_recurquant_experiment013_calibration_launcher_for_stage_a" +) +IDENTITY_SCHEMA: Final = 5 +BASE_RUNTIME_ROOT_NAME: Final = "base-runtime" +_SHA256_RE: Final = re.compile(r"[0-9a-f]{64}") +_BOUND_ARTIFACT_OPTIONS: Final = { + "calibration_runtime_manifest_file_sha256": "--runtime-manifest", + "model_file_manifest_file_sha256": "--model-file-manifest", + "parquet_materialization_manifest_file_sha256": ("--parquet-materialization-manifest"), + "repository_source_manifest_file_sha256": "--repository-source-manifest", +} +_EXPECTED_DIGEST_OPTIONS: Final = { + "calibration_runtime_manifest_file_sha256": "--expected-runtime-manifest-sha256", + "model_file_manifest_file_sha256": "--expected-model-file-manifest-sha256", + "parquet_materialization_manifest_file_sha256": ( + "--expected-parquet-materialization-manifest-sha256" + ), +} +_REQUIRED_OPTIONS: Final = frozenset( + { + "--frozen-identity", + "--stage-a-calibration-binding", + "--repository-root", + "--source-commit", + "--identity-commit", + "--model-root", + "--cache-root", + "--input-bundle-root", + "--ruler-root", + "--output-dir", + *_BOUND_ARTIFACT_OPTIONS.values(), + *_EXPECTED_DIGEST_OPTIONS.values(), + } +) + + +class SealedStageALaunchError(RuntimeError): + """Raised before the protected Stage-A runner can be entered.""" + + +def _canonical_json_bytes(value: object) -> bytes: + try: + return ( + json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ) + + "\n" + ).encode("utf-8") + except (TypeError, ValueError) as error: + raise SealedStageALaunchError("value is not canonical JSON data") from error + + +def _pretty_json_bytes(value: object) -> bytes: + try: + return (json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n").encode("utf-8") + except (TypeError, ValueError) as error: + raise SealedStageALaunchError("value is not pretty JSON data") from error + + +def _sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _sha256(value: object, *, context: str) -> str: + if not isinstance(value, str) or _SHA256_RE.fullmatch(value) is None: + raise SealedStageALaunchError(f"{context} must be a lowercase SHA-256 digest") + return value + + +def _strict_json(data: bytes, *, context: str) -> dict[str, object]: + def unique(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise SealedStageALaunchError(f"{context} contains a duplicate JSON key") + result[key] = value + return result + + def reject_constant(value: str) -> None: + raise SealedStageALaunchError(f"{context} contains a non-finite JSON constant: {value}") + + try: + value = json.loads( + data.decode("utf-8"), + object_pairs_hook=unique, + parse_constant=reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise SealedStageALaunchError(f"{context} is not strict UTF-8 JSON") from error + if not isinstance(value, dict): + raise SealedStageALaunchError(f"{context} must be a JSON object") + return value + + +def _exact_fields(value: Mapping[str, object], expected: set[str], *, context: str) -> None: + if set(value) != expected: + raise SealedStageALaunchError(f"{context} fields differ from the frozen schema") + + +def _relative_path(value: object, *, context: str) -> str: + if not isinstance(value, str) or not value or value != value.strip(): + raise SealedStageALaunchError(f"{context} is not a canonical path") + if any(character in value for character in ("\\", "\0", "\n", "\r", ":")): + raise SealedStageALaunchError(f"{context} is unsafe") + path = PurePosixPath(value) + if ( + path.is_absolute() + or path.as_posix() != value + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise SealedStageALaunchError(f"{context} is not a canonical relative path") + return value + + +def _extract_options(arguments: Sequence[str]) -> dict[str, str]: + result: dict[str, str] = {} + index = 0 + while index < len(arguments): + item = arguments[index] + if item in _REQUIRED_OPTIONS: + if ( + item in result + or index + 1 >= len(arguments) + or arguments[index + 1].startswith("--") + ): + raise SealedStageALaunchError(f"runner option is duplicated or incomplete: {item}") + result[item] = arguments[index + 1] + index += 2 + elif item.startswith("--"): + raise SealedStageALaunchError(f"runner option is not in the frozen CLI: {item}") + else: + index += 1 + if set(result) != _REQUIRED_OPTIONS: + missing = sorted(_REQUIRED_OPTIONS - set(result)) + raise SealedStageALaunchError(f"runner arguments omit required inputs: {missing}") + if not arguments or arguments[0] not in { + "prepare-inputs", + "preflight", + "execute", + "recover", + }: + raise SealedStageALaunchError( + "runner mode must be prepare-inputs, preflight, execute, or recover" + ) + return result + + +def _parse_identity(data: bytes) -> dict[str, str]: + root = _strict_json(data, context="frozen Stage-A identity") + _exact_fields(root, {"canonical_evidence_sha256", "evidence"}, context="identity") + if _canonical_json_bytes(root) != data: + raise SealedStageALaunchError("frozen Stage-A identity is not canonical JSON") + evidence = root.get("evidence") + if not isinstance(evidence, dict): + raise SealedStageALaunchError("frozen Stage-A identity evidence is missing") + if ( + type(evidence.get("schema_version")) is not int + or evidence.get("schema_version") != IDENTITY_SCHEMA + or evidence.get("identity_schema") != "recurquant.experiment013.identity-frozen.v5" + or evidence.get("status") != "frozen" + or evidence.get("phase") != "stage_a" + or evidence.get("identity_only") is not True + or evidence.get("promotion_required") is not False + ): + raise SealedStageALaunchError("identity is not a promoted Stage-A resolver-v5 artifact") + promotion = evidence.get("promotion") + if not isinstance(promotion, dict) or promotion.get("explicit") is not True: + raise SealedStageALaunchError("Stage-A identity lacks explicit promotion") + recorded = _sha256( + root.get("canonical_evidence_sha256"), context="identity canonical evidence SHA-256" + ) + if recorded != _sha256_bytes(_canonical_json_bytes(evidence)): + raise SealedStageALaunchError("identity canonical evidence SHA-256 drifted") + bindings = evidence.get("execution_bindings") + if not isinstance(bindings, dict): + raise SealedStageALaunchError("identity execution bindings are missing") + _exact_fields(bindings, set(_BOUND_ARTIFACT_OPTIONS), context="identity bindings") + return { + name: _sha256(bindings[name], context=f"identity binding {name}") + for name in sorted(bindings) + } + + +def _parse_source(data: bytes) -> dict[str, object]: + root = _strict_json(data, context="repository source manifest") + _exact_fields( + root, + { + "canonical_manifest_sha256", + "git_executable", + "object_format", + "paths", + "profile", + "repository_binding", + "schema", + "source_commit", + }, + context="repository source manifest", + ) + if _pretty_json_bytes(root) != data: + raise SealedStageALaunchError("repository source manifest is not canonical JSON") + payload = dict(root) + claimed = _sha256(payload.pop("canonical_manifest_sha256"), context="source self-hash") + if claimed != _sha256_bytes(_pretty_json_bytes(payload)): + raise SealedStageALaunchError("repository source manifest self-hash drifted") + if ( + root["schema"] != "recurquant.experiment013.source-manifest.v2" + or root["profile"] != "experiment-013-static-q468-frozen-source-v2" + or root["object_format"] != "sha1" + ): + raise SealedStageALaunchError("repository source manifest profile drifted") + raw_git = root["git_executable"] + if not isinstance(raw_git, dict): + raise SealedStageALaunchError("repository source Git executable record is missing") + _exact_fields(raw_git, {"sha256", "size_bytes"}, context="source Git executable") + if ( + isinstance(raw_git["size_bytes"], bool) + or not isinstance(raw_git["size_bytes"], int) + or raw_git["size_bytes"] <= 0 + ): + raise SealedStageALaunchError("repository source Git executable size is invalid") + git_executable = { + "sha256": _sha256(raw_git["sha256"], context="source Git executable SHA-256"), + "size_bytes": raw_git["size_bytes"], + } + raw_paths = root.get("paths") + if not isinstance(raw_paths, list) or not raw_paths: + raise SealedStageALaunchError("repository source manifest has no paths") + paths: list[dict[str, str]] = [] + for index, entry in enumerate(raw_paths): + if not isinstance(entry, dict): + raise SealedStageALaunchError(f"source paths[{index}] must be an object") + _exact_fields( + entry, + { + "git_blob_oid", + "index_blob_oid", + "mode", + "path", + "raw_sha256", + "worktree_blob_oid", + }, + context=f"source paths[{index}]", + ) + path = _relative_path(entry["path"], context=f"source paths[{index}].path") + paths.append( + { + "path": path, + "raw_sha256": _sha256( + entry["raw_sha256"], context=f"source paths[{index}].raw_sha256" + ), + } + ) + names = [entry["path"] for entry in paths] + if names != sorted(names) or len({name.casefold() for name in names}) != len(names): + raise SealedStageALaunchError("source path inventory is not unique and sorted") + required = {RUNNER_SOURCE_PATH, LAUNCHER_SOURCE_PATH, CALIBRATION_LAUNCHER_SOURCE_PATH} + if not set(names) >= required: + raise SealedStageALaunchError("source manifest omits a sealed Stage-A launch path") + return {"document": root, "git_executable": git_executable, "paths": paths} + + +def _verify_source(source: Mapping[str, object], repository_root: Path) -> Path: + root = repository_root.resolve(strict=True) + if not root.is_dir() or root.is_symlink(): + raise SealedStageALaunchError("repository root is not a stable directory") + runner: Path | None = None + for entry in source["paths"]: # type: ignore[union-attr] + relative = str(entry["path"]) + path = (root / PurePosixPath(relative)).resolve(strict=True) + try: + path.relative_to(root) + except ValueError as error: + raise SealedStageALaunchError("source path escapes repository root") from error + if path.is_symlink() or not path.is_file(): + raise SealedStageALaunchError(f"source path is not a regular file: {relative}") + before = path.stat() + data = path.read_bytes() + after = path.stat() + if before.st_size != after.st_size or before.st_mtime_ns != after.st_mtime_ns: + raise SealedStageALaunchError(f"source path changed while read: {relative}") + if _sha256_bytes(data) != entry["raw_sha256"]: + raise SealedStageALaunchError(f"source bytes drifted: {relative}") + if relative == RUNNER_SOURCE_PATH: + runner = path + if runner is None: + raise SealedStageALaunchError("source manifest omits Stage-A runner") + return runner + + +def _load_calibration_launcher( + repository_root: Path, + source: Mapping[str, object], +) -> ModuleType: + entries = {str(item["path"]): item for item in source["paths"]} # type: ignore[index] + path = (repository_root / PurePosixPath(CALIBRATION_LAUNCHER_SOURCE_PATH)).resolve(strict=True) + if _sha256_bytes(path.read_bytes()) != entries[CALIBRATION_LAUNCHER_SOURCE_PATH]["raw_sha256"]: + raise SealedStageALaunchError("calibration launcher source bytes drifted") + if CALIBRATION_LAUNCHER_MODULE_NAME in sys.modules: + raise SealedStageALaunchError("calibration launcher module name is already occupied") + spec = importlib.util.spec_from_file_location(CALIBRATION_LAUNCHER_MODULE_NAME, path) + if spec is None or spec.loader is None: + raise SealedStageALaunchError("cannot load authenticated calibration launcher") + module = importlib.util.module_from_spec(spec) + sys.modules[CALIBRATION_LAUNCHER_MODULE_NAME] = module + try: + spec.loader.exec_module(module) + except BaseException: + sys.modules.pop(CALIBRATION_LAUNCHER_MODULE_NAME, None) + raise + return module + + +def _stage_a_bootstrap(calibration_launcher: ModuleType) -> str: + source = getattr(calibration_launcher, "SEALED_BOOTSTRAP", None) + if not isinstance(source, str): + raise SealedStageALaunchError("authenticated calibration launcher has no bootstrap") + stage_a_options = """_stage_a_options = { + "--cache-root", + "--expected-model-file-manifest-sha256", + "--expected-parquet-materialization-manifest-sha256", + "--expected-runtime-manifest-sha256", + "--frozen-identity", + "--identity-commit", + "--input-bundle-root", + "--model-file-manifest", + "--model-root", + "--output-dir", + "--parquet-materialization-manifest", + "--repository-root", + "--repository-source-manifest", + "--ruler-root", + "--runtime-manifest", + "--source-commit", + "--stage-a-calibration-binding", +} + +def _options(arguments): + if not arguments or arguments[0] not in { + "prepare-inputs", "preflight", "execute", "recover"}: + _fail("Stage-A runner mode drifted") + result = {} + index = 1 + while index < len(arguments): + item = arguments[index] + if item not in _stage_a_options: + _fail("runner option is not in the frozen Stage-A CLI: " + item) + if (item in result or index + 1 >= len(arguments) + or arguments[index + 1].startswith("--")): + _fail("runner option is duplicated or incomplete: " + item) + result[item] = arguments[index + 1] + index += 2 + if set(result) != _stage_a_options: + _fail("runner arguments omit required Stage-A inputs") + return result +""" + stage_a_identity = """def _identity(data): + root = _json(data, "frozen Stage-A identity") + _fields(root, {"canonical_evidence_sha256", "evidence"}, "frozen Stage-A identity") + if _canonical(root) != data: + _fail("frozen Stage-A identity is not canonical JSON") + evidence = root["evidence"] + promotion = evidence.get("promotion") if isinstance(evidence, dict) else None + if (not isinstance(evidence, dict) or type(evidence.get("schema_version")) is not int + or evidence.get("schema_version") != 5 + or evidence.get("identity_schema") != "recurquant.experiment013.identity-frozen.v5" + or evidence.get("status") != "frozen" or evidence.get("phase") != "stage_a" + or evidence.get("identity_only") is not True + or evidence.get("promotion_required") is not False + or not isinstance(promotion, dict) or promotion.get("explicit") is not True): + _fail("frozen Stage-A identity state, schema, or promotion drifted") + if _digest(root["canonical_evidence_sha256"], "identity evidence hash") != _h.sha256( + _canonical(evidence) + ).hexdigest(): + _fail("frozen Stage-A identity evidence hash drifted") + bindings = evidence.get("execution_bindings") + if not isinstance(bindings, dict): + _fail("frozen Stage-A identity bindings are missing") + _fields(bindings, set(_binding_options), "identity bindings") + return {key: _digest(value, "identity binding") for key, value in bindings.items()} +""" + original_environment = ( + "_environment = {key.upper(): value for key, value in _o.environ.items()}\n" + """_required_environment = {"LANG", "LC_ALL", "TEMP", "TMP", "TZ"} +_os_environment = { + "SYSTEMROOT", "WINDIR", "COMSPEC", + "PROCESSOR_ARCHITECTURE", "PROCESSOR_ARCHITEW6432", +} +_allowed_environment = _required_environment | _os_environment +if (not _required_environment.issubset(_environment) + or not set(_environment).issubset(_allowed_environment) + or _environment["LANG"] != "C" or _environment["LC_ALL"] != "C" + or _environment["TZ"] != "UTC" + or _p.Path(_environment["TEMP"]).resolve(strict=True) != _scratch + or _p.Path(_environment["TMP"]).resolve(strict=True) != _scratch + or any(not value or "\\0" in value or "\\n" in value or "\\r" in value + for key, value in _environment.items() + if key in _os_environment)): + _fail("sealed child environment differs from the minimal contract")""" + ) + stage_a_environment = ( + "_environment = {key.upper(): value for key, value in _o.environ.items()}\n" + """_stage_a_mode = _s.argv[7] if len(_s.argv) > 7 else None +_stage_a_offline_values = { + "HF_DATASETS_OFFLINE": "1", + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", +} +_stage_a_offline = _stage_a_mode != "prepare-inputs" +_required_environment = {"LANG", "LC_ALL", "TEMP", "TMP", "TZ"} | ( + set(_stage_a_offline_values) if _stage_a_offline else set() +) +_os_environment = { + "SYSTEMROOT", "WINDIR", "COMSPEC", + "PROCESSOR_ARCHITECTURE", "PROCESSOR_ARCHITEW6432", +} +_allowed_environment = _required_environment | _os_environment +if (not _required_environment.issubset(_environment) + or not set(_environment).issubset(_allowed_environment) + or _environment["LANG"] != "C" or _environment["LC_ALL"] != "C" + or _environment["TZ"] != "UTC" + or _p.Path(_environment["TEMP"]).resolve(strict=True) != _scratch + or _p.Path(_environment["TMP"]).resolve(strict=True) != _scratch + or (any(_environment.get(name) != value + for name, value in _stage_a_offline_values.items()) + if _stage_a_offline + else any(name in _environment for name in _stage_a_offline_values)) + or any(not value or "\\0" in value or "\\n" in value or "\\r" in value + for key, value in _environment.items() + if key in _os_environment)): + _fail("sealed child environment differs from the mode-specific minimal contract") +if _stage_a_offline: + _forbidden_network_events = { + "socket.connect", "socket.connect_ex", "socket.getaddrinfo", + "socket.gethostbyaddr", "socket.gethostbyname", "socket.gethostbyname_ex", + "socket.getnameinfo", "socket.sendto", + } + def _reject_network(event, _arguments): + if event in _forbidden_network_events: + raise RuntimeError("sealed offline Stage-A child forbids network access: " + event) + _s.addaudithook(_reject_network)""" + ) + + def replace_exact(value: str, old: str, new: str, *, count: int) -> str: + if value.count(old) != count: + raise SealedStageALaunchError( + "calibration bootstrap transformation anchor is missing or duplicated" + ) + return value.replace(old, new) + + def replace_section(value: str, start: str, end: str, replacement: str) -> str: + if value.count(start) != 1 or value.count(end) != 1: + raise SealedStageALaunchError( + "calibration bootstrap section anchor is missing or duplicated" + ) + prefix, remainder = value.split(start, 1) + _discarded, suffix = remainder.split(end, 1) + return prefix + replacement + "\n" + end + suffix + + result = source + result = replace_exact( + result, + '"_recurquant_experiment013_sealed_runner"', + f'"{RUNNER_MODULE_NAME}"', + count=3, + ) + result = replace_exact( + result, + "scripts/run_static_q468_calibration.py", + RUNNER_SOURCE_PATH, + count=4, + ) + result = replace_section(result, "_smoke_options = {", "def _fail(message):", "") + result = replace_section( + result, + "def _options(arguments):", + "def _identity(data):", + stage_a_options, + ) + result = replace_section( + result, + "def _identity(data):", + "def _source(data):", + stage_a_identity, + ) + result = replace_exact( + result, + original_environment, + stage_a_environment, + count=1, + ) + result = replace_exact(result, "_smoke(_runner_options)\n", "", count=1) + forbidden = ( + "scripts/run_static_q468_calibration.py", + "--fisher-h1-smoke", + "--prior-fisher-h1-smoke-report", + "--prior-fisher-h1-smoke-complete-marker", + "full calibration", + ) + if any(value in result for value in forbidden): + raise SealedStageALaunchError("Stage-A bootstrap retained calibration-only runner logic") + return result + + +def _split_arguments(argv: Sequence[str]) -> tuple[list[str], list[str]]: + positions = [index for index, value in enumerate(argv) if value == "--"] + if len(positions) != 1: + raise SealedStageALaunchError("launcher requires exactly one -- separator") + position = positions[0] + host = list(argv[:position]) + runner = list(argv[position + 1 :]) + if not runner: + raise SealedStageALaunchError("launcher requires exact runner arguments after --") + return host, runner + + +def _parse_package_root(value: str) -> tuple[str, Path]: + if "=" not in value: + raise argparse.ArgumentTypeError("package roots use NAME=PATH") + name, path = value.split("=", 1) + if re.fullmatch(r"[a-z][a-z0-9-]{0,63}", name) is None or not path: + raise argparse.ArgumentTypeError("package root name or path is invalid") + return name, Path(path) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-runtime-root", required=True, type=Path) + parser.add_argument("--git-executable", required=True, type=Path) + parser.add_argument("--package-root", required=True, action="append", type=_parse_package_root) + parser.add_argument("--runtime-manifest", required=True, type=Path) + return parser + + +def _sealed_argv( + *, + interpreter: Path, + bootstrap: bytes, + runtime_manifest: Path, + base_runtime_root: Path, + package_roots: Mapping[str, Path], + git_executable: Path, + pycache_prefix: Path, + scratch_directory: Path, + runner_arguments: Sequence[str], +) -> list[str]: + roots = json.dumps( + {name: str(package_roots[name]) for name in sorted(package_roots)}, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + return [ + str(interpreter), + "-I", + "-S", + "-B", + "-X", + f"pycache_prefix={pycache_prefix}", + "-X", + "utf8", + "-c", + _authenticated_stdin_loader(bootstrap), + str(runtime_manifest), + str(base_runtime_root), + roots, + str(pycache_prefix), + str(git_executable), + str(scratch_directory), + *runner_arguments, + ] + + +def _sealed_environment(*, scratch_directory: Path, offline: bool) -> dict[str, str]: + scratch = scratch_directory.resolve(strict=True) + inherited = {key.upper(): (key, value) for key, value in os.environ.items()} + environment = { + inherited[name][0]: inherited[name][1] + for name in ( + "SYSTEMROOT", + "WINDIR", + "COMSPEC", + "PROCESSOR_ARCHITECTURE", + "PROCESSOR_ARCHITEW6432", + ) + if name in inherited + } + environment.update( + { + "LANG": "C", + "LC_ALL": "C", + "TEMP": str(scratch), + "TMP": str(scratch), + "TZ": "UTC", + } + ) + if offline: + environment.update( + { + "HF_DATASETS_OFFLINE": "1", + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + } + ) + return environment + + +def _authenticated_stdin_loader(payload: bytes) -> str: + """Return a small ``-c`` loader bound to the exact Stage-A bootstrap.""" + digest = _sha256_bytes(payload) + size = len(payload) + return ( + "import hashlib as _h,sys as _s\n" + f"_p=_s.stdin.buffer.read({size + 1})\n" + f"if len(_p)!={size} or _h.sha256(_p).hexdigest()!='{digest}':" + " raise RuntimeError('sealed Stage-A bootstrap stdin authentication failed')\n" + "exec(compile(_p,'','exec',dont_inherit=True))" + ) + + +def _verify_bound_inputs( + options: Mapping[str, str], + *, + runtime_manifest_path: Path, +) -> tuple[dict[str, str], dict[str, object], Path]: + identity_bytes = Path(options["--frozen-identity"]).read_bytes() + bindings = _parse_identity(identity_bytes) + for binding, option in _BOUND_ARTIFACT_OPTIONS.items(): + try: + data = Path(options[option]).read_bytes() + except OSError as error: + raise SealedStageALaunchError(f"bound artifact is unavailable: {option}") from error + if _sha256_bytes(data) != bindings[binding]: + raise SealedStageALaunchError(f"identity binding mismatch: {option}") + for binding, option in _EXPECTED_DIGEST_OPTIONS.items(): + if _sha256(options[option], context=f"runner option {option}") != bindings[binding]: + raise SealedStageALaunchError(f"runner digest binding mismatch: {option}") + if runtime_manifest_path.resolve(strict=True) != Path(options["--runtime-manifest"]).resolve( + strict=True + ): + raise SealedStageALaunchError("host and runner runtime-manifest paths differ") + source_bytes = Path(options["--repository-source-manifest"]).read_bytes() + source = _parse_source(source_bytes) + runner_path = _verify_source(source, Path(options["--repository-root"])) + return bindings, source, runner_path + + +def _run_sealed_child( + *, + calibration_launcher: ModuleType, + bootstrap: bytes, + runtime_manifest_path: Path, + runtime_manifest: Mapping[str, object], + interpreter: Path, + base_runtime_root: Path, + package_roots: Mapping[str, Path], + git_executable: Path, + source: Mapping[str, object], + options: Mapping[str, str], + runner_arguments: Sequence[str], + offline: bool, +) -> int: + pycache = Path(tempfile.mkdtemp(prefix="recurquant-exp013-stage-a-pycache-")) + scratch = Path(tempfile.mkdtemp(prefix="recurquant-exp013-stage-a-scratch-")) + if any(pycache.iterdir()): + raise SealedStageALaunchError("new Stage-A pycache directory is not empty") + calibration_launcher._verify_empty_scratch(scratch) + try: + command = _sealed_argv( + interpreter=interpreter, + bootstrap=bootstrap, + runtime_manifest=runtime_manifest_path, + base_runtime_root=base_runtime_root, + package_roots=package_roots, + git_executable=git_executable, + pycache_prefix=pycache, + scratch_directory=scratch, + runner_arguments=runner_arguments, + ) + completed = subprocess.run( + command, + check=False, + cwd=base_runtime_root, + env=_sealed_environment(scratch_directory=scratch, offline=offline), + input=bootstrap, + ) + if any(pycache.iterdir()): + raise SealedStageALaunchError("sealed Stage-A runner wrote bytecode") + calibration_launcher._verify_empty_scratch(scratch) + _bindings, repeated_source, _runner = _verify_bound_inputs( + options, + runtime_manifest_path=runtime_manifest_path, + ) + if repeated_source["git_executable"] != source["git_executable"]: + raise SealedStageALaunchError( + "source Git executable binding changed during Stage-A execution" + ) + calibration_launcher._verify_runtime( + runtime_manifest, + base_runtime_root=base_runtime_root, + package_roots=package_roots, + git_executable_path=git_executable, + require_current_process=False, + ) + return int(completed.returncode) + finally: + if any(pycache.iterdir()): + raise SealedStageALaunchError("Stage-A pycache changed before cleanup") + shutil.rmtree(pycache, ignore_errors=False) + calibration_launcher._assert_scratch_tree_has_no_reparse(scratch) + shutil.rmtree(scratch, ignore_errors=False) + if os.path.lexists(scratch): + raise SealedStageALaunchError("Stage-A scratch survived cleanup") + + +def launch(argv: Sequence[str]) -> int: + if list(argv) in (["-h"], ["--help"]): + _parser().print_help() + print("\nAppend -- followed by exact screen_static_q468_stage_a.py arguments.") + return 0 + host_arguments, runner_arguments = _split_arguments(argv) + args = _parser().parse_args(host_arguments) + options = _extract_options(runner_arguments) + package_roots: dict[str, Path] = {} + for name, path in args.package_root: + if name in package_roots or name == BASE_RUNTIME_ROOT_NAME: + raise SealedStageALaunchError(f"duplicate or reserved package root: {name}") + package_roots[name] = path + _bindings, source, _runner = _verify_bound_inputs( + options, runtime_manifest_path=args.runtime_manifest + ) + repository_root = Path(options["--repository-root"]) + calibration_launcher = _load_calibration_launcher(repository_root, source) + try: + runtime_bytes = args.runtime_manifest.read_bytes() + runtime_manifest = calibration_launcher._parse_runtime_manifest(runtime_bytes) + if source["git_executable"] != { + "sha256": runtime_manifest["git_executable"]["sha256"], + "size_bytes": runtime_manifest["git_executable"]["size_bytes"], + }: + raise SealedStageALaunchError( + "source and runtime manifests bind different Git executable bytes" + ) + base, packages, _import_paths, interpreter, git_executable = ( + calibration_launcher._verify_runtime( + runtime_manifest, + base_runtime_root=args.base_runtime_root, + package_roots=package_roots, + git_executable_path=args.git_executable, + require_current_process=False, + ) + ) + bootstrap = _stage_a_bootstrap(calibration_launcher).encode("utf-8") + runtime_manifest_path = args.runtime_manifest.resolve(strict=True) + mode = runner_arguments[0] + runs: list[tuple[list[str], bool]] = [] + if mode in {"execute", "preflight"}: + runs.append((["prepare-inputs", *runner_arguments[1:]], False)) + runs.append((list(runner_arguments), mode != "prepare-inputs")) + for index, (child_arguments, offline) in enumerate(runs): + return_code = _run_sealed_child( + calibration_launcher=calibration_launcher, + bootstrap=bootstrap, + runtime_manifest_path=runtime_manifest_path, + runtime_manifest=runtime_manifest, + interpreter=interpreter, + base_runtime_root=base, + package_roots=packages, + git_executable=git_executable, + source=source, + options=options, + runner_arguments=child_arguments, + offline=offline, + ) + if return_code != 0 or index == len(runs) - 1: + return return_code + raise AssertionError("Stage-A child execution schedule was empty") + finally: + sys.modules.pop(CALIBRATION_LAUNCHER_MODULE_NAME, None) + + +def main(argv: Sequence[str] | None = None) -> int: + return launch(list(sys.argv[1:] if argv is None else argv)) + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/scripts/resolve_static_q468_identity.py b/scripts/resolve_static_q468_identity.py index 7f28f54..7c5807f 100644 --- a/scripts/resolve_static_q468_identity.py +++ b/scripts/resolve_static_q468_identity.py @@ -20,20 +20,23 @@ import hashlib import json import os +import re import string import sys import tempfile import unicodedata from collections.abc import Mapping, Sequence from copy import deepcopy +from dataclasses import dataclass from pathlib import Path +from types import MappingProxyType from typing import Any, Final -INPUT_SCHEMA: Final = "recurquant.experiment013.identity-input.v4" -CANDIDATE_SCHEMA: Final = "recurquant.experiment013.identity-candidate.v4" -FROZEN_SCHEMA: Final = "recurquant.experiment013.identity-frozen.v4" +INPUT_SCHEMA: Final = "recurquant.experiment013.identity-input.v5" +CANDIDATE_SCHEMA: Final = "recurquant.experiment013.identity-candidate.v5" +FROZEN_SCHEMA: Final = "recurquant.experiment013.identity-frozen.v5" ARTIFACT_KIND: Final = "recurquant_static_rht_q468_identity" -RESOLVER_VERSION: Final = 4 +RESOLVER_VERSION: Final = 5 PARQUET_MATERIALIZATION_MANIFEST_FILE_SHA256: Final = ( "ee5628e50e5d3516fd79077542d355fd915455ac0e53128d372f4177ad63d39c" ) @@ -70,6 +73,45 @@ "ruler": "configuration_id", "humaneval_plus": "task_id", } +FROZEN_DATASET_CONFIGS: Final = { + "mbpp": MBPP_CONFIG, + "pg19": "default", + "ruler": "official-generator", + "humaneval_plus": "default", +} +FROZEN_DATASET_SPLITS: Final = { + "calibration": { + "mbpp": "train", + "pg19": "train", + "ruler": "generated", + "humaneval_plus": "test", + }, + "stage_a": { + "mbpp": "train", + "pg19": "validation", + "ruler": "generated", + "humaneval_plus": "test", + }, +} +FROZEN_FORMATTER_IDS: Final = { + "mbpp": "recurquant.mbpp-prompt-code.v1", + "pg19": "recurquant.pg19-token-slice.v1", + "ruler": "recurquant.ruler-official-generated-record.v1", + "humaneval_plus": "recurquant.humaneval-plus-prompt-solution.v1", +} +FROZEN_STATIC_FORMATTER_SHA256: Final = { + "mbpp": "882e20ec9f5cbcb7e6f1310cbf46d19153721beac41ecc7ee308c39be17532ff", + "pg19": "faea2480bf85adcd34339cae88a0e9b631b705eb547020572db74402bf525730", + "humaneval_plus": "12204389715ddc210e5a8b1b291f4fbcaf2b64fde94c22699873de80d682204c", +} +MAX_METADATA_STRING_LENGTH: Final = 512 +PG19_CANONICAL_URL_RE: Final = re.compile( + r"http://www\.gutenberg\.org/ebooks/(?P[1-9][0-9]{0,7})\Z" +) +HUMANEVAL_PLUS_TASK_ID_RE: Final = re.compile(r"HumanEval/(?P0|[1-9][0-9]{0,2})\Z") +HUMANEVAL_PLUS_TASK_COUNT: Final = 164 +TOKENIZER_CLASS_RE: Final = re.compile(r"[A-Za-z_][A-Za-z0-9_.]{0,127}\Z") +TOKENIZER_FILE_NAME_RE: Final = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}\Z") PG19_TRAIN_NAMESPACE: Final = "recurquant.experiment013.pg19.train.v1\0" PG19_VALIDATION_NAMESPACE: Final = "recurquant.experiment013.pg19.validation.v1\0" @@ -78,10 +120,17 @@ HUMANEVAL_C_NAMESPACE: Final = "recurquant.experiment013.humaneval-plus.stage-c.v1\0" CALIBRATION_SPLIT_NAMESPACE: Final = "recurquant.experiment013.calibration-split.v1\0" IDENTITY_RECORD_NAMESPACE: Final = "recurquant.experiment013.identity-record.v1\0" +FISHER_BOUNDARY_SCHEMA: Final = "recurquant.experiment013.fisher-boundary.v1" +FISHER_BOUNDARY_NAMESPACE: Final = "recurquant.experiment013.fisher-boundary.v1\0" +FISHER_BOUNDARY_TOKEN_NAMESPACE: Final = ( + "recurquant.experiment013.fisher-boundary-token-sequence.v1\0" +) +FISHER_BOUNDARY_HORIZON: Final = 1 RULER_CALIBRATION_SELECTION_NAMESPACE: Final = ( "recurquant.experiment013.ruler.calibration-sequence.v1\0" ) RULER_STAGE_A_SELECTION_NAMESPACE: Final = "recurquant.experiment013.ruler.stage-a-sequence.v1\0" +RULER_SEQUENCE_NAMESPACE: Final = "recurquant.experiment013.ruler.sequence.v1" RULER_CATEGORIES: Final = ( "retrieval", "multi_hop_tracing", @@ -175,6 +224,7 @@ "tokenizer_manifest_sha256", "token_span", "anchor_manifest_sha256", + "fisher_boundary", "identity_record_sha256", } ) @@ -189,13 +239,29 @@ "cache_exposed_stop", } ) +FISHER_BOUNDARY_FIELDS: Final = frozenset( + { + "schema", + "horizon", + "boundary_positions", + "input_positions", + "target_positions", + "input_token_ids_sha256", + "target_token_ids_sha256", + "fisher_boundary_sha256", + } +) +FISHER_BOUNDARY_PAYLOAD_FIELDS: Final = FISHER_BOUNDARY_FIELDS - {"fisher_boundary_sha256"} CALIBRATION_BINDING_FIELDS: Final = frozenset( { "calibration_identity_file_sha256", "calibration_score_artifact_file_sha256", + "comparator_score_artifact_file_sha256", "split_half_stability_artifact_file_sha256", + "static_fisher_k29334_policy_file_sha256", "static_k27030_policy_file_sha256", "static_k29334_policy_file_sha256", + "static_mse_k29334_policy_file_sha256", } ) EXECUTION_BINDING_FIELDS: Final = frozenset( @@ -239,8 +305,8 @@ "anchor_positions_sha256", } STAGE_A_BINDING_ARTIFACT_KIND: Final = "recurquant_experiment013_stage_a_calibration_binding" -STAGE_A_BINDING_ARTIFACT_SCHEMA_VERSION: Final = 2 -STAGE_A_BINDING_ARTIFACT_REVISION: Final = "experiment-013-stage-a-calibration-binding-v2" +STAGE_A_BINDING_ARTIFACT_SCHEMA_VERSION: Final = 3 +STAGE_A_BINDING_ARTIFACT_REVISION: Final = "experiment-013-stage-a-calibration-binding-v3" def canonical_json_bytes(value: object) -> bytes: @@ -248,7 +314,7 @@ def canonical_json_bytes(value: object) -> bytes: return ( json.dumps( - value, + _deep_thaw(value), ensure_ascii=False, allow_nan=False, sort_keys=True, @@ -294,11 +360,88 @@ def require_int(value: object, *, context: str, minimum: int = 0) -> int: return value -def require_string(value: object, *, context: str, allow_empty: bool = False) -> str: +def require_string( + value: object, + *, + context: str, + allow_empty: bool = False, + maximum_length: int = MAX_METADATA_STRING_LENGTH, +) -> str: if not isinstance(value, str) or (not allow_empty and not value): raise ValueError(f"{context} must be a non-empty string") if value != unicodedata.normalize("NFC", value) or value != value.strip(): raise ValueError(f"{context} must be stripped NFC text") + if len(value) > maximum_length: + raise ValueError(f"{context} exceeds the metadata length limit of {maximum_length}") + if any(unicodedata.category(character).startswith("C") for character in value): + raise ValueError(f"{context} contains a forbidden control character") + if any(character.isspace() for character in value): + raise ValueError(f"{context} cannot contain whitespace or raw content") + return value + + +def _validate_pg19_canonical_url(value: str, *, context: str) -> None: + """Require the exact URL shape emitted by the pinned PG19 extractor.""" + + match = PG19_CANONICAL_URL_RE.fullmatch(value) + if match is None: + raise ValueError( + f"{context} must be an exact http://www.gutenberg.org/ebooks/ URL" + ) + + +def _validate_humaneval_plus_task_id(value: str, *, context: str) -> None: + """Require one canonical HumanEval+ task ID from the pinned 164-row split.""" + + match = HUMANEVAL_PLUS_TASK_ID_RE.fullmatch(value) + if match is None or int(match.group("task_number")) >= HUMANEVAL_PLUS_TASK_COUNT: + raise ValueError(f"{context} must be a canonical HumanEval/0..163 task ID") + + +class _FrozenSequence(tuple[Any, ...]): + """Tuple storage with structural equality against ordinary JSON arrays.""" + + def __new__(cls, values: Sequence[Any]) -> _FrozenSequence: + return super().__new__(cls, values) + + def __eq__(self, other: object) -> bool: + if isinstance(other, Sequence) and not isinstance(other, (str, bytes, bytearray)): + return tuple(self) == tuple(other) + return NotImplemented + + __hash__ = tuple.__hash__ + + @staticmethod + def _immutable(*_args: object, **_kwargs: object) -> None: + raise TypeError("verified resolver sequences are immutable") + + append = _immutable + clear = _immutable + extend = _immutable + insert = _immutable + pop = _immutable + remove = _immutable + reverse = _immutable + sort = _immutable + + +def _deep_freeze(value: Any) -> Any: + """Recursively detach and freeze JSON-like verified data.""" + + if isinstance(value, Mapping): + return MappingProxyType({key: _deep_freeze(item) for key, item in value.items()}) + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return _FrozenSequence(tuple(_deep_freeze(item) for item in value)) + return value + + +def _deep_thaw(value: Any) -> Any: + """Convert immutable DTO views back to ordinary canonical-JSON containers.""" + + if isinstance(value, Mapping): + return {key: _deep_thaw(item) for key, item in value.items()} + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return [_deep_thaw(item) for item in value] return value @@ -336,6 +479,126 @@ def sequence_token_ids_sha256(token_ids: Sequence[int]) -> str: return sha256_bytes(canonical_json_bytes(normalized)) +def _fisher_boundary_token_ids_sha256(token_ids: Sequence[int], *, role: str) -> str: + """Hash one ordered Fisher token sequence under an exact role binding.""" + + if role not in {"input", "target"}: + raise ValueError("Fisher boundary token role must be input or target") + if isinstance(token_ids, (str, bytes, bytearray)) or not isinstance(token_ids, Sequence): + raise ValueError("Fisher boundary token IDs must be an integer sequence") + normalized = [ + require_int(token_id, context=f"Fisher boundary {role} token IDs[{index}]") + for index, token_id in enumerate(token_ids) + ] + payload = {"role": role, "token_ids": normalized} + return sha256_bytes( + FISHER_BOUNDARY_TOKEN_NAMESPACE.encode("utf-8") + canonical_json_bytes(payload) + ) + + +def fisher_boundary_sha256(boundary: Mapping[str, Any]) -> str: + """Hash the public Fisher boundary payload under its dedicated domain.""" + + missing = FISHER_BOUNDARY_PAYLOAD_FIELDS - set(boundary) + if missing: + raise ValueError(f"Fisher boundary payload is missing fields: {sorted(missing)}") + payload = {name: boundary[name] for name in sorted(FISHER_BOUNDARY_PAYLOAD_FIELDS)} + return sha256_bytes(FISHER_BOUNDARY_NAMESPACE.encode("utf-8") + canonical_json_bytes(payload)) + + +def build_fisher_boundary_contract(token_ids: Sequence[int]) -> dict[str, Any]: + """Bind the frozen H=1 Fisher input/target pairs without exposing token IDs.""" + + if isinstance(token_ids, (str, bytes, bytearray)) or not isinstance(token_ids, Sequence): + raise ValueError("Fisher boundary token IDs must be an integer sequence") + normalized = [ + require_int(token_id, context=f"Fisher boundary sequence token IDs[{index}]") + for index, token_id in enumerate(token_ids) + ] + if len(normalized) < 3: + raise ValueError("Fisher boundary requires a sequence length of at least three tokens") + boundary_positions = list(anchor_positions(len(normalized) - 2)) + input_positions = [boundary + FISHER_BOUNDARY_HORIZON for boundary in boundary_positions] + target_positions = [position + 1 for position in input_positions] + payload: dict[str, Any] = { + "schema": FISHER_BOUNDARY_SCHEMA, + "horizon": FISHER_BOUNDARY_HORIZON, + "boundary_positions": boundary_positions, + "input_positions": input_positions, + "target_positions": target_positions, + "input_token_ids_sha256": _fisher_boundary_token_ids_sha256( + [normalized[position] for position in input_positions], + role="input", + ), + "target_token_ids_sha256": _fisher_boundary_token_ids_sha256( + [normalized[position] for position in target_positions], + role="target", + ), + } + payload["fisher_boundary_sha256"] = fisher_boundary_sha256(payload) + return payload + + +def _normalize_fisher_boundary( + value: object, *, sequence_length: int, context: str +) -> dict[str, Any]: + """Strictly validate a redacted Fisher boundary against B(T), H, and itself.""" + + boundary = require_mapping(value, context=context) + require_exact_fields(boundary, FISHER_BOUNDARY_FIELDS, context=context) + if boundary["schema"] != FISHER_BOUNDARY_SCHEMA: + raise ValueError(f"{context} schema drifted") + horizon = require_int(boundary["horizon"], context=f"{context}.horizon", minimum=1) + if horizon != FISHER_BOUNDARY_HORIZON: + raise ValueError(f"{context} horizon must equal H=1") + if sequence_length < 3: + raise ValueError(f"{context} requires a sequence length of at least three tokens") + + expected_boundary_positions = list(anchor_positions(sequence_length - 2)) + expected_input_positions = [position + horizon for position in expected_boundary_positions] + expected_target_positions = [position + 1 for position in expected_input_positions] + + def positions(name: str) -> list[int]: + raw_positions = require_sequence(boundary[name], context=f"{context}.{name}") + return [ + require_int(position, context=f"{context}.{name}[{index}]") + for index, position in enumerate(raw_positions) + ] + + normalized_boundary_positions = positions("boundary_positions") + normalized_input_positions = positions("input_positions") + normalized_target_positions = positions("target_positions") + if normalized_boundary_positions != expected_boundary_positions: + raise ValueError(f"{context} boundary positions differ from B(T)=anchor_positions(T-2)") + if normalized_input_positions != expected_input_positions: + raise ValueError(f"{context} input positions differ from x[b+1]") + if normalized_target_positions != expected_target_positions: + raise ValueError(f"{context} target positions differ from x[b+2]") + + normalized = { + "schema": FISHER_BOUNDARY_SCHEMA, + "horizon": horizon, + "boundary_positions": normalized_boundary_positions, + "input_positions": normalized_input_positions, + "target_positions": normalized_target_positions, + "input_token_ids_sha256": require_sha256( + boundary["input_token_ids_sha256"], + context=f"{context}.input_token_ids_sha256", + ), + "target_token_ids_sha256": require_sha256( + boundary["target_token_ids_sha256"], + context=f"{context}.target_token_ids_sha256", + ), + "fisher_boundary_sha256": require_sha256( + boundary["fisher_boundary_sha256"], + context=f"{context}.fisher_boundary_sha256", + ), + } + if normalized["fisher_boundary_sha256"] != fisher_boundary_sha256(normalized): + raise ValueError(f"{context} self-hash drifted") + return normalized + + def identity_anchor_manifest_sha256( *, canonical_id: str, @@ -379,6 +642,15 @@ def selection_sha256(namespace: str, canonical_id: str) -> str: return sha256_bytes(namespace.encode("utf-8") + canonical_id.encode("utf-8")) +def ruler_canonical_id(*, category: str, config: str, configured_length: int, seed: int) -> str: + """Return the sole canonical identity for one frozen RULER generation tuple.""" + + return ( + f"{RULER_SEQUENCE_NAMESPACE}:{RULER_REVISION}:{category}:{config}:" + f"length={configured_length}:seed={seed}:sample=0" + ) + + def mbpp_selection_sha256(canonical_id: str) -> str: return sha256_bytes(f"{MBPP_SELECTION_NAMESPACE}|{canonical_id}".encode()) @@ -422,6 +694,9 @@ def mbpp_calibration_identity() -> tuple[tuple[str, ...], str]: def _validate_sha_rank_order(rows: Sequence[Mapping[str, Any]], *, context: str) -> None: + selection_keys = [(str(row["selection_sha256"]), str(row["canonical_id"])) for row in rows] + if len(selection_keys) != len(set(selection_keys)): + raise ValueError(f"{context} contains duplicate canonical selection keys") ranked = sorted( rows, key=lambda row: (str(row["selection_sha256"]), str(row["canonical_id"])), @@ -451,8 +726,13 @@ def pairs_hook(pairs: list[tuple[str, Any]]) -> dict[str, Any]: def _validate_dataset_contracts( - value: object, *, expected_revisions: Mapping[str, str] + value: object, + *, + expected_revisions: Mapping[str, str], + phase: str, ) -> tuple[dict[str, Any], ...]: + if phase not in ALLOWED_PHASES: + raise ValueError(f"dataset contract phase is unsupported: {phase!r}") entries = require_sequence(value, context="datasets") if len(entries) != len(DATASET_KEYS): raise ValueError("datasets must contain exactly four contracts") @@ -496,6 +776,20 @@ def _validate_dataset_contracts( raise ValueError( f"{key} canonical ID field must be {FROZEN_CANONICAL_ID_FIELDS[key]!r}" ) + if contract["config"] != FROZEN_DATASET_CONFIGS[key]: + raise ValueError(f"{key} dataset config must be {FROZEN_DATASET_CONFIGS[key]!r}") + if contract["split"] != FROZEN_DATASET_SPLITS[phase][key]: + raise ValueError( + f"{phase} {key} dataset split must be {FROZEN_DATASET_SPLITS[phase][key]!r}" + ) + if contract["formatter_id"] != FROZEN_FORMATTER_IDS[key]: + raise ValueError(f"{key} formatter ID must be {FROZEN_FORMATTER_IDS[key]!r}") + expected_formatter_sha256 = FROZEN_STATIC_FORMATTER_SHA256.get(key) + if ( + expected_formatter_sha256 is not None + and contract["formatter_sha256"] != expected_formatter_sha256 + ): + raise ValueError(f"{key} formatter SHA-256 drifted from the frozen specification") if key == "mbpp" and ( contract["dataset_id"] != MBPP_DATASET_ID or contract["config"] != MBPP_CONFIG ): @@ -526,6 +820,8 @@ def _validate_tokenizer(value: object) -> dict[str, Any]: if tokenizer["transformers_version"] != TRANSFORMERS_VERSION: raise ValueError("Transformers version drifted") tokenizer_class = require_string(tokenizer["class"], context="tokenizer.class") + if TOKENIZER_CLASS_RE.fullmatch(tokenizer_class) is None: + raise ValueError("tokenizer.class must be a bounded Python identifier") raw_files = require_sequence(tokenizer["files"], context="tokenizer.files") if not raw_files: raise ValueError("tokenizer.files cannot be empty") @@ -535,7 +831,11 @@ def _validate_tokenizer(value: object) -> dict[str, Any]: item = require_mapping(raw, context=f"tokenizer.files[{index}]") require_exact_fields(item, TOKENIZER_FILE_FIELDS, context=f"tokenizer.files[{index}]") name = require_string(item["name"], context=f"tokenizer.files[{index}].name") - if Path(name).name != name or name in names: + if ( + Path(name).name != name + or TOKENIZER_FILE_NAME_RE.fullmatch(name) is None + or name in names + ): raise ValueError("tokenizer file names must be unique basenames") names.add(name) files.append( @@ -593,6 +893,16 @@ def _normalize_record( raise ValueError(f"records[{index}].family is unknown") canonical_id = require_string(item["canonical_id"], context=f"records[{index}].canonical_id") config = require_string(item["config"], context=f"records[{index}].config", allow_empty=True) + if family == "pg19": + _validate_pg19_canonical_url( + canonical_id, + context=f"records[{index}].canonical_id", + ) + elif family == "humaneval_plus": + _validate_humaneval_plus_task_id( + canonical_id, + context=f"records[{index}].canonical_id", + ) rank = require_int(item["selection_rank"], context=f"records[{index}].selection_rank") seed_value = item["seed"] seed = None if seed_value is None else require_int(seed_value, context=f"records[{index}].seed") @@ -628,14 +938,27 @@ def _normalize_record( expected_category = RULER_CONFIG_CATEGORY.get(config) if expected_category is None or ruler_category != expected_category: raise ValueError(f"records[{index}] RULER config/category binding drifted") - if configured_length is None or generator_receipt_sha256 is None: + if configured_length is None or generator_receipt_sha256 is None or seed is None: raise ValueError( - f"records[{index}] RULER configured length and generator receipt are required" + f"records[{index}] RULER configured length, seed, and generator receipt " + "are required" ) + expected_canonical_id = ruler_canonical_id( + category=ruler_category, + config=config, + configured_length=configured_length, + seed=seed, + ) + if canonical_id != expected_canonical_id: + raise ValueError(f"records[{index}] RULER canonical ID drifted") elif any( value is not None for value in (configured_length, ruler_category, generator_receipt_sha256) ): raise ValueError(f"records[{index}] non-RULER rows cannot carry RULER-only fields") + elif config != FROZEN_DATASET_CONFIGS[family]: + raise ValueError( + f"records[{index}] {family} config must be {FROZEN_DATASET_CONFIGS[family]!r}" + ) sequence_length = require_int( item["sequence_length"], context=f"records[{index}].sequence_length", @@ -699,6 +1022,11 @@ def _normalize_record( if configured_length is not None and sequence_length > configured_length: raise ValueError(f"records[{index}] actual sequence exceeds the RULER configured length") positions = anchor_positions(sequence_length) + fisher_boundary = _normalize_fisher_boundary( + item["fisher_boundary"], + sequence_length=sequence_length, + context=f"records[{index}].fisher_boundary", + ) sequence_hash = require_sha256( item["sequence_token_ids_sha256"], context=f"records[{index}].sequence_token_ids_sha256", @@ -746,6 +1074,7 @@ def _normalize_record( "tokenizer_manifest_sha256": tokenizer_hash, "token_span": normalized_span, "anchor_manifest_sha256": recorded_anchor_hash, + "fisher_boundary": fisher_boundary, "identity_record_sha256": require_sha256( item["identity_record_sha256"], context=f"records[{index}].identity_record_sha256", @@ -834,17 +1163,7 @@ def _validate_calibration_records(records: Sequence[Mapping[str, Any]]) -> None: for row in grouped["mbpp"]: if row["seed"] is not None: raise ValueError("MBPP calibration records cannot have a generator seed") - identities = [ - ( - row["family"], - row["canonical_id"], - row["ruler_category"], - row["config"], - row["configured_length"], - row["seed"], - ) - for row in records - ] + identities = [(row["family"], row["canonical_id"]) for row in records] if len(identities) != len(set(identities)): raise ValueError("calibration canonical identities are not unique") @@ -910,15 +1229,7 @@ def _validate_stage_a_records(records: Sequence[Mapping[str, Any]]) -> None: ) if row["sequence_length"] != span["scored_stop"]: raise ValueError("Stage-A HumanEval+ sequence length must equal span stop") - identities = [ - ( - row["family"], - row["canonical_id"], - row["ruler_category"], - row["config"], - ) - for row in records - ] + identities = [(row["family"], row["canonical_id"]) for row in records] if len(identities) != len(set(identities)): raise ValueError("Stage-A canonical identities are not unique") @@ -1008,7 +1319,7 @@ def build_candidate( raise ValueError("identity input schema drifted") if phase not in ALLOWED_PHASES: if phase in PROTECTED_STAGES: - raise PermissionError(f"{phase} is protected and unavailable in resolver v4") + raise PermissionError(f"{phase} is protected and unavailable in resolver v5") raise ValueError(f"unsupported identity phase: {phase!r}") if source["model_weights_loaded"] is not False: raise ValueError("identity resolution must occur before model weights") @@ -1029,16 +1340,17 @@ def build_candidate( } if revisions != FROZEN_DATASET_REVISIONS: raise ValueError("CLI dataset revisions do not match the frozen upstream commits") - datasets = _validate_dataset_contracts(source["datasets"], expected_revisions=revisions) + datasets = _validate_dataset_contracts( + source["datasets"], + expected_revisions=revisions, + phase=str(phase), + ) tokenizer = _validate_tokenizer(source["tokenizer"]) execution_bindings = _validate_execution_bindings(source["execution_bindings"]) parquet_materialization_manifest_file_sha256 = execution_bindings[ "parquet_materialization_manifest_file_sha256" ] - if ( - parquet_materialization_manifest_file_sha256 - != PARQUET_MATERIALIZATION_MANIFEST_FILE_SHA256 - ): + if parquet_materialization_manifest_file_sha256 != PARQUET_MATERIALIZATION_MANIFEST_FILE_SHA256: raise ValueError("Parquet materialization manifest file SHA-256 drifted") raw_records = require_sequence(source["records"], context="records") records = [ @@ -1065,7 +1377,7 @@ def build_candidate( content_manifest_hash = sha256_bytes(canonical_json_bytes(records)) source_hash = sha256_bytes(canonical_json_bytes(source)) evidence: dict[str, Any] = { - "schema_version": 4, + "schema_version": 5, "artifact_kind": ARTIFACT_KIND, "identity_schema": CANDIDATE_SCHEMA, "resolver_version": RESOLVER_VERSION, @@ -1136,7 +1448,7 @@ def validate_candidate_artifact(artifact: Mapping[str, Any]) -> None: raise ValueError("candidate canonical evidence SHA-256 drifted") phase = evidence["phase"] exact_scalars = { - "schema_version": 4, + "schema_version": 5, "artifact_kind": ARTIFACT_KIND, "identity_schema": CANDIDATE_SCHEMA, "resolver_version": RESOLVER_VERSION, @@ -1147,8 +1459,7 @@ def validate_candidate_artifact(artifact: Mapping[str, Any]) -> None: } for name, expected in exact_scalars.items(): if ( - isinstance(expected, (bool, int)) - and type(evidence[name]) is not type(expected) + isinstance(expected, (bool, int)) and type(evidence[name]) is not type(expected) ) or evidence[name] != expected: raise ValueError(f"candidate {name} drifted") if phase not in ALLOWED_PHASES: @@ -1201,6 +1512,7 @@ def validate_candidate_artifact(artifact: Mapping[str, Any]) -> None: datasets = _validate_dataset_contracts( evidence["datasets"], expected_revisions=FROZEN_DATASET_REVISIONS, + phase=str(phase), ) if list(datasets) != evidence["datasets"]: raise ValueError("candidate dataset contracts are not canonical") @@ -1330,101 +1642,71 @@ def promote_candidate( } +@dataclass(frozen=True, slots=True) class FrozenCalibrationIdentityArtifact: """Strictly verified frozen calibration identity and its binding commitments.""" - __slots__ = ( - "file_sha256", - "canonical_evidence_sha256", - "records", - "assignment", - "assignment_sha256", - "tokenizer_manifest_sha256", - "parquet_materialization_manifest_file_sha256", - "execution_bindings", - ) - - def __init__( - self, - *, - file_sha256: str, - canonical_evidence_sha256: str, - records: tuple[dict[str, Any], ...], - assignment: tuple[dict[str, Any], ...], - assignment_sha256: str, - tokenizer_manifest_sha256: str, - parquet_materialization_manifest_file_sha256: str, - execution_bindings: dict[str, str], - ) -> None: - self.file_sha256 = file_sha256 - self.canonical_evidence_sha256 = canonical_evidence_sha256 - self.records = records - self.assignment = assignment - self.assignment_sha256 = assignment_sha256 - self.tokenizer_manifest_sha256 = tokenizer_manifest_sha256 - self.parquet_materialization_manifest_file_sha256 = ( - parquet_materialization_manifest_file_sha256 + file_sha256: str + canonical_evidence_sha256: str + records: tuple[Mapping[str, Any], ...] + assignment: tuple[Mapping[str, Any], ...] + assignment_sha256: str + tokenizer_manifest_sha256: str + parquet_materialization_manifest_file_sha256: str + execution_bindings: Mapping[str, str] + + def __post_init__(self) -> None: + object.__setattr__( + self, + "records", + tuple(_deep_freeze(record) for record in self.records), + ) + object.__setattr__( + self, + "assignment", + tuple(_deep_freeze(item) for item in self.assignment), ) - self.execution_bindings = execution_bindings + object.__setattr__(self, "execution_bindings", _deep_freeze(self.execution_bindings)) +@dataclass(frozen=True, slots=True) class FrozenStageAIdentityArtifact: - """Strictly verified frozen Stage-A identity and five-file calibration binding.""" - - __slots__ = ( - "file_sha256", - "canonical_evidence_sha256", - "records", - "tokenizer_manifest_sha256", - "calibration_binding", - "parquet_materialization_manifest_file_sha256", - "execution_bindings", - ) - - def __init__( - self, - *, - file_sha256: str, - canonical_evidence_sha256: str, - records: tuple[dict[str, Any], ...], - tokenizer_manifest_sha256: str, - calibration_binding: dict[str, str], - parquet_materialization_manifest_file_sha256: str, - execution_bindings: dict[str, str], - ) -> None: - self.file_sha256 = file_sha256 - self.canonical_evidence_sha256 = canonical_evidence_sha256 - self.records = records - self.tokenizer_manifest_sha256 = tokenizer_manifest_sha256 - self.calibration_binding = calibration_binding - self.parquet_materialization_manifest_file_sha256 = ( - parquet_materialization_manifest_file_sha256 + """Strictly verified frozen Stage-A identity and eight-file calibration binding.""" + + file_sha256: str + canonical_evidence_sha256: str + records: tuple[Mapping[str, Any], ...] + tokenizer_manifest_sha256: str + calibration_binding: Mapping[str, str] + parquet_materialization_manifest_file_sha256: str + execution_bindings: Mapping[str, str] + + def __post_init__(self) -> None: + object.__setattr__( + self, + "records", + tuple(_deep_freeze(record) for record in self.records), ) - self.execution_bindings = execution_bindings + object.__setattr__(self, "calibration_binding", _deep_freeze(self.calibration_binding)) + object.__setattr__(self, "execution_bindings", _deep_freeze(self.execution_bindings)) +@dataclass(frozen=True, slots=True) class StageACalibrationBindingArtifact: - """Verified five-field Stage-A binding and authenticated dependency hashes.""" - - __slots__ = ( - "binding", - "dependency_file_sha256", - "canonical_evidence_sha256", - "file_sha256", - ) - - def __init__( - self, - *, - binding: dict[str, str], - dependency_file_sha256: dict[str, str], - canonical_evidence_sha256: str, - file_sha256: str, - ) -> None: - self.binding = binding - self.dependency_file_sha256 = dependency_file_sha256 - self.canonical_evidence_sha256 = canonical_evidence_sha256 - self.file_sha256 = file_sha256 + """Verified eight-field Stage-A binding and authenticated dependency hashes.""" + + binding: Mapping[str, str] + dependency_file_sha256: Mapping[str, str] + canonical_evidence_sha256: str + file_sha256: str + + def __post_init__(self) -> None: + object.__setattr__(self, "binding", _deep_freeze(self.binding)) + object.__setattr__( + self, + "dependency_file_sha256", + _deep_freeze(self.dependency_file_sha256), + ) def deserialize_frozen_calibration_identity_artifact( @@ -1465,7 +1747,7 @@ def deserialize_frozen_calibration_identity_artifact( if canonical_evidence_sha256 != sha256_bytes(canonical_json_bytes(evidence)): raise ValueError("frozen identity canonical evidence SHA-256 drifted") exact_scalars = { - "schema_version": 4, + "schema_version": 5, "artifact_kind": ARTIFACT_KIND, "identity_schema": FROZEN_SCHEMA, "resolver_version": RESOLVER_VERSION, @@ -1478,8 +1760,7 @@ def deserialize_frozen_calibration_identity_artifact( } for name, expected in exact_scalars.items(): if ( - isinstance(expected, (bool, int)) - and type(evidence[name]) is not type(expected) + isinstance(expected, (bool, int)) and type(evidence[name]) is not type(expected) ) or evidence[name] != expected: raise ValueError(f"frozen identity {name} drifted") require_sha256( @@ -1489,9 +1770,7 @@ def deserialize_frozen_calibration_identity_artifact( execution_bindings = _validate_execution_bindings(evidence["execution_bindings"]) if dict(evidence["execution_bindings"]) != execution_bindings: raise ValueError("frozen execution bindings are not canonical") - parquet_manifest_sha256 = execution_bindings[ - "parquet_materialization_manifest_file_sha256" - ] + parquet_manifest_sha256 = execution_bindings["parquet_materialization_manifest_file_sha256"] if parquet_manifest_sha256 != PARQUET_MATERIALIZATION_MANIFEST_FILE_SHA256: raise ValueError("frozen Parquet materialization manifest file SHA-256 drifted") @@ -1530,6 +1809,7 @@ def deserialize_frozen_calibration_identity_artifact( datasets = _validate_dataset_contracts( evidence["datasets"], expected_revisions=FROZEN_DATASET_REVISIONS, + phase="calibration", ) if list(datasets) != evidence["datasets"]: raise ValueError("frozen dataset contracts are not canonical") @@ -1784,9 +2064,7 @@ def deserialize_frozen_stage_a_identity_artifact( context="frozen Stage-A tokenizer manifest SHA-256", ) execution_bindings = _validate_execution_bindings(candidate_evidence["execution_bindings"]) - parquet_manifest_sha256 = execution_bindings[ - "parquet_materialization_manifest_file_sha256" - ] + parquet_manifest_sha256 = execution_bindings["parquet_materialization_manifest_file_sha256"] if parquet_manifest_sha256 != PARQUET_MATERIALIZATION_MANIFEST_FILE_SHA256: raise ValueError("frozen Stage-A Parquet materialization manifest file SHA-256 drifted") return FrozenStageAIdentityArtifact( @@ -1877,24 +2155,48 @@ def _derive_stage_a_calibration_binding( split_half_stability_artifact: bytes, static_k27030_policy_artifact: bytes, static_k29334_policy_artifact: bytes, + comparator_score_artifact: bytes, + static_fisher_k29334_policy_artifact: bytes, + static_mse_k29334_policy_artifact: bytes, ) -> tuple[dict[str, str], dict[str, str]]: from recurquant.static_q468 import ( FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + FROZEN_STATELEASE_RESIDENT_BYTES, FROZEN_STATIC_Q468_ABLATION_STEPS, FROZEN_STATIC_Q468_PRIMARY_STEPS, + FROZEN_TRANSFORMERS_VERSION, + PRIMARY_MODEL_ID, + PRIMARY_MODEL_REVISION, + PRIMARY_TOKENIZER_ID, + PRIMARY_TOKENIZER_REVISION, STATIC_Q468_ABLATION_METHOD, + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + STATIC_Q468_MSE_METHOD, STATIC_Q468_PRIMARY_METHOD, + build_static_rht_q468_policy, deserialize_static_rht_q468_policy, + serialize_static_rht_q468_policy, + static_q468_byte_ledger, + static_q468_distortion_sha256, ) from recurquant.static_q468_calibration import ( CALIBRATION_SCORE_ARTIFACT_KIND, + FROZEN_COMPARATOR_PROFILE_ORDER, + FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE, + FROZEN_UNWEIGHTED_MSE_PROFILE, calibration_identity_record_manifest_sha256, deserialize_calibration_score_artifact, + deserialize_comparator_score_artifact, deserialize_frozen_split_half_stability_artifact, + static_q468_code_map_sha256, ) identity = deserialize_frozen_calibration_identity_artifact(frozen_identity_artifact) + if identity.file_sha256 != sha256_bytes(frozen_identity_artifact): + raise ValueError("frozen identity decoder returned the wrong dependency file hash") scores = deserialize_calibration_score_artifact(calibration_score_artifact) + if scores.file_sha256 != sha256_bytes(calibration_score_artifact): + raise ValueError("score decoder returned the wrong dependency file hash") if scores.artifact_kind != CALIBRATION_SCORE_ARTIFACT_KIND: raise ValueError("Stage-A binding requires the official frozen score artifact") expected_identity_manifest = calibration_identity_record_manifest_sha256(identity.records) @@ -1902,12 +2204,33 @@ def _derive_stage_a_calibration_binding( raise ValueError("score artifact is not bound to the frozen identity file") if scores.aggregate.identity_record_manifest_sha256 != expected_identity_manifest: raise ValueError("score identity-record manifest differs from the frozen identity") + comparators = deserialize_comparator_score_artifact( + comparator_score_artifact, + expected_calibration_identity_sha256=identity.file_sha256, + ) + if ( + comparators.file_sha256 != sha256_bytes(comparator_score_artifact) + or comparators.calibration_identity_sha256 != identity.file_sha256 + or tuple(comparators.selectors) != FROZEN_COMPARATOR_PROFILE_ORDER + ): + raise ValueError("comparator score artifact must contain exactly the two official profiles") + for method_id in FROZEN_COMPARATOR_PROFILE_ORDER: + if ( + comparators.selectors[method_id].aggregate.identity_record_manifest_sha256 + != expected_identity_manifest + ): + raise ValueError( + f"comparator {method_id} identity-record manifest differs from the " + "complete frozen identity" + ) split = deserialize_frozen_split_half_stability_artifact( split_half_stability_artifact, expected_identity_file_sha256=identity.file_sha256, expected_canonical_identity_sha256=identity.canonical_evidence_sha256, expected_resolver_assignment_sha256=identity.assignment_sha256, ) + if split.file_sha256 != sha256_bytes(split_half_stability_artifact): + raise ValueError("split-half decoder returned the wrong dependency file hash") if ( split.identity_file_sha256 != identity.file_sha256 or split.canonical_identity_sha256 != identity.canonical_evidence_sha256 @@ -1941,6 +2264,7 @@ def _derive_stage_a_calibration_binding( ), ) allocations = {steps: (codes, digest) for steps, codes, digest in scores.allocations} + torch = __import__("torch") for policy, method_id, steps in expected_policy_contracts: if ( policy.method_id != method_id @@ -1952,6 +2276,14 @@ def _derive_stage_a_calibration_binding( raise ValueError(f"policy {method_id} is not bound to the frozen identity file") if policy.tokenizer_manifest_sha256 != identity.tokenizer_manifest_sha256: raise ValueError(f"policy {method_id} tokenizer manifest differs from identity") + if ( + policy.model_id != PRIMARY_MODEL_ID + or policy.model_revision != PRIMARY_MODEL_REVISION + or policy.tokenizer_id != PRIMARY_TOKENIZER_ID + or policy.tokenizer_revision != PRIMARY_TOKENIZER_REVISION + or policy.transformers_version != FROZEN_TRANSFORMERS_VERSION + ): + raise ValueError(f"policy {method_id} frozen model contract drifted") if ( policy.calibration_manifest_sha256 != scores.aggregate.sequence_score_manifest_sha256 or policy.calibration_scores_sha256 != scores.calibration_scores_sha256 @@ -1960,27 +2292,147 @@ def _derive_stage_a_calibration_binding( if steps not in allocations: raise ValueError(f"official score artifact is missing exact K{steps}") allocation_codes, allocation_hash = allocations[steps] - if policy.code_map_sha256 != allocation_hash or not __import__("torch").equal( + if policy.code_map_sha256 != allocation_hash or not torch.equal( policy.precision_codes().reshape(-1).to("cpu"), - allocation_codes, + allocation_codes.reshape(-1).to("cpu"), ): raise ValueError(f"policy {method_id} code map differs from exact allocation") if policy27030.source_commit != policy29334.source_commit: raise ValueError("K27030 and K29334 policies must share one source commit") + source_commit_h0 = policy29334.source_commit + + comparator_policies = ( + ( + deserialize_static_rht_q468_policy(static_mse_k29334_policy_artifact), + STATIC_Q468_MSE_METHOD, + FROZEN_UNWEIGHTED_MSE_PROFILE, + ), + ( + deserialize_static_rht_q468_policy(static_fisher_k29334_policy_artifact), + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE, + ), + ) + for policy, method_id, selector_profile in comparator_policies: + if method_id != selector_profile: + raise RuntimeError("frozen comparator method/profile constants disagree") + selector = comparators.selectors[selector_profile] + if ( + policy.method_id != method_id + or policy.marginal_steps != FROZEN_STATIC_Q468_PRIMARY_STEPS + or policy.geometry != FROZEN_QWEN35_STATIC_Q468_GEOMETRY + ): + raise ValueError( + f"comparator policy {method_id} does not satisfy its frozen K29334 geometry" + ) + if ( + policy.model_id != PRIMARY_MODEL_ID + or policy.model_revision != PRIMARY_MODEL_REVISION + or policy.tokenizer_id != PRIMARY_TOKENIZER_ID + or policy.tokenizer_revision != PRIMARY_TOKENIZER_REVISION + or policy.transformers_version != FROZEN_TRANSFORMERS_VERSION + ): + raise ValueError(f"comparator policy {method_id} frozen model contract drifted") + if ( + policy.identity_artifact_sha256 != identity.file_sha256 + or policy.tokenizer_manifest_sha256 != identity.tokenizer_manifest_sha256 + ): + raise ValueError(f"comparator policy {method_id} frozen identity binding drifted") + if policy.source_commit != source_commit_h0: + raise ValueError(f"comparator policy {method_id} source commit differs from H0") + if ( + policy.calibration_manifest_sha256 != selector.aggregate.sequence_score_manifest_sha256 + or selector.calibration_scores_sha256 != selector.aggregate.aggregate_scores_sha256 + ): + raise ValueError( + f"comparator policy {method_id} differs from its decoded selector scores" + ) + expected_policy_score_sha256 = static_q468_distortion_sha256( + *selector.aggregate.scores(), + geometry=FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + ) + if policy.calibration_scores_sha256 != expected_policy_score_sha256: + raise ValueError( + f"comparator policy {method_id} raw distortion hash differs from its " + "decoded selector arrays" + ) + expected_policy = build_static_rht_q468_policy( + *selector.aggregate.scores(), + geometry=FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + marginal_steps=FROZEN_STATIC_Q468_PRIMARY_STEPS, + calibration_manifest_sha256=(selector.aggregate.sequence_score_manifest_sha256), + identity_artifact_sha256=identity.file_sha256, + tokenizer_manifest_sha256=identity.tokenizer_manifest_sha256, + source_commit=source_commit_h0, + calibration_scores_sha256=expected_policy_score_sha256, + method_id=method_id, + ) + if serialize_static_rht_q468_policy(expected_policy) != ( + static_mse_k29334_policy_artifact + if method_id == STATIC_Q468_MSE_METHOD + else static_fisher_k29334_policy_artifact + ): + raise ValueError( + f"comparator policy {method_id} bytes differ from deterministic reconstruction" + ) + selector_codes = selector.precision_codes.reshape(-1).to("cpu") + expected_code_map_sha256 = static_q468_code_map_sha256( + selector_codes, + geometry=FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + marginal_steps=FROZEN_STATIC_Q468_PRIMARY_STEPS, + ) + if ( + selector.method_id != method_id + or selector.marginal_steps != FROZEN_STATIC_Q468_PRIMARY_STEPS + or selector.code_map_sha256 != expected_code_map_sha256 + or policy.code_map_sha256 != expected_code_map_sha256 + or not torch.equal( + policy.precision_codes().reshape(-1).to("cpu"), + selector_codes, + ) + ): + raise ValueError( + f"comparator policy {method_id} code map differs from its exact allocation" + ) + ledger = static_q468_byte_ledger( + policy.geometry, + policy.marginal_steps, + method_id=policy.method_id, + ) + if ( + ledger.method_id != method_id + or ledger.selected_units != FROZEN_STATIC_Q468_PRIMARY_STEPS + or ledger.resident_bytes != FROZEN_STATELEASE_RESIDENT_BYTES + or ledger.target_resident_bytes != FROZEN_STATELEASE_RESIDENT_BYTES + or ledger.budget_delta_bytes != 0 + or ledger.exact_budget_eligible is not True + ): + raise ValueError( + f"comparator policy {method_id} does not realize the exact " + f"{FROZEN_STATELEASE_RESIDENT_BYTES}-byte ledger" + ) binding = { "calibration_identity_file_sha256": identity.file_sha256, "calibration_score_artifact_file_sha256": scores.file_sha256, + "comparator_score_artifact_file_sha256": comparators.file_sha256, "split_half_stability_artifact_file_sha256": split.file_sha256, + "static_fisher_k29334_policy_file_sha256": sha256_bytes( + static_fisher_k29334_policy_artifact + ), "static_k27030_policy_file_sha256": sha256_bytes(static_k27030_policy_artifact), "static_k29334_policy_file_sha256": sha256_bytes(static_k29334_policy_artifact), + "static_mse_k29334_policy_file_sha256": sha256_bytes(static_mse_k29334_policy_artifact), } dependency_hashes = { "calibration_score_artifact": scores.file_sha256, + "comparator_score_artifact": comparators.file_sha256, "frozen_identity_artifact": identity.file_sha256, "split_half_stability_artifact": split.file_sha256, + "static_fisher_k29334_policy_artifact": sha256_bytes(static_fisher_k29334_policy_artifact), "static_k27030_policy_artifact": sha256_bytes(static_k27030_policy_artifact), "static_k29334_policy_artifact": sha256_bytes(static_k29334_policy_artifact), + "static_mse_k29334_policy_artifact": sha256_bytes(static_mse_k29334_policy_artifact), } return binding, dependency_hashes @@ -1992,15 +2444,21 @@ def build_stage_a_calibration_binding_artifact( split_half_stability_artifact: bytes, static_k27030_policy_artifact: bytes, static_k29334_policy_artifact: bytes, + comparator_score_artifact: bytes, + static_fisher_k29334_policy_artifact: bytes, + static_mse_k29334_policy_artifact: bytes, ) -> bytes: - """Build the five-field Stage-A binding only from fully verified dependencies.""" + """Build the eight-field Stage-A binding only from fully verified dependencies.""" dependencies = { "calibration_score_artifact": calibration_score_artifact, + "comparator_score_artifact": comparator_score_artifact, "frozen_identity_artifact": frozen_identity_artifact, "split_half_stability_artifact": split_half_stability_artifact, + "static_fisher_k29334_policy_artifact": static_fisher_k29334_policy_artifact, "static_k27030_policy_artifact": static_k27030_policy_artifact, "static_k29334_policy_artifact": static_k29334_policy_artifact, + "static_mse_k29334_policy_artifact": static_mse_k29334_policy_artifact, } binding, dependency_hashes = _derive_stage_a_calibration_binding( frozen_identity_artifact=frozen_identity_artifact, @@ -2008,6 +2466,9 @@ def build_stage_a_calibration_binding_artifact( split_half_stability_artifact=split_half_stability_artifact, static_k27030_policy_artifact=static_k27030_policy_artifact, static_k29334_policy_artifact=static_k29334_policy_artifact, + comparator_score_artifact=comparator_score_artifact, + static_fisher_k29334_policy_artifact=static_fisher_k29334_policy_artifact, + static_mse_k29334_policy_artifact=static_mse_k29334_policy_artifact, ) evidence = { "artifact_revision": STAGE_A_BINDING_ARTIFACT_REVISION, @@ -2090,10 +2551,13 @@ def deserialize_stage_a_calibration_binding_artifact( dependency_names = frozenset( { "calibration_score_artifact", + "comparator_score_artifact", "frozen_identity_artifact", "split_half_stability_artifact", + "static_fisher_k29334_policy_artifact", "static_k27030_policy_artifact", "static_k29334_policy_artifact", + "static_mse_k29334_policy_artifact", } ) require_exact_fields( @@ -2108,16 +2572,40 @@ def deserialize_stage_a_calibration_binding_artifact( ) for name in sorted(dependency_names) } + recorded_dependency_hashes = require_mapping( + evidence["dependency_file_sha256"], + context="Stage-A binding dependency hashes", + ) + require_exact_fields( + recorded_dependency_hashes, + dependency_names, + context="Stage-A binding dependency hashes", + ) + normalized_dependency_hashes = { + name: require_sha256( + recorded_dependency_hashes[name], + context=f"Stage-A dependency {name} file SHA-256", + ) + for name in sorted(dependency_names) + } + embedded_dependency_hashes = { + name: sha256_bytes(dependencies[name]) for name in sorted(dependency_names) + } + if normalized_dependency_hashes != embedded_dependency_hashes: + raise ValueError("Stage-A calibration dependency bytes differ from their file hashes") binding, dependency_hashes = _derive_stage_a_calibration_binding( frozen_identity_artifact=dependencies["frozen_identity_artifact"], calibration_score_artifact=dependencies["calibration_score_artifact"], split_half_stability_artifact=dependencies["split_half_stability_artifact"], static_k27030_policy_artifact=dependencies["static_k27030_policy_artifact"], static_k29334_policy_artifact=dependencies["static_k29334_policy_artifact"], + comparator_score_artifact=dependencies["comparator_score_artifact"], + static_fisher_k29334_policy_artifact=(dependencies["static_fisher_k29334_policy_artifact"]), + static_mse_k29334_policy_artifact=dependencies["static_mse_k29334_policy_artifact"], ) if evidence["binding"] != binding: raise ValueError("Stage-A calibration binding fields drifted") - if evidence["dependency_file_sha256"] != dependency_hashes: + if normalized_dependency_hashes != dependency_hashes: raise ValueError("Stage-A calibration dependency hashes drifted") return StageACalibrationBindingArtifact( binding=binding, diff --git a/scripts/run_static_q468_calibration.py b/scripts/run_static_q468_calibration.py index b19a35a..e04ac50 100644 --- a/scripts/run_static_q468_calibration.py +++ b/scripts/run_static_q468_calibration.py @@ -22,6 +22,7 @@ import importlib import importlib.metadata import json +import math import os import platform import re @@ -47,21 +48,37 @@ CALIBRATION_API_PATH: Final = "src/recurquant/experiment013_calibration_api.py" SOURCE_VERIFIER_PATH: Final = "src/recurquant/experiment013_source.py" SOURCE_CAPTURE_MODULE: Final = "recurquant_experiment013_source_capture" +MODEL_STAGING_SOURCE_MODULE: Final = "recurquant_experiment013_source_for_model_staging" +MODEL_STAGING_RESOLVER_MODULE: Final = "recurquant_experiment013_resolver_for_model_staging" RUNNER_SOURCE_PATH: Final = "scripts/run_static_q468_calibration.py" IDENTITY_RESOLVER_SOURCE_PATH: Final = "scripts/resolve_static_q468_identity.py" CANONICAL_ADAPTER_SPEC: Final = "recurquant.experiment013_qwen35_adapter:create_adapter" CANONICAL_ADAPTER_MODULE: Final = "recurquant.experiment013_qwen35_adapter" CANONICAL_ADAPTER_PATH: Final = "src/recurquant/experiment013_qwen35_adapter.py" -RUNNER_REVISION: Final = "experiment-013-static-q468-calibration-runner-v1" +RUNNER_REVISION: Final = "experiment-013-static-q468-calibration-runner-v2" +FROZEN_IDENTITY_SCHEMA_VERSION: Final = 5 +FISHER_BOUNDARY_SCHEMA: Final = "recurquant.experiment013.fisher-boundary.v1" +FISHER_BOUNDARY_NAMESPACE: Final = b"recurquant.experiment013.fisher-boundary.v1\0" +FISHER_BOUNDARY_HORIZON: Final = 1 +FISHER_BOUNDARY_FIELDS: Final = { + "boundary_positions", + "fisher_boundary_sha256", + "horizon", + "input_positions", + "input_token_ids_sha256", + "schema", + "target_positions", + "target_token_ids_sha256", +} MODEL_FILE_MANIFEST_KIND: Final = "recurquant_experiment013_model_file_manifest" MODEL_FILE_MANIFEST_SCHEMA: Final = 1 MODEL_FILE_MANIFEST_DERIVATION: Final = "huggingface-hub-pinned-tree-lfs-v1" MODEL_FILE_SELECTION_PROFILE: Final = "qwen35-config-index-safetensors-v1" RUNTIME_MANIFEST_KIND: Final = "recurquant_experiment013_calibration_runtime_manifest" -RUNTIME_MANIFEST_SCHEMA: Final = 3 +RUNTIME_MANIFEST_SCHEMA: Final = 4 RUN_REPORT_KIND: Final = "recurquant_experiment013_calibration_run" -RUN_REPORT_SCHEMA: Final = 1 +RUN_REPORT_SCHEMA: Final = 2 QUERY_EMA_DECAY: Final = 2.0 ** (-1.0 / 32.0) QUERY_ENERGY_EPSILON: Final = 1.0e-6 @@ -76,13 +93,19 @@ ) SCORE_FILENAME: Final = "calibration-scores.json" +COMPARATOR_SCORE_FILENAME: Final = "comparator-scores.json" SPLIT_FILENAME: Final = "split-half-stability.json" K27030_FILENAME: Final = "static-k27030-policy.json" K29334_FILENAME: Final = "static-k29334-policy.json" +MSE_K29334_FILENAME: Final = "static-mse-k29334-policy.json" +FISHER_K29334_FILENAME: Final = "static-diagonal-fisher-h1-k29334-policy.json" Q48_FILENAME: Final = "static-q48-p14739-policy.json" BINDING_FILENAME: Final = "stage-a-calibration-binding.json" REPORT_FILENAME: Final = "calibration-run-report.json" COMPLETE_FILENAME: Final = "CALIBRATION_COMPLETE" +FISHER_SMOKE_REPORT_FILENAME: Final = "fisher-h1-smoke-report.json" +FISHER_SMOKE_COMPLETE_FILENAME: Final = "FISHER_H1_SMOKE_COMPLETE" +FISHER_SMOKE_COMPLETE_BYTES: Final = b"recurquant-experiment013-fisher-h1-smoke-complete-v1\n" PREPARED_RUNTIME_MANIFEST_FILENAME: Final = "calibration-runtime-manifest.json" PREPARED_RUNTIME_COMPLETE_FILENAME: Final = "RUNTIME_PREPARED" DEFAULT_PACKAGE_RUNTIME_ROOT_NAME: Final = "calibration-packages" @@ -92,9 +115,7 @@ _WINDOWS_REPARSE_POINT: Final = 0x400 _RUNTIME_ROOT_NAME_RE: Final = re.compile(r"[a-z][a-z0-9-]{0,63}") BASE_RUNTIME_ROOT_NAME: Final = "base-runtime" -_FORBIDDEN_RUNTIME_SUFFIXES: Final = frozenset( - {"._pth", ".egg-link", ".pth", ".pyc", ".pyo"} -) +_FORBIDDEN_RUNTIME_SUFFIXES: Final = frozenset({"._pth", ".egg-link", ".pth", ".pyc", ".pyo"}) _FORBIDDEN_RUNTIME_DIRECTORY_NAMES: Final = frozenset({"__pycache__"}) _FORBIDDEN_RUNTIME_FILENAMES: Final = frozenset( {"pyvenv.cfg", "sitecustomize.py", "usercustomize.py"} @@ -240,7 +261,7 @@ class BootstrapSource: def _bootstrap_identity_bindings(data: bytes) -> BootstrapIdentityBindings: - """Strictly extract only the v4 execution bindings using stdlib code. + """Strictly extract only the v5 execution bindings using stdlib code. Full semantic decoding remains the authenticated resolver's job. This minimal pass exists solely to authenticate the code that performs it. @@ -259,7 +280,7 @@ def _bootstrap_identity_bindings(data: bytes) -> BootstrapIdentityBindings: raise CalibrationRunError("frozen calibration identity bootstrap evidence is missing") if ( type(evidence.get("schema_version")) is not int - or evidence.get("schema_version") != 4 + or evidence.get("schema_version") != FROZEN_IDENTITY_SCHEMA_VERSION or evidence.get("status") != "frozen" or evidence.get("phase") != "calibration" or evidence.get("identity_only") is not True @@ -320,16 +341,26 @@ def _canonical_relative_path(value: object, *, context: str) -> str: if "\\" in value or "\0" in value or "\n" in value or "\r" in value: raise CalibrationRunError(f"{context} must be a single-line POSIX path") path = PurePosixPath(value) + parts = path.parts if ( - path.is_absolute() - or path.parts[0].endswith(":") - or any(part in {"", ".", ".."} for part in path.parts) + not parts + or path.is_absolute() + or parts[0].endswith(":") + or any(part in {"", ".", ".."} for part in parts) or path.as_posix() != value ): raise CalibrationRunError(f"{context} must be repository-relative") return value +def _canonical_base_sys_path_entry(value: object, *, context: str) -> str: + """Accept only the runtime-root sentinel beyond canonical relative paths.""" + + if isinstance(value, str) and value == ".": + return "." + return _canonical_relative_path(value, context=context) + + def _is_link_or_reparse(path: Path) -> bool: try: stat_result = path.lstat() @@ -372,6 +403,7 @@ def _bootstrap_source_manifest( manifest, { "canonical_manifest_sha256", + "git_executable", "object_format", "paths", "profile", @@ -387,6 +419,22 @@ def _bootstrap_source_manifest( ) if claimed != _source_manifest_canonical_sha256(manifest): raise CalibrationRunError("repository source manifest canonical SHA-256 drifted") + if ( + manifest["schema"] != "recurquant.experiment013.source-manifest.v2" + or manifest["profile"] != "experiment-013-static-q468-frozen-source-v2" + or manifest["object_format"] != "sha1" + ): + raise CalibrationRunError("repository source manifest profile drifted") + git_record = manifest["git_executable"] + if not isinstance(git_record, dict): + raise CalibrationRunError("repository source Git executable record is missing") + _exact_fields( + git_record, + {"sha256", "size_bytes"}, + context="repository source Git executable", + ) + _sha256(git_record["sha256"], context="repository source Git executable SHA-256") + _positive_int(git_record["size_bytes"], context="repository source Git executable size") source_commit = _git_revision( manifest["source_commit"], context="repository source manifest source commit", @@ -526,19 +574,36 @@ def _load_source_capture_module(repository_root: Path) -> ModuleType: def _sanitized_git_environment() -> dict[str, str]: - environment = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} + inherited = {key.upper(): (key, value) for key, value in os.environ.items()} + environment = { + inherited[name][0]: inherited[name][1] + for name in ("SYSTEMROOT", "WINDIR", "COMSPEC") + if name in inherited + } environment.update( { + "GIT_AUTHOR_NAME": "RecurQuant Experiment 013", + "GIT_AUTHOR_EMAIL": "experiment013@invalid", + "GIT_COMMITTER_NAME": "RecurQuant Experiment 013", + "GIT_COMMITTER_EMAIL": "experiment013@invalid", "GIT_CONFIG_GLOBAL": os.devnull, "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_SYSTEM": os.devnull, "GIT_OPTIONAL_LOCKS": "0", "GIT_TERMINAL_PROMPT": "0", + "LC_ALL": "C", + "LANG": "C", } ) return environment -def _assert_source_manifest_output_location(repository_root: Path, output: Path) -> Path: +def _assert_source_manifest_output_location( + repository_root: Path, + output: Path, + *, + git_executable: AuthenticatedGitExecutable, +) -> Path: """Allow source manifests only outside the repository or at an ignored path.""" try: @@ -557,7 +622,14 @@ def _assert_source_manifest_output_location(repository_root: Path, output: Path) if not relative.parts or relative.parts[0].casefold() == ".git": raise CalibrationRunError("source manifest output cannot be repository metadata") process = subprocess.run( - ["git", "check-ignore", "--quiet", "--no-index", "--", relative.as_posix()], + [ + str(git_executable.path), + "check-ignore", + "--quiet", + "--no-index", + "--", + relative.as_posix(), + ], cwd=root, check=False, capture_output=True, @@ -635,13 +707,27 @@ class FrozenCalibrationIdentity: @dataclass(frozen=True, slots=True) class CapturedSequence: - """Only the current sequence's frozen anchor tensors, all on CPU FP64.""" + """One causal pass worth of post-token and H=1 endpoint tensors.""" anchor_positions: tuple[int, ...] query_energy: Any q4_mse: Any q6_mse: Any q8_mse: Any + fisher_boundary_positions: tuple[int, ...] + fisher_q4_risk: Any + fisher_q6_risk: Any + fisher_q8_risk: Any + fisher_target_nlls: Any + + +@dataclass(frozen=True, slots=True) +class ReducedSequenceScores: + """Candidate, unweighted-MSE, and H=1 Fisher scores for one identity row.""" + + candidate: Any + mse: Any + fisher: Any @dataclass(frozen=True, slots=True) @@ -664,6 +750,15 @@ class ModelFileManifest: file_sha256: str +@dataclass(frozen=True, slots=True) +class ModelStagingAuthorization: + identity: FrozenCalibrationIdentity + model_manifest: ModelFileManifest + frozen_identity_file_sha256: str + identity_commit: str + source_commit: str + + @dataclass(frozen=True, slots=True) class RuntimeFileRecord: path: str @@ -712,6 +807,14 @@ class RuntimeRequirement: version: str +@dataclass(frozen=True, slots=True) +class AuthenticatedGitExecutable: + path: Path + absolute_path_sha256: str + sha256: str + size_bytes: int + + @dataclass(frozen=True, slots=True) class CalibrationRuntimeManifest: python_implementation: str @@ -731,6 +834,9 @@ class CalibrationRuntimeManifest: interpreter_relative_path: str interpreter_size_bytes: int interpreter_sha256: str + git_executable_absolute_path_sha256: str + git_executable_sha256: str + git_executable_size_bytes: int runtime_trees: tuple[RuntimeTreeRecord, ...] distributions: tuple[RuntimeDistributionRecord, ...] file_sha256: str @@ -743,6 +849,9 @@ class AuthenticatedRuntime: python_version: str python_cache_tag: str interpreter_sha256: str + git_executable_absolute_path_sha256: str + git_executable_sha256: str + git_executable_size_bytes: int machine_name: str base_runtime_file_count: int package_root_count: int @@ -757,6 +866,7 @@ class SealedRuntimeContext: base_runtime_root: Path package_roots: Mapping[str, Path] package_import_paths: Mapping[str, str] + git_executable_path: Path pycache_prefix: Path @@ -765,14 +875,15 @@ class CalibrationArtifacts: """In-memory pass artifacts; no publication occurs until all are built.""" score: bytes + comparator_score: bytes split_half: bytes static_k27030: bytes static_k29334: bytes + static_mse_k29334: bytes + static_fisher_k29334: bytes static_q48: bytes stage_a_binding: bytes stability: Mapping[str, object] - calibration_scores_sha256: str - sequence_score_manifest_sha256: str @dataclass(frozen=True, slots=True) @@ -797,6 +908,9 @@ class CalibrationRunConfig: expected_runtime_manifest_sha256: str output_dir: Path require_cuda: bool = True + fisher_h1_smoke: bool = False + prior_fisher_h1_smoke_report_bytes: bytes | None = None + prior_fisher_h1_smoke_complete_bytes: bytes | None = None class CalibrationBackend(Protocol): @@ -823,6 +937,7 @@ def finalize( SourceVerifier = Callable[[Mapping[str, object], Path], tuple[dict[str, object], str]] AdapterValidator = Callable[[Any], None] DistortionFunction = Callable[[Any, Geometry], tuple[Any, Any, Any]] +FisherDistortionFunction = Callable[[Any, Any, Geometry], tuple[Any, Any, Any]] ModelAuthenticator = Callable[[Path, ModelFileManifest], Any] RuntimeAuthenticator = Callable[[CalibrationRuntimeManifest], AuthenticatedRuntime] @@ -831,13 +946,23 @@ def finalize( class RunnerServices: backend: CalibrationBackend calibration_api: ModuleType + identity_resolver: Any | None verify_repository_source: SourceVerifier validate_adapter: AdapterValidator distortion_function: DistortionFunction + fisher_distortion_function: FisherDistortionFunction authenticate_model_files: ModelAuthenticator authenticate_runtime: RuntimeAuthenticator +def _selected_model_tree_path(path: str) -> bool: + if "/" in path: + return False + return path in {"config.json", "model.safetensors.index.json"} or bool( + _WEIGHT_FILE_RE.fullmatch(path) + ) + + def parse_model_file_manifest(data: bytes) -> ModelFileManifest: """Strictly decode a canonical, immutable local-model file manifest.""" @@ -925,8 +1050,11 @@ def parse_model_file_manifest(data: bytes) -> ModelFileManifest: raise ValueError("model LFS identity must equal the local content contract") else: raise ValueError("model LFS SHA-256 and size must either both be null or both be set") - if _WEIGHT_FILE_RE.search(name) and lfs_sha256 is None: + is_weight = _WEIGHT_FILE_RE.fullmatch(name) is not None + if is_weight and lfs_sha256 is None: raise ValueError("safetensors weights require a pinned Hub LFS identity") + if not is_weight and lfs_sha256 is not None: + raise ValueError("model config and index files must be ordinary Git blobs") files.append( ModelFileRecord( name=name, @@ -940,9 +1068,15 @@ def parse_model_file_manifest(data: bytes) -> ModelFileManifest: names = [item.name for item in files] if names != sorted(names) or len(names) != len(set(names)): raise ValueError("model file manifest names must be unique and sorted") + if len({name.casefold() for name in names}) != len(names): + raise ValueError("model file manifest names contain a case-insensitive collision") + if any(not _selected_model_tree_path(name) for name in names): + raise ValueError("model file manifest contains a file outside its selection profile") if "config.json" not in names: raise ValueError("model file manifest must authenticate config.json") - if not any(_WEIGHT_FILE_RE.search(name) for name in names): + if "model.safetensors.index.json" not in names: + raise ValueError("model file manifest must authenticate model.safetensors.index.json") + if not any(_WEIGHT_FILE_RE.fullmatch(name) for name in names): raise ValueError( "model file manifest must authenticate at least one safetensors weight file" ) @@ -977,14 +1111,6 @@ def _hub_value(value: object, name: str, *, default: object = None) -> object: return getattr(value, name, default) -def _selected_model_tree_path(path: str) -> bool: - if "/" in path: - return False - return path in {"config.json", "model.safetensors.index.json"} or bool( - _WEIGHT_FILE_RE.fullmatch(path) - ) - - def capture_model_file_manifest_from_hub( model_id: str, revision: str, @@ -993,7 +1119,7 @@ def capture_model_file_manifest_from_hub( api: object | None = None, tree_entries: Sequence[object] | None = None, resolved_revision: str | None = None, - token: str | bool | None = None, + token: bool = False, ) -> bytes: """Build the local-file contract using only pinned Hub tree/LFS metadata. @@ -1004,6 +1130,8 @@ def capture_model_file_manifest_from_hub( if not isinstance(model_id, str) or not model_id or model_id != model_id.strip(): raise ValueError("model_id must be a non-empty canonical string") + if token is not False: + raise CalibrationRunError("Experiment 013 Hub metadata access must be unauthenticated") pinned_revision = _git_revision(revision, context="model Hub revision") if ( not isinstance(transformers_version, str) @@ -1014,12 +1142,12 @@ def capture_model_file_manifest_from_hub( if api is None: from huggingface_hub import HfApi - api = HfApi() + api = HfApi(token=False, endpoint="https://huggingface.co") info = api.model_info( # type: ignore[attr-defined] model_id, revision=pinned_revision, files_metadata=False, - token=token, + token=False, ) resolved = _git_revision( getattr(info, "sha", None), @@ -1034,7 +1162,7 @@ def capture_model_file_manifest_from_hub( expand=False, revision=pinned_revision, repo_type="model", - token=token, + token=False, ) ) else: @@ -1147,6 +1275,55 @@ def _stream_file_sha256(path: Path) -> tuple[str, int]: return digest.hexdigest(), size +def _absolute_path_sha256(path: Path) -> str: + normalized = os.path.normcase(str(path.resolve(strict=True))) + return sha256_bytes(normalized.encode("utf-8")) + + +def _assert_absolute_path_components_not_links(path: Path, *, context: str) -> None: + current = Path(path.anchor) + for part in path.parts[1:]: + current /= part + if _is_link_or_reparse(current): + raise CalibrationRunError(f"{context} traverses a link or reparse point") + + +def _authenticate_git_executable( + executable: str | os.PathLike[str] | None, +) -> AuthenticatedGitExecutable: + selected: str | os.PathLike[str] + if executable is None: + discovered = shutil.which("git") + if discovered is None: + raise CalibrationRunError("Git executable is unavailable") + selected = discovered + else: + selected = executable + try: + resolved = Path(selected).resolve(strict=True) + except (OSError, TypeError, ValueError) as exc: + raise CalibrationRunError("Git executable is unavailable") from exc + if resolved.name.casefold() == "git.exe" and resolved.parent.name.casefold() == "cmd": + try: + resolved = (resolved.parent.parent / "mingw64" / "bin" / "git.exe").resolve(strict=True) + except OSError as exc: + raise CalibrationRunError( + "Git-for-Windows cmd shim has no canonical mingw64 executable" + ) from exc + _assert_absolute_path_components_not_links(resolved, context="Git executable") + if not resolved.is_file() or _is_link_or_reparse(resolved): + raise CalibrationRunError("Git executable must be a regular non-link file") + digest, size = _stream_file_sha256(resolved) + if size <= 0: + raise CalibrationRunError("Git executable must be non-empty") + return AuthenticatedGitExecutable( + path=resolved, + absolute_path_sha256=_absolute_path_sha256(resolved), + sha256=digest, + size_bytes=size, + ) + + def _stream_model_file_identity(path: Path) -> tuple[str, str, int]: before = path.stat() size = before.st_size @@ -1171,13 +1348,11 @@ def _stream_model_file_identity(path: Path) -> tuple[str, str, int]: return sha256.hexdigest(), git_blob.hexdigest(), size -def authenticate_local_model_files( +def _verify_exact_local_model_tree( model_root: Path, manifest: ModelFileManifest, - *, - calibration_api: ModuleType, -) -> Any: - """Hash every exact local model file immediately before model loading.""" +) -> Path: + """Authenticate one exact, regular, link-free local model tree.""" root = Path(os.path.abspath(model_root)) if not root.is_dir(): @@ -1196,6 +1371,8 @@ def authenticate_local_model_files( f"local model path is not a regular file/directory: {relative}" ) actual_names.sort() + if len({name.casefold() for name in actual_names}) != len(actual_names): + raise CalibrationRunError("local model file set has a case-insensitive collision") expected_names = [item.name for item in manifest.files] if actual_names != expected_names: raise CalibrationRunError( @@ -1213,6 +1390,18 @@ def authenticate_local_model_files( ) if size != item.size_bytes or not content_matches: raise CalibrationRunError(f"local model file authentication failed: {item.name}") + return root + + +def authenticate_local_model_files( + model_root: Path, + manifest: ModelFileManifest, + *, + calibration_api: ModuleType, +) -> Any: + """Hash every exact local model file immediately before model loading.""" + + root = _verify_exact_local_model_tree(model_root, manifest) identities = tuple( calibration_api.ModelFileIdentity( name=item.name, @@ -1235,6 +1424,557 @@ def authenticate_local_model_files( ) +def _read_stable_regular_bytes(path: Path, *, context: str) -> bytes: + candidate = Path(os.path.abspath(path)) + try: + before = candidate.lstat() + except OSError as exc: + raise CalibrationRunError(f"{context} is unavailable") from exc + if _is_link_or_reparse(candidate) or not stat.S_ISREG(before.st_mode): + raise CalibrationRunError(f"{context} must be a regular non-link file") + try: + data = candidate.read_bytes() + after = candidate.lstat() + except OSError as exc: + raise CalibrationRunError(f"cannot read {context}") from exc + if ( + _is_link_or_reparse(candidate) + or not stat.S_ISREG(after.st_mode) + or len(data) != after.st_size + or before.st_size != after.st_size + or before.st_mtime_ns != after.st_mtime_ns + or before.st_dev != after.st_dev + or before.st_ino != after.st_ino + ): + raise CalibrationRunError(f"{context} changed while it was read") + return data + + +def _git_blob_oid_bytes(data: bytes) -> str: + digest = hashlib.sha1(usedforsecurity=False) + digest.update(f"blob {len(data)}\0".encode("ascii")) + digest.update(data) + return digest.hexdigest() + + +def _run_model_staging_git( + git_executable: AuthenticatedGitExecutable, + repository_root: Path, + *arguments: str, +) -> subprocess.CompletedProcess[bytes]: + return subprocess.run( + [str(git_executable.path), "-C", str(repository_root), *arguments], + check=False, + capture_output=True, + timeout=30, + env=_sanitized_git_environment(), + ) + + +def _one_nul_git_record(process: subprocess.CompletedProcess[bytes], *, context: str) -> bytes: + if process.returncode != 0: + raise CalibrationRunError(f"cannot authenticate {context}") + records = process.stdout.split(b"\0") + if records[-1:] != [b""] or len(records) != 2 or not records[0]: + raise CalibrationRunError(f"{context} is not exactly one Git record") + return records[0] + + +def _verify_committed_frozen_identity( + git_executable: AuthenticatedGitExecutable, + repository_root: Path, + identity_path: Path, + identity_bytes: bytes, + *, + identity_commit: str, +) -> str: + """Require the promoted identity to be the exact clean blob at current HEAD.""" + + root = Path(os.path.abspath(repository_root)).resolve(strict=True) + declared = Path(os.path.abspath(identity_path)) + try: + relative_path = declared.relative_to(root).as_posix() + except ValueError as exc: + raise CalibrationRunError( + "frozen identity must be committed inside the repository" + ) from exc + if not relative_path or "\t" in relative_path: + raise CalibrationRunError("frozen identity repository path is not canonical") + authenticated_path = _assert_no_link_components(root, PurePosixPath(relative_path)) + if authenticated_path.read_bytes() != identity_bytes: + raise CalibrationRunError("committed frozen identity changed after authorization") + + expected_commit = _git_revision(identity_commit, context="identity commit") + head = _run_model_staging_git( + git_executable, + root, + "rev-parse", + "--verify", + "HEAD^{commit}", + ) + if head.returncode != 0: + raise CalibrationRunError("cannot resolve current HEAD for frozen identity provenance") + actual_head = _git_revision( + head.stdout.decode("ascii", errors="strict").strip(), + context="current identity commit", + ) + if actual_head != expected_commit: + raise CalibrationRunError("current HEAD differs from the explicit identity commit") + + tree_record = _one_nul_git_record( + _run_model_staging_git( + git_executable, + root, + "ls-tree", + "-z", + "--full-tree", + expected_commit, + "--", + relative_path, + ), + context="frozen identity HEAD entry", + ) + try: + tree_header, tree_path = tree_record.split(b"\t", 1) + mode, object_type, head_oid = tree_header.decode("ascii").split(" ", 2) + decoded_tree_path = tree_path.decode("utf-8", errors="strict") + except (UnicodeError, ValueError) as exc: + raise CalibrationRunError("frozen identity HEAD entry is malformed") from exc + if mode != "100644" or object_type != "blob" or decoded_tree_path != relative_path: + raise CalibrationRunError("frozen identity HEAD entry has the wrong mode, type, or path") + + index_record = _one_nul_git_record( + _run_model_staging_git( + git_executable, + root, + "ls-files", + "--stage", + "-z", + "--", + relative_path, + ), + context="frozen identity index entry", + ) + try: + index_header, index_path = index_record.split(b"\t", 1) + index_mode, index_oid, stage = index_header.decode("ascii").split(" ", 2) + decoded_index_path = index_path.decode("utf-8", errors="strict") + except (UnicodeError, ValueError) as exc: + raise CalibrationRunError("frozen identity index entry is malformed") from exc + expected_oid = _git_blob_oid_bytes(identity_bytes) + if ( + index_mode != "100644" + or stage != "0" + or decoded_index_path != relative_path + or head_oid != expected_oid + or index_oid != expected_oid + ): + raise CalibrationRunError("frozen identity HEAD, index, and worktree bytes differ") + return actual_head + + +def _authenticate_model_staging_authorization( + *, + git_executable: AuthenticatedGitExecutable, + frozen_identity_path: Path, + expected_frozen_identity_sha256: str, + identity_commit: str, + repository_root: Path, + repository_source_manifest_path: Path, + source_commit: str, + model_file_manifest_path: Path, + expected_model_file_manifest_sha256: str, +) -> ModelStagingAuthorization: + """Authenticate promotion, committed provenance, source, and model metadata.""" + + expected_identity_sha256 = _sha256( + expected_frozen_identity_sha256, + context="expected frozen identity SHA-256", + ) + identity_bytes = _read_stable_regular_bytes( + frozen_identity_path, + context="frozen identity", + ) + if sha256_bytes(identity_bytes) != expected_identity_sha256: + raise CalibrationRunError("frozen identity bytes differ from the explicit SHA-256") + bindings = _bootstrap_identity_bindings(identity_bytes) + + source_manifest_bytes = _read_stable_regular_bytes( + repository_source_manifest_path, + context="repository source manifest", + ) + if sha256_bytes(source_manifest_bytes) != bindings.repository_source_manifest_file_sha256: + raise CalibrationRunError("repository source manifest differs from the frozen identity") + bootstrap_source = _bootstrap_source_manifest( + source_manifest_bytes, + repository_root=repository_root, + require_adapter=False, + ) + requested_source_commit = _git_revision(source_commit, context="requested source commit") + if requested_source_commit != bootstrap_source.source_commit: + raise CalibrationRunError("requested source commit differs from the identity-bound H0") + + source_module = _load_exact_source_module( + MODEL_STAGING_SOURCE_MODULE, + SOURCE_VERIFIER_PATH, + repository_root=repository_root, + entry=bootstrap_source.entries[SOURCE_VERIFIER_PATH], + ) + resolver_module: ModuleType | None = None + try: + verified_source = source_module.verify_experiment013_source_manifest( + bootstrap_source.manifest, + repo_root=repository_root, + git_executable=git_executable.path, + ) + if verified_source != bootstrap_source.manifest: + raise CalibrationRunError("source verifier returned different model-staging evidence") + if verified_source.get("source_commit") != requested_source_commit: + raise CalibrationRunError( + "verified source-manifest commit differs from requested frozen source commit" + ) + resolver_module = _load_exact_source_module( + MODEL_STAGING_RESOLVER_MODULE, + IDENTITY_RESOLVER_SOURCE_PATH, + repository_root=repository_root, + entry=bootstrap_source.entries[IDENTITY_RESOLVER_SOURCE_PATH], + ) + identity = _identity_view_from_resolver( + identity_bytes, + resolver_module, + expected_file_sha256=expected_identity_sha256, + ) + finally: + if resolver_module is not None: + sys.modules.pop(MODEL_STAGING_RESOLVER_MODULE, None) + sys.modules.pop(MODEL_STAGING_SOURCE_MODULE, None) + if ( + identity.repository_source_manifest_file_sha256 + != bindings.repository_source_manifest_file_sha256 + or identity.runtime_manifest_file_sha256 != bindings.runtime_manifest_file_sha256 + or identity.model_file_manifest_file_sha256 != bindings.model_file_manifest_file_sha256 + or identity.parquet_materialization_manifest_file_sha256 + != bindings.parquet_materialization_manifest_file_sha256 + ): + raise CalibrationRunError("full frozen identity differs from its bootstrap bindings") + + committed_at = _verify_committed_frozen_identity( + git_executable, + repository_root, + frozen_identity_path, + identity_bytes, + identity_commit=identity_commit, + ) + model_manifest_bytes = _read_stable_regular_bytes( + model_file_manifest_path, + context="model file manifest", + ) + expected_model_sha256 = _sha256( + expected_model_file_manifest_sha256, + context="expected model file manifest SHA-256", + ) + actual_model_sha256 = sha256_bytes(model_manifest_bytes) + if ( + actual_model_sha256 != expected_model_sha256 + or actual_model_sha256 != bindings.model_file_manifest_file_sha256 + ): + raise CalibrationRunError( + "model file manifest differs from the frozen identity/CLI binding" + ) + model_manifest = parse_model_file_manifest(model_manifest_bytes) + _model_contract_matches(identity, model_manifest) + return ModelStagingAuthorization( + identity=identity, + model_manifest=model_manifest, + frozen_identity_file_sha256=expected_identity_sha256, + identity_commit=committed_at, + source_commit=requested_source_commit, + ) + + +def _path_is_within(path: Path, root: Path) -> bool: + try: + path.relative_to(root) + except ValueError: + return False + return True + + +def _ensure_regular_directory(path: Path, *, context: str) -> Path: + absolute = Path(os.path.abspath(path)) + if os.path.lexists(absolute): + if _is_link_or_reparse(absolute) or not absolute.is_dir(): + raise CalibrationRunError(f"{context} must be a regular non-link directory") + else: + absolute.mkdir(parents=True, exist_ok=False) + candidate = Path(absolute.anchor) + for part in absolute.parts[1:]: + candidate /= part + if _is_link_or_reparse(candidate) or not candidate.is_dir(): + raise CalibrationRunError(f"{context} traverses a link or non-directory") + return absolute.resolve(strict=True) + + +def _validate_model_staging_roots( + *, + repository_root: Path, + hub_cache_root: Path, + output_root: Path, +) -> tuple[Path, Path]: + repository = Path(os.path.abspath(repository_root)).resolve(strict=True) + destination = Path(os.path.abspath(output_root)) + if not destination.name: + raise CalibrationRunError("model output root cannot be a filesystem root") + if os.path.lexists(destination): + raise FileExistsError(f"refusing to overwrite staged model root: {destination}") + parent = _ensure_regular_directory(destination.parent, context="model output parent") + cache = _ensure_regular_directory(hub_cache_root, context="Hub cache root") + resolved_destination = parent / destination.name + roots = (("model output root", resolved_destination), ("Hub cache root", cache)) + for context, candidate in roots: + if _path_is_within(candidate, repository): + raise CalibrationRunError(f"{context} must be outside the repository") + if _path_is_within(resolved_destination, cache) or _path_is_within(cache, resolved_destination): + raise CalibrationRunError("Hub cache and staged model roots must not be nested") + return cache, resolved_destination + + +def _assert_regular_cache_payload(cache_root: Path, returned_path: object) -> Path: + if not isinstance(returned_path, (str, os.PathLike)): + raise CalibrationRunError("Hub downloader did not return a filesystem path") + cache = cache_root.resolve(strict=True) + returned = Path(os.path.abspath(returned_path)) + if not _path_is_within(returned, cache): + raise CalibrationRunError("Hub downloader returned a path outside the explicit cache") + candidate = cache + relative = returned.relative_to(cache) + if not relative.parts: + raise CalibrationRunError("Hub downloader returned the cache root instead of a file") + for part in relative.parts[:-1]: + candidate /= part + if _is_link_or_reparse(candidate) or not candidate.is_dir(): + raise CalibrationRunError("Hub cache payload path traverses a link or non-directory") + if not os.path.lexists(returned): + raise CalibrationRunError("Hub downloader returned an unavailable cache path") + try: + payload = returned.resolve(strict=True) + payload.relative_to(cache) + except (OSError, ValueError) as exc: + raise CalibrationRunError("Hub cache pointer escapes the explicit cache") from exc + payload_relative = payload.relative_to(cache) + candidate = cache + for part in payload_relative.parts: + candidate /= part + if _is_link_or_reparse(candidate): + raise CalibrationRunError( + "resolved Hub cache payload traverses a link or reparse point" + ) + try: + status = payload.stat() + except OSError as exc: + raise CalibrationRunError("resolved Hub cache payload is unavailable") from exc + if not stat.S_ISREG(status.st_mode): + raise CalibrationRunError("resolved Hub cache payload is not a regular file") + return payload + + +def _copy_authenticated_model_payload( + source: Path, + destination: Path, + record: ModelFileRecord, +) -> None: + try: + before = source.stat() + except OSError as exc: + raise CalibrationRunError(f"cannot stat cached model file: {record.name}") from exc + if not stat.S_ISREG(before.st_mode) or _is_link_or_reparse(source): + raise CalibrationRunError(f"cached model payload is not a regular file: {record.name}") + if before.st_size != record.size_bytes: + raise CalibrationRunError(f"cached model file size differs from manifest: {record.name}") + destination.parent.mkdir(parents=True, exist_ok=True) + sha256 = hashlib.sha256() + git_blob = hashlib.sha1(usedforsecurity=False) + git_blob.update(f"blob {record.size_bytes}\0".encode("ascii")) + copied = 0 + try: + with source.open("rb") as reader, destination.open("xb") as writer: + for chunk in iter(lambda: reader.read(1024 * 1024), b""): + sha256.update(chunk) + git_blob.update(chunk) + writer.write(chunk) + copied += len(chunk) + writer.flush() + os.fsync(writer.fileno()) + after = source.stat() + except OSError as exc: + raise CalibrationRunError(f"cannot stage cached model file: {record.name}") from exc + if ( + copied != record.size_bytes + or before.st_size != after.st_size + or before.st_mtime_ns != after.st_mtime_ns + or before.st_dev != after.st_dev + or before.st_ino != after.st_ino + ): + raise CalibrationRunError(f"cached model file changed while staging: {record.name}") + content_matches = ( + sha256.hexdigest() == record.lfs_sha256 + if record.lfs_sha256 is not None + else git_blob.hexdigest() == record.git_blob_oid + ) + if not content_matches: + raise CalibrationRunError(f"cached model file authentication failed: {record.name}") + + +def _atomic_rename_directory_no_overwrite(source: Path, destination: Path) -> None: + """Publish a sibling directory atomically without replacement races.""" + + if source.parent != destination.parent: + raise CalibrationRunError("atomic model publication requires sibling directories") + if os.path.lexists(destination): + raise FileExistsError(f"refusing to overwrite staged model root: {destination}") + if os.name == "nt": + try: + source.rename(destination) + except OSError as exc: + if os.path.lexists(destination): + raise FileExistsError( + f"refusing to overwrite staged model root: {destination}" + ) from exc + raise + return + if sys.platform.startswith("linux"): + import ctypes + import errno + + library = ctypes.CDLL(None, use_errno=True) + renameat2 = getattr(library, "renameat2", None) + if renameat2 is None: + raise CalibrationRunError("atomic no-replace directory publication is unavailable") + renameat2.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_uint, + ] + renameat2.restype = ctypes.c_int + result = renameat2( + -100, + os.fsencode(source), + -100, + os.fsencode(destination), + 1, + ) + if result == 0: + return + error_number = ctypes.get_errno() + if error_number in {errno.EEXIST, errno.ENOTEMPTY}: + raise FileExistsError(f"refusing to overwrite staged model root: {destination}") + raise OSError(error_number, os.strerror(error_number), str(destination)) + raise CalibrationRunError("atomic no-replace directory publication is unsupported") + + +def stage_identity_bound_model( + *, + git_executable_path: Path | None = None, + frozen_identity_path: Path, + expected_frozen_identity_sha256: str, + identity_commit: str, + repository_root: Path, + repository_source_manifest_path: Path, + source_commit: str, + model_file_manifest_path: Path, + expected_model_file_manifest_sha256: str, + hub_cache_root: Path, + output_root: Path, + local_files_only: bool = False, + downloader: Callable[..., object] | None = None, +) -> dict[str, object]: + """Stage only identity-bound model files after committed promotion.""" + + if type(local_files_only) is not bool: + raise TypeError("local_files_only must be bool") + git_executable = _authenticate_git_executable(git_executable_path) + authorization = _authenticate_model_staging_authorization( + git_executable=git_executable, + frozen_identity_path=frozen_identity_path, + expected_frozen_identity_sha256=expected_frozen_identity_sha256, + identity_commit=identity_commit, + repository_root=repository_root, + repository_source_manifest_path=repository_source_manifest_path, + source_commit=source_commit, + model_file_manifest_path=model_file_manifest_path, + expected_model_file_manifest_sha256=expected_model_file_manifest_sha256, + ) + cache, destination = _validate_model_staging_roots( + repository_root=repository_root, + hub_cache_root=hub_cache_root, + output_root=output_root, + ) + if downloader is None: + from huggingface_hub import hf_hub_download + + downloader = hf_hub_download + + prefix = f".{destination.name}.staging-" + staging = Path(tempfile.mkdtemp(prefix=prefix, dir=destination.parent)) + owned_staging = True + try: + for record in authorization.model_manifest.files: + returned = downloader( + repo_id=authorization.model_manifest.model_id, + filename=record.name, + repo_type="model", + revision=authorization.model_manifest.revision, + cache_dir=cache, + local_files_only=local_files_only, + token=False, + endpoint="https://huggingface.co", + ) + source = _assert_regular_cache_payload(cache, returned) + _copy_authenticated_model_payload(source, staging / record.name, record) + _verify_exact_local_model_tree(staging, authorization.model_manifest) + + repeated = _authenticate_model_staging_authorization( + git_executable=git_executable, + frozen_identity_path=frozen_identity_path, + expected_frozen_identity_sha256=expected_frozen_identity_sha256, + identity_commit=identity_commit, + repository_root=repository_root, + repository_source_manifest_path=repository_source_manifest_path, + source_commit=source_commit, + model_file_manifest_path=model_file_manifest_path, + expected_model_file_manifest_sha256=expected_model_file_manifest_sha256, + ) + if repeated != authorization: + raise CalibrationRunError("model-staging authorization changed before publication") + _verify_exact_local_model_tree(staging, repeated.model_manifest) + _atomic_rename_directory_no_overwrite(staging, destination) + owned_staging = False + finally: + if owned_staging: + try: + staging.relative_to(destination.parent) + except ValueError as exc: + raise RuntimeError("owned model staging directory escaped its parent") from exc + if not staging.name.startswith(prefix): + raise RuntimeError("owned model staging directory name drifted") + shutil.rmtree(staging, ignore_errors=False) + _verify_exact_local_model_tree(destination, authorization.model_manifest) + return { + "file_count": len(authorization.model_manifest.files), + "frozen_identity_file_sha256": authorization.frozen_identity_file_sha256, + "identity_commit": authorization.identity_commit, + "model_id": authorization.model_manifest.model_id, + "model_manifest_file_sha256": authorization.model_manifest.file_sha256, + "model_root": str(destination), + "revision": authorization.model_manifest.revision, + "source_commit": authorization.source_commit, + "status": "staged_authenticated_model", + "total_size_bytes": sum(item.size_bytes for item in authorization.model_manifest.files), + } + + def _normalized_distribution_name(value: object) -> str: if not isinstance(value, str) or not value.strip(): raise CalibrationRunError("installed distribution has no canonical name") @@ -1311,9 +2051,7 @@ def _normalized_package_import_paths( if not isinstance(package_import_paths, Mapping) or set(package_import_paths) != set( package_roots ): - raise CalibrationRunError( - "package import paths must exactly match the named package roots" - ) + raise CalibrationRunError("package import paths must exactly match the named package roots") normalized: dict[str, str] = {} for name in sorted(package_roots): relative = _canonical_relative_path( @@ -1346,14 +2084,15 @@ def _capture_base_sys_path( ) -> tuple[str, ...]: if supplied is not None: values = tuple( - _canonical_relative_path(item, context="base sys.path entry") for item in supplied + _canonical_base_sys_path_entry(item, context="base sys.path entry") for item in supplied ) else: root = base_runtime_root.resolve(strict=True) - if Path(sys.prefix).resolve(strict=True) != root or Path(sys.base_prefix).resolve( - strict=True - ) != root: - raise CalibrationRunError( + if ( + Path(sys.prefix).resolve(strict=True) != root + or Path(sys.base_prefix).resolve(strict=True) != root + ): + raise CalibrationRunError( "runtime capture must run from the staged base interpreter with no virtualenv" ) captured: list[str] = [] @@ -1366,9 +2105,7 @@ def _capture_base_sys_path( raise CalibrationRunError( "isolated base sys.path escapes the staged base runtime" ) from exc - captured.append( - _canonical_relative_path(relative, context="base sys.path entry") - ) + captured.append(_canonical_base_sys_path_entry(relative, context="base sys.path entry")) values = tuple(captured) if not values or len(set(item.casefold() for item in values)) != len(values): raise CalibrationRunError("base sys.path entries must be non-empty and unique") @@ -1425,9 +2162,7 @@ def _runtime_tree_files(root: Path, *, kind: str) -> tuple[RuntimeFileRecord, .. try: status = entry.stat(follow_symlinks=False) except OSError as exc: - raise CalibrationRunError( - f"runtime tree path is unavailable: {relative}" - ) from exc + raise CalibrationRunError(f"runtime tree path is unavailable: {relative}") from exc if entry.is_symlink() or ( getattr(status, "st_file_attributes", 0) & _WINDOWS_REPARSE_POINT ): @@ -1492,9 +2227,7 @@ def _record_paths(distribution: Any, *, name: str) -> tuple[str, ...]: try: for index, row in enumerate(csv.reader(StringIO(record_text, newline=""))): if len(row) != 3: - raise CalibrationRunError( - f"distribution {name} RECORD row {index} is malformed" - ) + raise CalibrationRunError(f"distribution {name} RECORD row {index} is malformed") parsed.append( _raw_record_path( row[0], @@ -1563,9 +2296,7 @@ def _distribution_record( ) rendered_paths.append(matched_path.relative_to(package_roots[matched_root]).as_posix()) assert selected_root is not None - import_root = package_roots[selected_root] / PurePosixPath( - package_import_paths[selected_root] - ) + import_root = package_roots[selected_root] / PurePosixPath(package_import_paths[selected_root]) try: import_root.resolve(strict=True).relative_to(package_roots[selected_root]) except (OSError, ValueError) as exc: @@ -1621,6 +2352,7 @@ def _installed_distribution_map( def capture_calibration_runtime_manifest( *, + git_executable_path: Path | None = None, base_runtime_root: Path, package_roots: Mapping[str, Path], package_import_paths: Mapping[str, str], @@ -1637,6 +2369,7 @@ def capture_calibration_runtime_manifest( accept them. """ + git_executable = _authenticate_git_executable(git_executable_path) roots = _runtime_root_map(base_runtime_root, package_roots) packages = {name: roots[name] for name in sorted(roots) if name != BASE_RUNTIME_ROOT_NAME} import_paths = _normalized_package_import_paths(packages, package_import_paths) @@ -1769,6 +2502,11 @@ def capture_calibration_runtime_manifest( "sha256": interpreter.sha256, "size_bytes": interpreter.size_bytes, }, + "git_executable": { + "absolute_path_sha256": git_executable.absolute_path_sha256, + "sha256": git_executable.sha256, + "size_bytes": git_executable.size_bytes, + }, "launch_policy": dict(SEALED_LAUNCH_POLICY), "machine": dict( zip( @@ -1777,9 +2515,7 @@ def capture_calibration_runtime_manifest( strict=True, ) ), - "package_roots": [ - {"import_path": import_paths[name], "name": name} for name in packages - ], + "package_roots": [{"import_path": import_paths[name], "name": name} for name in packages], "python": { "abi_flags": python_identity[3], "cache_tag": python_identity[2], @@ -1840,9 +2576,7 @@ def _parse_runtime_tree(raw_tree: object, *, index: int) -> RuntimeTreeRecord: size_bytes=_nonnegative_int( raw_file["size_bytes"], context=f"runtime tree {name} file size" ), - sha256=_sha256( - raw_file["sha256"], context=f"runtime tree {name} file SHA-256" - ), + sha256=_sha256(raw_file["sha256"], context=f"runtime tree {name} file SHA-256"), ) ) if [item.path for item in files] != sorted(item.path for item in files) or len( @@ -1861,6 +2595,7 @@ def parse_calibration_runtime_manifest(data: bytes) -> CalibrationRuntimeManifes "base_runtime_root", "base_sys_path", "distributions", + "git_executable", "interpreter", "launch_policy", "machine", @@ -1885,6 +2620,27 @@ def parse_calibration_runtime_manifest(data: bytes) -> CalibrationRuntimeManifes context="calibration runtime launch policy", ) + raw_git_executable = root["git_executable"] + if not isinstance(raw_git_executable, dict): + raise ValueError("calibration runtime Git executable record must be an object") + _exact_fields( + raw_git_executable, + {"absolute_path_sha256", "sha256", "size_bytes"}, + context="calibration runtime Git executable", + ) + git_absolute_path_sha256 = _sha256( + raw_git_executable["absolute_path_sha256"], + context="runtime Git executable absolute-path SHA-256", + ) + git_sha256 = _sha256( + raw_git_executable["sha256"], + context="runtime Git executable SHA-256", + ) + git_size_bytes = _positive_int( + raw_git_executable["size_bytes"], + context="runtime Git executable size", + ) + python_record = root["python"] if not isinstance(python_record, dict): raise ValueError("calibration runtime python record must be an object") @@ -1922,7 +2678,7 @@ def parse_calibration_runtime_manifest(data: bytes) -> CalibrationRuntimeManifes if not isinstance(raw_base_sys_path, list) or not raw_base_sys_path: raise ValueError("base_sys_path must be a non-empty list") base_sys_path = tuple( - _canonical_relative_path(item, context="base sys.path entry") + _canonical_base_sys_path_entry(item, context="base sys.path entry") for item in raw_base_sys_path ) if len({item.casefold() for item in base_sys_path}) != len(base_sys_path): @@ -1978,16 +2734,14 @@ def parse_calibration_runtime_manifest(data: bytes) -> CalibrationRuntimeManifes interpreter_path = _canonical_relative_path( interpreter["relative_path"], context="runtime interpreter relative path" ) - interpreter_sha256 = _sha256( - interpreter["sha256"], context="runtime interpreter SHA-256" - ) - interpreter_size = _positive_int( - interpreter["size_bytes"], context="runtime interpreter size" - ) + interpreter_sha256 = _sha256(interpreter["sha256"], context="runtime interpreter SHA-256") + interpreter_size = _positive_int(interpreter["size_bytes"], context="runtime interpreter size") base_files = {item.path: item for item in trees[0].files} for sys_path_entry in base_sys_path: - present = sys_path_entry in base_files or any( - path.startswith(f"{sys_path_entry}/") for path in base_files + present = ( + sys_path_entry == "." + or sys_path_entry in base_files + or any(path.startswith(f"{sys_path_entry}/") for path in base_files) ) optional_zip = re.fullmatch(r"python[0-9]+\.zip", sys_path_entry) is not None if not present and not optional_zip: @@ -2069,6 +2823,9 @@ def parse_calibration_runtime_manifest(data: bytes) -> CalibrationRuntimeManifes interpreter_relative_path=interpreter_path, interpreter_size_bytes=interpreter_size, interpreter_sha256=interpreter_sha256, + git_executable_absolute_path_sha256=git_absolute_path_sha256, + git_executable_sha256=git_sha256, + git_executable_size_bytes=git_size_bytes, runtime_trees=trees, distributions=tuple(parsed), file_sha256=sha256_bytes(data), @@ -2082,11 +2839,19 @@ def authenticate_calibration_runtime( package_roots: Mapping[str, Path], distributions: Sequence[Any] | None = None, interpreter_path: Path | None = None, + git_executable_path: Path | None = None, ) -> AuthenticatedRuntime: """Rehash both complete staged trees and exact RECORD inventories.""" if not isinstance(manifest, CalibrationRuntimeManifest): raise TypeError("manifest must be CalibrationRuntimeManifest") + git_executable = _authenticate_git_executable(git_executable_path) + if ( + git_executable.absolute_path_sha256 != manifest.git_executable_absolute_path_sha256 + or git_executable.sha256 != manifest.git_executable_sha256 + or git_executable.size_bytes != manifest.git_executable_size_bytes + ): + raise CalibrationRunError("Git executable differs from the frozen runtime manifest") roots = _runtime_root_map(Path(base_runtime_root), package_roots) packages = {name: roots[name] for name in sorted(roots) if name != BASE_RUNTIME_ROOT_NAME} manifest_package_names = tuple(item.name for item in manifest.package_roots) @@ -2171,6 +2936,9 @@ def authenticate_calibration_runtime( python_version=manifest.python_version, python_cache_tag=manifest.python_cache_tag, interpreter_sha256=manifest.interpreter_sha256, + git_executable_absolute_path_sha256=manifest.git_executable_absolute_path_sha256, + git_executable_sha256=manifest.git_executable_sha256, + git_executable_size_bytes=manifest.git_executable_size_bytes, machine_name=manifest.machine_name, base_runtime_file_count=len(manifest.runtime_trees[0].files), package_root_count=len(manifest.package_roots), @@ -2488,7 +3256,7 @@ def _copy_record_only_packages( raise CalibrationRunError("selected distributions contributed no staged package files") -_RUNTIME_PROBE_SOURCE: Final = r''' +_RUNTIME_PROBE_SOURCE: Final = r""" import json import os import platform @@ -2538,7 +3306,7 @@ def _copy_record_only_packages( }, } print(json.dumps(payload, allow_nan=False, sort_keys=True, separators=(",", ":"))) -'''.strip() +""".strip() def _probe_staged_interpreter( @@ -2596,7 +3364,7 @@ def _probe_staged_interpreter( context="runtime probe pointer_bits", ), base_sys_path=tuple( - _canonical_relative_path(item, context="runtime probe base sys.path") + _canonical_base_sys_path_entry(item, context="runtime probe base sys.path") for item in raw_paths ), ) @@ -2607,6 +3375,7 @@ def prepare_calibration_runtime( source_python: Path, requirements_file: Path, output_root: Path, + git_executable_path: Path | None = None, package_root_name: str = DEFAULT_PACKAGE_RUNTIME_ROOT_NAME, ) -> dict[str, object]: """Stage independent base bytes and only exact wheel-RECORD package bytes.""" @@ -2644,6 +3413,7 @@ def prepare_calibration_runtime( package_roots = {name: package_root} package_import_paths = {name: import_path} payload = capture_calibration_runtime_manifest( + git_executable_path=git_executable_path, base_runtime_root=base_root, package_roots=package_roots, package_import_paths=package_import_paths, @@ -2655,6 +3425,7 @@ def prepare_calibration_runtime( if manifest_path.read_bytes() != payload: raise CalibrationRunError("prepared runtime manifest changed after publication") repeated = capture_calibration_runtime_manifest( + git_executable_path=git_executable_path, base_runtime_root=base_root, package_roots=package_roots, package_import_paths=package_import_paths, @@ -2742,8 +3513,7 @@ def _verify_sealed_launch_state( str(base_root / PurePosixPath(relative)) for relative in manifest.base_sys_path ] expected_sys_path.extend( - str(roots[item.name] / PurePosixPath(item.import_path)) - for item in manifest.package_roots + str(roots[item.name] / PurePosixPath(item.import_path)) for item in manifest.package_roots ) if [os.path.abspath(item) for item in sys.path] != [ os.path.abspath(item) for item in expected_sys_path @@ -2759,6 +3529,7 @@ def _authenticate_sealed_runtime_context( package_roots: Mapping[str, Path], package_import_paths: Mapping[str, str], interpreter_path: Path, + git_executable_path: Path, pycache_prefix: Path, ) -> tuple[CalibrationRuntimeManifest, SealedRuntimeContext, AuthenticatedRuntime]: """Reauthenticate explicit launcher inputs without copying them to globals.""" @@ -2768,9 +3539,7 @@ def _authenticate_sealed_runtime_context( declared_names = tuple(item.name for item in manifest.package_roots) actual_names = tuple(name for name in roots if name != BASE_RUNTIME_ROOT_NAME) if actual_names != declared_names: - raise CalibrationRunError( - "bootstrap package roots differ from the frozen runtime manifest" - ) + raise CalibrationRunError("bootstrap package roots differ from the frozen runtime manifest") normalized_import_paths = _normalized_package_import_paths( {item.name: roots[item.name] for item in manifest.package_roots}, package_import_paths, @@ -2790,12 +3559,14 @@ def _authenticate_sealed_runtime_context( base_runtime_root=base_runtime_root, package_roots=package_roots, interpreter_path=interpreter_path, + git_executable_path=git_executable_path, ) context = SealedRuntimeContext( manifest_file_sha256=manifest.file_sha256, base_runtime_root=roots[BASE_RUNTIME_ROOT_NAME], package_roots={item.name: roots[item.name] for item in manifest.package_roots}, package_import_paths=normalized_import_paths, + git_executable_path=_authenticate_git_executable(git_executable_path).path, pycache_prefix=verified_pycache, ) return manifest, context, authenticated @@ -2812,20 +3583,151 @@ def _load_identity_resolver(repository_root: Path) -> Any: return module -def _identity_view(data: bytes, repository_root: Path) -> FrozenCalibrationIdentity: - resolver = _load_identity_resolver(repository_root) - decoded = resolver.deserialize_frozen_calibration_identity_artifact(data) +def _identity_records_with_fisher_boundary( + decoded: object, + evidence: Mapping[str, object], +) -> tuple[dict[str, object], ...]: + """Preserve each resolver-validated schema-v5 Fisher boundary exactly.""" + + raw_records = evidence.get("records") + decoded_records = getattr(decoded, "records", None) + if not isinstance(raw_records, list) or not isinstance(decoded_records, (list, tuple)): + raise CalibrationRunError("schema-v5 frozen identity records are missing") + if len(raw_records) != len(decoded_records): + raise CalibrationRunError("decoded schema-v5 record inventory differs from evidence") + + normalized_records: list[dict[str, object]] = [] + for index, (raw_record, decoded_record) in enumerate( + zip(raw_records, decoded_records, strict=True) + ): + if not isinstance(raw_record, Mapping) or not isinstance(decoded_record, Mapping): + raise CalibrationRunError(f"schema-v5 records[{index}] is not an object") + raw_boundary = raw_record.get("fisher_boundary") + decoded_boundary = decoded_record.get("fisher_boundary") + if not isinstance(raw_boundary, Mapping) or not isinstance(decoded_boundary, Mapping): + raise CalibrationRunError(f"schema-v5 records[{index}].fisher_boundary is missing") + try: + _exact_fields( + raw_boundary, + FISHER_BOUNDARY_FIELDS, + context=f"identity evidence records[{index}].fisher_boundary", + ) + _exact_fields( + decoded_boundary, + FISHER_BOUNDARY_FIELDS, + context=f"decoded records[{index}].fisher_boundary", + ) + except ValueError as exc: + raise CalibrationRunError( + f"schema-v5 records[{index}].fisher_boundary fields drifted" + ) from exc + if dict(decoded_boundary) != dict(raw_boundary): + raise CalibrationRunError( + f"decoded records[{index}].fisher_boundary differs from identity evidence" + ) + if decoded_boundary["schema"] != FISHER_BOUNDARY_SCHEMA: + raise CalibrationRunError(f"schema-v5 records[{index}].fisher_boundary schema drifted") + horizon = decoded_boundary["horizon"] + if type(horizon) is not int or horizon != FISHER_BOUNDARY_HORIZON: + raise CalibrationRunError(f"schema-v5 records[{index}].fisher_boundary horizon drifted") + + positions: dict[str, list[int]] = {} + for name in ("boundary_positions", "input_positions", "target_positions"): + values = decoded_boundary[name] + if ( + not isinstance(values, list) + or not values + or any(type(value) is not int or value < 0 for value in values) + ): + raise CalibrationRunError( + f"schema-v5 records[{index}].fisher_boundary {name} is invalid" + ) + positions[name] = list(values) + if not ( + len(positions["boundary_positions"]) + == len(positions["input_positions"]) + == len(positions["target_positions"]) + ): + raise CalibrationRunError( + f"schema-v5 records[{index}].fisher_boundary position lengths drifted" + ) + if positions["input_positions"] != [ + value + FISHER_BOUNDARY_HORIZON for value in positions["boundary_positions"] + ] or positions["target_positions"] != [value + 1 for value in positions["input_positions"]]: + raise CalibrationRunError( + f"schema-v5 records[{index}].fisher_boundary H=1 positions drifted" + ) + sequence_length = decoded_record.get("sequence_length") + if type(sequence_length) is not int or sequence_length < 3: + raise CalibrationRunError( + f"schema-v5 records[{index}] cannot support an H=1 Fisher boundary" + ) + if positions["boundary_positions"] != list(frozen_anchor_positions(sequence_length - 2)): + raise CalibrationRunError( + f"schema-v5 records[{index}].fisher_boundary B(T) positions drifted" + ) + try: + for name in ( + "input_token_ids_sha256", + "target_token_ids_sha256", + "fisher_boundary_sha256", + ): + _sha256( + decoded_boundary[name], + context=f"records[{index}].fisher_boundary.{name}", + ) + except ValueError as exc: + raise CalibrationRunError( + f"schema-v5 records[{index}].fisher_boundary hash is invalid" + ) from exc + boundary_payload = { + name: decoded_boundary[name] + for name in FISHER_BOUNDARY_FIELDS - {"fisher_boundary_sha256"} + } + if decoded_boundary["fisher_boundary_sha256"] != sha256_bytes( + FISHER_BOUNDARY_NAMESPACE + canonical_json_bytes(boundary_payload) + ): + raise CalibrationRunError( + f"schema-v5 records[{index}].fisher_boundary self-hash drifted" + ) + + normalized_boundary = dict(decoded_boundary) + normalized_boundary.update(positions) + normalized_record = dict(decoded_record) + normalized_record["fisher_boundary"] = normalized_boundary + normalized_records.append(normalized_record) + return tuple(normalized_records) + + +def _identity_view_from_resolver( + data: bytes, + resolver: Any, + *, + expected_file_sha256: str | None = None, +) -> FrozenCalibrationIdentity: + if expected_file_sha256 is None: + decoded = resolver.deserialize_frozen_calibration_identity_artifact(data) + else: + decoded = resolver.deserialize_frozen_calibration_identity_artifact( + data, + expected_file_sha256=expected_file_sha256, + ) root = _strict_json_bytes(data, context="frozen calibration identity") evidence = root.get("evidence") if not isinstance(evidence, dict): # independently decoded above; defensive only raise ValueError("frozen identity evidence is missing") + if ( + type(evidence.get("schema_version")) is not int + or evidence.get("schema_version") != FROZEN_IDENTITY_SCHEMA_VERSION + ): + raise CalibrationRunError("frozen identity is not strict schema v5") model_contracts = cast(dict[str, object], evidence["model_contracts"]) primary = cast(dict[str, object], model_contracts["primary"]) tokenizer = cast(dict[str, object], evidence["tokenizer"]) execution_bindings = getattr(decoded, "execution_bindings", None) if not isinstance(execution_bindings, Mapping): raise CalibrationRunError( - "frozen identity does not contain schema-v4 execution_bindings; " + "frozen identity does not contain schema-v5 execution_bindings; " "runtime/model access remains unauthorized" ) _exact_fields( @@ -2840,10 +3742,11 @@ def _identity_view(data: bytes, repository_root: Path) -> FrozenCalibrationIdent ) if evidence.get("execution_bindings") != dict(execution_bindings): raise CalibrationRunError("decoded execution bindings differ from identity evidence") + records = _identity_records_with_fisher_boundary(decoded, evidence) return FrozenCalibrationIdentity( file_sha256=decoded.file_sha256, canonical_evidence_sha256=decoded.canonical_evidence_sha256, - records=tuple(dict(record) for record in decoded.records), + records=records, assignment=tuple(dict(item) for item in decoded.assignment), assignment_sha256=decoded.assignment_sha256, tokenizer_manifest_sha256=decoded.tokenizer_manifest_sha256, @@ -2874,9 +3777,15 @@ def _identity_view(data: bytes, repository_root: Path) -> FrozenCalibrationIdent ) +def _identity_view(data: bytes, repository_root: Path) -> FrozenCalibrationIdentity: + return _identity_view_from_resolver(data, _load_identity_resolver(repository_root)) + + def verify_repository_source_manifest( expected: Mapping[str, object], repository_root: Path, + *, + git_executable_path: Path | None = None, ) -> tuple[dict[str, object], str]: """Use the frozen source API to reauthenticate code at point of use.""" @@ -2887,6 +3796,7 @@ def verify_repository_source_manifest( verified = module.verify_experiment013_source_manifest( normalized_expected, repo_root=repository_root, + git_executable=git_executable_path, ) if verified != normalized_expected: raise CalibrationRunError("repository source verification returned a different manifest") @@ -2907,25 +3817,6 @@ def validate_adapter_contract(adapter: Any, *, calibration_api: ModuleType) -> N raise TypeError("reviewed adapter does not implement CalibrationAdapter") -def _verify_repository_commit(repository_root: Path, expected: str) -> str: - revision = _git_revision(expected, context="expected source commit") - process = subprocess.run( - ["git", "-C", str(repository_root), "rev-parse", "HEAD"], - check=False, - capture_output=True, - text=True, - timeout=30, - ) - if process.returncode != 0: - raise CalibrationRunError("cannot resolve repository HEAD for policy provenance") - actual = process.stdout.strip() - if actual != revision: - raise CalibrationRunError( - f"repository HEAD differs from expected source commit: {actual!r} != {revision!r}" - ) - return revision - - def _record_int(record: Mapping[str, object], name: str) -> int: value = record.get(name) if isinstance(value, bool) or not isinstance(value, int) or value < 0: @@ -2949,6 +3840,7 @@ def validate_materialized_sequence( materialized: Any, *, calibration_api: ModuleType, + identity_resolver: Any, ) -> tuple[int, ...]: """Reauthenticate all sequence commitments without retaining source text.""" @@ -2989,6 +3881,27 @@ def validate_materialized_sequence( for name, actual in exact.items(): if actual != record.get(name): raise CalibrationRunError(f"materialized {name} differs from frozen identity") + build_fisher_boundary = getattr( + identity_resolver, + "build_fisher_boundary_contract", + None, + ) + if not callable(build_fisher_boundary): + raise CalibrationRunError( + "authenticated identity resolver lacks the Fisher-boundary contract builder" + ) + try: + expected_fisher_boundary = build_fisher_boundary(token_ids) + except (TypeError, ValueError) as exc: + raise CalibrationRunError( + "materialized token IDs cannot satisfy the frozen Fisher-boundary contract" + ) from exc + if not isinstance(expected_fisher_boundary, Mapping) or dict( + expected_fisher_boundary + ) != record.get("fisher_boundary"): + raise CalibrationRunError( + "materialized Fisher input/target tokens differ from frozen identity" + ) return token_ids @@ -3006,51 +3919,65 @@ def compute_anchor_distortions( state: Any, geometry: Geometry, ) -> tuple[Any, Any, Any]: - """Return per-row RHT Q4/Q6/Q8 MSE, releasing layer workspaces eagerly.""" + """Return Q4/Q6/Q8 endpoint MSE with deterministic CPU-FP64 reduction.""" + + from recurquant.static_q468 import StaticRhtQ468Geometry + from recurquant.static_q468_calibration import compute_rht_unweighted_mse_endpoints + + endpoint_geometry = StaticRhtQ468Geometry( + layer_indices=geometry.layer_indices, + heads=geometry.heads, + key_rows=geometry.key_rows, + value_width=geometry.value_width, + # Endpoint score math does not consume the packing target. A positive + # inert value keeps the shared geometry validator authoritative. + target_resident_bytes=1, + ) + try: + return cast( + tuple[Any, Any, Any], + compute_rht_unweighted_mse_endpoints( + state, + geometry=endpoint_geometry, + ), + ) + except (TypeError, ValueError) as exc: + raise CalibrationRunError("anchor state violates the frozen endpoint contract") from exc - torch = _torch_runtime() - from recurquant.quantization import QuantizationSpec, quantize_dequantize - from recurquant.rht import right_rht_encode - expected = ( - geometry.layers, - geometry.heads, - geometry.key_rows, - geometry.value_width, - ) - if not isinstance(state, torch.Tensor) or tuple(state.shape) != expected: - raise CalibrationRunError(f"anchor state must have shape {expected}") - if not state.is_floating_point() or not torch.isfinite(state).all().item(): - raise CalibrationRunError("anchor state must be finite floating point") - specifications = tuple( - QuantizationSpec( - bits=bits, - group_size=geometry.value_width, - scale_bits=16, - flatten_last_dims=1, - rounding="nearest", - seed=RHT_SEED, - ) - for bits in (4, 6, 8) - ) - per_bit: list[list[Any]] = [[], [], []] - with torch.no_grad(): - for local_index, layer_index in enumerate(geometry.layer_indices): - encoded = right_rht_encode( - state[local_index].unsqueeze(0), - layer_index=layer_index, - expected_heads=geometry.heads, - output_dtype=torch.float32, - ) - for destination, specification in zip(per_bit, specifications, strict=True): - restored = quantize_dequantize(encoded, specification).tensor - mse = (restored - encoded).square().mean(dim=-1).squeeze(0) - destination.append(mse.detach().to(device="cpu", dtype=torch.float64)) - return cast( - tuple[Any, Any, Any], - tuple(torch.stack(rows, dim=0).contiguous() for rows in per_bit), +def compute_fisher_distortions( + source_state: Any, + source_gradient: Any, + geometry: Geometry, +) -> tuple[Any, Any, Any]: + """Return causal H=1 diagonal-Fisher Q4/Q6/Q8 endpoint risks.""" + + from recurquant.static_q468 import StaticRhtQ468Geometry + from recurquant.static_q468_calibration import ( + compute_rht_diagonal_empirical_fisher_h1_endpoints, ) + endpoint_geometry = StaticRhtQ468Geometry( + layer_indices=geometry.layer_indices, + heads=geometry.heads, + key_rows=geometry.key_rows, + value_width=geometry.value_width, + target_resident_bytes=1, + ) + try: + return cast( + tuple[Any, Any, Any], + compute_rht_diagonal_empirical_fisher_h1_endpoints( + source_state, + source_gradient, + geometry=endpoint_geometry, + ), + ) + except (TypeError, ValueError) as exc: + raise CalibrationRunError( + "Fisher source state/gradient violates the frozen endpoint contract" + ) from exc + def capture_sequence_causally( adapter: Any, @@ -3062,30 +3989,106 @@ def capture_sequence_causally( calibration_api: ModuleType, require_cuda: bool, distortion_function: DistortionFunction = compute_anchor_distortions, + fisher_distortion_function: FisherDistortionFunction = compute_fisher_distortions, ) -> CapturedSequence: - """Process exactly one token per adapter call and retain only anchor tensors.""" + """Run exactly one forward per token and collect both frozen endpoint sets.""" torch = _torch_runtime() anchors = frozen_anchor_positions(len(token_ids)) anchor_set = set(anchors) + boundary = record.get("fisher_boundary") + if not isinstance(boundary, Mapping): + raise CalibrationRunError("identity record is missing its Fisher-boundary contract") + try: + fisher_boundaries = tuple(cast(Sequence[int], boundary["boundary_positions"])) + fisher_inputs = tuple(cast(Sequence[int], boundary["input_positions"])) + fisher_targets = tuple(cast(Sequence[int], boundary["target_positions"])) + except (KeyError, TypeError) as exc: + raise CalibrationRunError("identity Fisher-boundary positions are malformed") from exc + expected_boundaries = frozen_anchor_positions(len(token_ids) - 2) + expected_inputs = tuple(position + 1 for position in expected_boundaries) + expected_targets = tuple(position + 1 for position in expected_inputs) + if ( + fisher_boundaries != expected_boundaries + or fisher_inputs != expected_inputs + or fisher_targets != expected_targets + ): + raise CalibrationRunError("identity Fisher-boundary positions differ from B(T), H=1") + fisher_by_input = dict(zip(fisher_inputs, fisher_boundaries, strict=True)) query_ema: Any | None = None energies: list[Any] = [] q4_rows: list[Any] = [] q6_rows: list[Any] = [] q8_rows: list[Any] = [] + fisher_q4_rows: list[Any] = [] + fisher_q6_rows: list[Any] = [] + fisher_q8_rows: list[Any] = [] + fisher_target_nlls: list[float] = [] + expected_rows = (geometry.layers, geometry.heads, geometry.key_rows) + expected_state_shape = (*expected_rows, geometry.value_width) + + def validate_endpoint_triplet( + values: tuple[Any, Any, Any], + *, + context: str, + ) -> tuple[Any, Any, Any]: + for name, tensor in zip(("Q4", "Q6", "Q8"), values, strict=True): + if ( + not isinstance(tensor, torch.Tensor) + or tuple(tensor.shape) != expected_rows + or tensor.device.type != "cpu" + or tensor.dtype != torch.float64 + or not torch.isfinite(tensor).all().item() + or (tensor < 0).any().item() + ): + raise CalibrationRunError( + f"{context} {name} endpoint must be finite non-negative CPU FP64 " + f"{expected_rows}" + ) + return values + adapter.begin_sequence(model, record) completed = False try: for position, token_id in enumerate(token_ids): capture_state = position in anchor_set - observation = adapter.step_token( - model, - token_id=token_id, - position=position, - capture_state=capture_state, - ) + fisher_observation: Any | None = None + if position in fisher_by_input: + target_position = position + 1 + fisher_observation = adapter.step_token_with_fisher( + model, + token_id=token_id, + position=position, + target_token_id=token_ids[target_position], + capture_state=capture_state, + ) + if not isinstance( + fisher_observation, + calibration_api.FisherStepObservation, + ): + raise TypeError( + "adapter.step_token_with_fisher must return FisherStepObservation" + ) + if ( + fisher_observation.boundary_position != fisher_by_input[position] + or fisher_observation.input_position != position + or fisher_observation.target_position != target_position + or fisher_observation.input_token_id != token_id + or fisher_observation.target_token_id != token_ids[target_position] + ): + raise CalibrationRunError( + "adapter Fisher observation differs from the frozen H=1 causal pair" + ) + observation = fisher_observation.step_observation + else: + observation = adapter.step_token( + model, + token_id=token_id, + position=position, + capture_state=capture_state, + ) if not isinstance(observation, calibration_api.StepObservation): - raise TypeError("adapter.step_token must return StepObservation") + raise TypeError("adapter causal step must return StepObservation") if ( observation.position != position or observation.token_id != token_id @@ -3114,6 +4117,41 @@ def capture_sequence_causally( query_ema = QUERY_EMA_DECAY * query_ema + (1.0 - QUERY_EMA_DECAY) * energy if not torch.isfinite(query_ema).all().item() or (query_ema < 0).any().item(): raise CalibrationRunError("normalized-query-energy EMA became invalid") + + if fisher_observation is not None: + source_state = fisher_observation.source_recurrent_state + source_gradient = fisher_observation.source_state_gradient + for name, tensor in ( + ("source recurrent state", source_state), + ("source state gradient", source_gradient), + ): + if ( + not isinstance(tensor, torch.Tensor) + or tuple(tensor.shape) != expected_state_shape + or tensor.dtype != torch.float32 + or tensor.device != query.device + or not torch.isfinite(tensor).all().item() + ): + raise CalibrationRunError( + f"Fisher {name} must be finite FP32 {expected_state_shape} " + "on the recurrence-query device" + ) + if require_cuda and tensor.device.type != "cuda": + raise CalibrationRunError( + f"official Fisher {name} must be an actual CUDA tensor" + ) + target_nll = fisher_observation.target_nll + if type(target_nll) is not float or not math.isfinite(target_nll) or target_nll < 0: + raise CalibrationRunError("Fisher target-token NLL must be finite non-negative") + fisher_values = validate_endpoint_triplet( + fisher_distortion_function(source_state, source_gradient, geometry), + context="Fisher", + ) + fisher_q4_rows.append(fisher_values[0]) + fisher_q6_rows.append(fisher_values[1]) + fisher_q8_rows.append(fisher_values[2]) + fisher_target_nlls.append(target_nll) + if capture_state: if observation.recurrent_state is None: raise CalibrationRunError("adapter omitted recurrent state at a frozen anchor") @@ -3130,25 +4168,10 @@ def capture_sequence_causally( ) if state.dtype != torch.float32: raise CalibrationRunError("reference recurrent state must be FP32") - d4, d6, d8 = distortion_function(state, geometry) - expected_rows = (geometry.layers, geometry.heads, geometry.key_rows) - for name, tensor in ( - ("D4", d4), - ("D6", d6), - ("D8", d8), - ): - if ( - not isinstance(tensor, torch.Tensor) - or tuple(tensor.shape) != expected_rows - or tensor.device.type != "cpu" - or tensor.dtype != torch.float64 - or not torch.isfinite(tensor).all().item() - or (tensor < 0).any().item() - ): - raise CalibrationRunError( - f"{name} distortion must be finite non-negative CPU FP64 " - f"{expected_rows}" - ) + d4, d6, d8 = validate_endpoint_triplet( + distortion_function(state, geometry), + context="MSE", + ) energies.append(query_ema.detach().to(device="cpu", dtype=torch.float64)) q4_rows.append(d4) q6_rows.append(d6) @@ -3158,14 +4181,28 @@ def capture_sequence_causally( completed = True finally: adapter.end_sequence(model, record) - if not completed or len(energies) != len(anchors): - raise CalibrationRunError("causal sequence capture did not complete every frozen anchor") + if ( + not completed + or len(energies) != len(anchors) + or len(fisher_q4_rows) != len(fisher_boundaries) + or len(fisher_q6_rows) != len(fisher_boundaries) + or len(fisher_q8_rows) != len(fisher_boundaries) + or len(fisher_target_nlls) != len(fisher_boundaries) + ): + raise CalibrationRunError( + "causal sequence capture did not complete every post-token and Fisher endpoint" + ) return CapturedSequence( anchor_positions=anchors, query_energy=torch.stack(energies).contiguous(), q4_mse=torch.stack(q4_rows).contiguous(), q6_mse=torch.stack(q6_rows).contiguous(), q8_mse=torch.stack(q8_rows).contiguous(), + fisher_boundary_positions=fisher_boundaries, + fisher_q4_risk=torch.stack(fisher_q4_rows).contiguous(), + fisher_q6_risk=torch.stack(fisher_q6_rows).contiguous(), + fisher_q8_risk=torch.stack(fisher_q8_rows).contiguous(), + fisher_target_nlls=torch.tensor(fisher_target_nlls, dtype=torch.float64), ) @@ -3219,18 +4256,25 @@ def reduce_sequence( captured: CapturedSequence, ) -> object: from recurquant.static_q468_calibration import ( + FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE, + FROZEN_UNWEIGHTED_MSE_PROFILE, AnchorDistortionBatch, + FrozenComparatorEndpointBatch, reduce_frozen_anchor_distortions, + reduce_frozen_comparator_endpoints, ) - batch = AnchorDistortionBatch( - family=cast(Any, record["family"]), - config=cast(str, record["config"]), - ruler_category=cast(Any, record["ruler_category"]), - canonical_id=cast(str, record["canonical_id"]), - seed=cast(int | None, record["seed"]), - configured_length=cast(int | None, record["configured_length"]), - token_count=len(token_ids), + metadata = { + "family": cast(Any, record["family"]), + "config": cast(str, record["config"]), + "ruler_category": cast(Any, record["ruler_category"]), + "canonical_id": cast(str, record["canonical_id"]), + "seed": cast(int | None, record["seed"]), + "configured_length": cast(int | None, record["configured_length"]), + "token_count": len(token_ids), + } + candidate_batch = AnchorDistortionBatch( + **metadata, anchor_positions=captured.anchor_positions, query_energy=captured.query_energy, q4_mse=captured.q4_mse, @@ -3239,7 +4283,32 @@ def reduce_sequence( sequence_token_ids=token_ids, identity_record=record, ) - return reduce_frozen_anchor_distortions(batch) + mse_batch = FrozenComparatorEndpointBatch( + selector_profile=FROZEN_UNWEIGHTED_MSE_PROFILE, + **metadata, + endpoint_positions=captured.anchor_positions, + q4_scores=captured.q4_mse, + q6_scores=captured.q6_mse, + q8_scores=captured.q8_mse, + sequence_token_ids=token_ids, + identity_record=record, + ) + fisher_batch = FrozenComparatorEndpointBatch( + selector_profile=FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE, + **metadata, + endpoint_positions=captured.fisher_boundary_positions, + q4_scores=captured.fisher_q4_risk, + q6_scores=captured.fisher_q6_risk, + q8_scores=captured.fisher_q8_risk, + sequence_token_ids=token_ids, + identity_record=record, + target_nlls=captured.fisher_target_nlls, + ) + return ReducedSequenceScores( + candidate=reduce_frozen_anchor_distortions(candidate_batch), + mse=reduce_frozen_comparator_endpoints(mse_batch), + fisher=reduce_frozen_comparator_endpoints(fisher_batch), + ) def finalize( self, @@ -3255,6 +4324,8 @@ def finalize( FROZEN_STATIC_Q468_PRIMARY_STEPS, STATIC_Q48_COMPARATOR_METHOD, STATIC_Q468_ABLATION_METHOD, + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + STATIC_Q468_MSE_METHOD, STATIC_Q468_PRIMARY_METHOD, build_static_rht_q48_policy, build_static_rht_q468_policy, @@ -3264,20 +4335,32 @@ def finalize( serialize_static_rht_q468_policy, ) from recurquant.static_q468_calibration import ( + FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE, + FROZEN_UNWEIGHTED_MSE_PROFILE, aggregate_calibration_scores, + aggregate_comparator_scores, build_frozen_calibration_score_artifact, + build_frozen_comparator_score_artifact, build_frozen_split_half_stability_artifact, deserialize_calibration_score_artifact, + deserialize_comparator_score_artifact, deserialize_frozen_split_half_stability_artifact, fit_split_half_policy, ) resolver = _load_identity_resolver(self.repository_root) - typed_scores = cast(list[Any], list(scores)) - aggregate = aggregate_calibration_scores(typed_scores) + if not scores or any(not isinstance(item, ReducedSequenceScores) for item in scores): + raise TypeError("scores must contain non-empty ReducedSequenceScores") + typed_scores = cast(list[ReducedSequenceScores], list(scores)) + candidate_scores = [item.candidate for item in typed_scores] + mse_scores = [item.mse for item in typed_scores] + fisher_scores = [item.fisher for item in typed_scores] + aggregate = aggregate_calibration_scores(candidate_scores) + mse_aggregate = aggregate_comparator_scores(mse_scores) + fisher_aggregate = aggregate_comparator_scores(fisher_scores) geometry = FROZEN_QWEN35_STATIC_Q468_GEOMETRY fit = fit_split_half_policy( - typed_scores, + candidate_scores, layer_indices=geometry.layer_indices, rows_per_layer=geometry.rows_per_layer, marginal_steps=FROZEN_STATIC_Q468_PRIMARY_STEPS, @@ -3300,6 +4383,19 @@ def finalize( calibration_identity_sha256=identity.file_sha256, ) decoded_score = deserialize_calibration_score_artifact(score_bytes) + comparator_score_bytes = build_frozen_comparator_score_artifact( + mse_aggregate, + fisher_aggregate, + calibration_identity_sha256=identity.file_sha256, + ) + decoded_comparator_score = deserialize_comparator_score_artifact( + comparator_score_bytes, + expected_calibration_identity_sha256=identity.file_sha256, + ) + mse_selector = decoded_comparator_score.selectors[FROZEN_UNWEIGHTED_MSE_PROFILE] + fisher_selector = decoded_comparator_score.selectors[ + FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE + ] split_bytes = build_frozen_split_half_stability_artifact( fit.half_a_aggregate, fit.half_b_aggregate, @@ -3339,6 +4435,29 @@ def finalize( method_id=STATIC_Q468_PRIMARY_METHOD, **policy_common, ) + comparator_policy_common = { + "geometry": geometry, + "identity_artifact_sha256": identity.file_sha256, + "tokenizer_manifest_sha256": identity.tokenizer_manifest_sha256, + "source_commit": source_commit, + "marginal_steps": FROZEN_STATIC_Q468_PRIMARY_STEPS, + } + mse_k29334 = build_static_rht_q468_policy( + mse_selector.aggregate.d4, + mse_selector.aggregate.d6, + mse_selector.aggregate.d8, + calibration_manifest_sha256=(mse_selector.aggregate.sequence_score_manifest_sha256), + method_id=STATIC_Q468_MSE_METHOD, + **comparator_policy_common, + ) + fisher_k29334 = build_static_rht_q468_policy( + fisher_selector.aggregate.d4, + fisher_selector.aggregate.d6, + fisher_selector.aggregate.d8, + calibration_manifest_sha256=(fisher_selector.aggregate.sequence_score_manifest_sha256), + method_id=STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + **comparator_policy_common, + ) q48 = build_static_rht_q48_policy( aggregate.d4, aggregate.d8, @@ -3352,9 +4471,13 @@ def finalize( ) k27030_bytes = serialize_static_rht_q468_policy(k27030) k29334_bytes = serialize_static_rht_q468_policy(k29334) + mse_k29334_bytes = serialize_static_rht_q468_policy(mse_k29334) + fisher_k29334_bytes = serialize_static_rht_q468_policy(fisher_k29334) q48_bytes = serialize_static_rht_q48_policy(q48) deserialize_static_rht_q468_policy(k27030_bytes) deserialize_static_rht_q468_policy(k29334_bytes) + deserialize_static_rht_q468_policy(mse_k29334_bytes) + deserialize_static_rht_q468_policy(fisher_k29334_bytes) deserialize_static_rht_q48_policy(q48_bytes) binding_bytes = resolver.build_stage_a_calibration_binding_artifact( frozen_identity_artifact=identity.artifact_bytes, @@ -3362,6 +4485,9 @@ def finalize( split_half_stability_artifact=split_bytes, static_k27030_policy_artifact=k27030_bytes, static_k29334_policy_artifact=k29334_bytes, + comparator_score_artifact=comparator_score_bytes, + static_fisher_k29334_policy_artifact=fisher_k29334_bytes, + static_mse_k29334_policy_artifact=mse_k29334_bytes, ) resolver.deserialize_stage_a_calibration_binding_artifact(binding_bytes) return FinalizationResult( @@ -3369,14 +4495,15 @@ def finalize( stability=stability, artifacts=CalibrationArtifacts( score=score_bytes, + comparator_score=comparator_score_bytes, split_half=split_bytes, static_k27030=k27030_bytes, static_k29334=k29334_bytes, + static_mse_k29334=mse_k29334_bytes, + static_fisher_k29334=fisher_k29334_bytes, static_q48=q48_bytes, stage_a_binding=binding_bytes, stability=stability, - calibration_scores_sha256=decoded_score.calibration_scores_sha256, - sequence_score_manifest_sha256=aggregate.sequence_score_manifest_sha256, ), ) @@ -3442,15 +4569,21 @@ def _report_bytes( model_files: Any, sequence_count: int, token_count: int, - anchor_count: int, + post_token_anchor_count: int, + fisher_boundary_count: int, + observed_fisher_step_count: int, stability: Mapping[str, object], artifacts: Mapping[str, bytes], runtime: Mapping[str, object], + fisher_h1_smoke_report_file_sha256: str | None, ) -> bytes: evidence = { "artifacts": {name: sha256_bytes(payload) for name, payload in sorted(artifacts.items())}, "calibration": { - "anchor_count": anchor_count, + "expected_fisher_step_count": fisher_boundary_count, + "observed_fisher_step_count": observed_fisher_step_count, + "fisher_boundary_count": fisher_boundary_count, + "post_token_anchor_count": post_token_anchor_count, "sequence_count": sequence_count, "token_count": token_count, }, @@ -3483,6 +4616,9 @@ def _report_bytes( "epsilon_hex": QUERY_ENERGY_EPSILON.hex(), "prior": "uniform_1_over_key_rows", }, + "prerequisites": { + "fisher_h1_smoke_report_file_sha256": (fisher_h1_smoke_report_file_sha256), + }, "repository": { "source_commit": source_commit, "source_manifest_file_sha256": source_manifest_file_sha256, @@ -3502,6 +4638,206 @@ def _report_bytes( return canonical_json_bytes(document) +def _authenticate_fisher_h1_smoke_prerequisite_unchecked( + report_bytes: bytes, + complete_marker_bytes: bytes, + *, + identity: FrozenCalibrationIdentity, + source_commit: str, + source_manifest_sha256: str, + source_manifest_file_sha256: str, + model_manifest: ModelFileManifest, + authenticated_runtime: AuthenticatedRuntime, +) -> str: + """Authenticate the mandatory one-sequence Fisher H=1 smoke receipt.""" + + if not isinstance(report_bytes, bytes) or not isinstance(complete_marker_bytes, bytes): + raise TypeError("Fisher H=1 smoke prerequisite files must be bytes") + if complete_marker_bytes != FISHER_SMOKE_COMPLETE_BYTES: + raise CalibrationRunError("Fisher H=1 smoke completion marker drifted") + root = _strict_json_bytes(report_bytes, context="Fisher H=1 smoke report") + _exact_fields( + root, + {"artifact_kind", "canonical_evidence_sha256", "evidence", "schema_version"}, + context="Fisher H=1 smoke report", + ) + if canonical_json_bytes(root) != report_bytes: + raise CalibrationRunError("Fisher H=1 smoke report is not canonical JSON") + if ( + root["artifact_kind"] != RUN_REPORT_KIND + or type(root["schema_version"]) is not int + or root["schema_version"] != RUN_REPORT_SCHEMA + ): + raise CalibrationRunError("Fisher H=1 smoke report kind or schema drifted") + evidence = root["evidence"] + if not isinstance(evidence, dict): + raise CalibrationRunError("Fisher H=1 smoke evidence is missing") + _exact_fields( + evidence, + { + "artifacts", + "calibration", + "identity", + "model_files", + "prerequisites", + "query_energy_ema", + "repository", + "runner_revision", + "runtime", + "stability", + "status", + }, + context="Fisher H=1 smoke evidence", + ) + canonical_hash = _sha256( + root["canonical_evidence_sha256"], + context="Fisher H=1 smoke canonical evidence SHA-256", + ) + if canonical_hash != sha256_bytes(canonical_json_bytes(evidence)): + raise CalibrationRunError("Fisher H=1 smoke canonical evidence SHA-256 drifted") + if ( + evidence["status"] != "fisher_h1_smoke_passed" + or evidence["runner_revision"] != RUNNER_REVISION + or evidence["artifacts"] != {} + or evidence["prerequisites"] != {"fisher_h1_smoke_report_file_sha256": None} + ): + raise CalibrationRunError("Fisher H=1 smoke status or provenance drifted") + _exact_typed_mapping( + evidence["stability"], + { + "checks": [], + "evaluated": False, + "passed": None, + "scope": "smoke_only", + }, + context="Fisher H=1 smoke stability", + ) + _exact_typed_mapping( + evidence["query_energy_ema"], + { + "decay_hex": QUERY_EMA_DECAY.hex(), + "epsilon_hex": QUERY_ENERGY_EPSILON.hex(), + "prior": "uniform_1_over_key_rows", + }, + context="Fisher H=1 smoke query-energy contract", + ) + + first_record = identity.records[0] + token_count = _positive_int( + first_record.get("sequence_length"), + context="first frozen smoke sequence length", + ) + raw_boundary = first_record.get("fisher_boundary") + if not isinstance(raw_boundary, Mapping): + raise CalibrationRunError("first frozen smoke sequence has no Fisher boundary") + boundary_positions = raw_boundary.get("boundary_positions") + if not isinstance(boundary_positions, list): + raise CalibrationRunError("first frozen smoke Fisher boundary positions are missing") + fisher_count = len(boundary_positions) + if fisher_count <= 0: + raise CalibrationRunError("first frozen smoke sequence has no Fisher H=1 steps") + _exact_typed_mapping( + evidence["calibration"], + { + "expected_fisher_step_count": fisher_count, + "observed_fisher_step_count": fisher_count, + "fisher_boundary_count": fisher_count, + "post_token_anchor_count": len(frozen_anchor_positions(token_count)), + "sequence_count": 1, + "token_count": token_count, + }, + context="Fisher H=1 smoke calibration receipt", + ) + _exact_typed_mapping( + evidence["identity"], + { + "canonical_evidence_sha256": identity.canonical_evidence_sha256, + "file_sha256": identity.file_sha256, + "identity_input_manifest_sha256": identity.identity_input_manifest_sha256, + "tokenizer_manifest_sha256": identity.tokenizer_manifest_sha256, + "execution_bindings": { + "calibration_runtime_manifest_file_sha256": (identity.runtime_manifest_file_sha256), + "model_file_manifest_file_sha256": identity.model_file_manifest_file_sha256, + "parquet_materialization_manifest_file_sha256": ( + identity.parquet_materialization_manifest_file_sha256 + ), + "repository_source_manifest_file_sha256": ( + identity.repository_source_manifest_file_sha256 + ), + }, + }, + context="Fisher H=1 smoke identity receipt", + ) + _exact_typed_mapping( + evidence["model_files"], + { + "file_count": len(model_manifest.files), + "hub_tree_manifest_sha256": model_manifest.hub_tree_manifest_sha256, + "manifest_file_sha256": model_manifest.file_sha256, + "model_id": model_manifest.model_id, + "revision": model_manifest.revision, + "transformers_version": model_manifest.transformers_version, + }, + context="Fisher H=1 smoke model receipt", + ) + _exact_typed_mapping( + evidence["repository"], + { + "source_commit": source_commit, + "source_manifest_file_sha256": source_manifest_file_sha256, + "source_manifest_sha256": source_manifest_sha256, + }, + context="Fisher H=1 smoke source receipt", + ) + runtime = evidence["runtime"] + if not isinstance(runtime, Mapping): + raise CalibrationRunError("Fisher H=1 smoke runtime receipt is missing") + if ( + runtime.get("runtime_manifest_file_sha256") != authenticated_runtime.manifest_file_sha256 + or runtime.get("authenticated_distribution_count") + != authenticated_runtime.distribution_count + or runtime.get("authenticated_file_count") != authenticated_runtime.file_count + or runtime.get("packages") != dict(authenticated_runtime.distributions) + or runtime.get("cuda_available") is not True + or not isinstance(runtime.get("gpu"), Mapping) + ): + raise CalibrationRunError("Fisher H=1 smoke runtime identity drifted") + adapter = runtime.get("adapter") + if not isinstance(adapter, Mapping) or adapter.get("fisher_step_count") != fisher_count: + raise CalibrationRunError("Fisher H=1 smoke adapter step receipt drifted") + return sha256_bytes(report_bytes) + + +def authenticate_fisher_h1_smoke_prerequisite( + report_bytes: bytes, + complete_marker_bytes: bytes, + *, + identity: FrozenCalibrationIdentity, + source_commit: str, + source_manifest_sha256: str, + source_manifest_file_sha256: str, + model_manifest: ModelFileManifest, + authenticated_runtime: AuthenticatedRuntime, +) -> str: + """Fail closed with one public error type for malformed smoke evidence.""" + + try: + return _authenticate_fisher_h1_smoke_prerequisite_unchecked( + report_bytes, + complete_marker_bytes, + identity=identity, + source_commit=source_commit, + source_manifest_sha256=source_manifest_sha256, + source_manifest_file_sha256=source_manifest_file_sha256, + model_manifest=model_manifest, + authenticated_runtime=authenticated_runtime, + ) + except CalibrationRunError: + raise + except (IndexError, KeyError, TypeError, ValueError) as exc: + raise CalibrationRunError(str(exc)) from exc + + def _atomic_publish_new(path: Path, payload: bytes) -> None: if not isinstance(payload, bytes): raise TypeError("artifact payload must be bytes") @@ -3521,7 +4857,14 @@ def _atomic_publish_new(path: Path, payload: bytes) -> None: temporary.unlink(missing_ok=True) -def _publish_output_directory(output_dir: Path, payloads: Mapping[str, bytes]) -> None: +def _publish_output_directory( + output_dir: Path, + payloads: Mapping[str, bytes], + *, + complete_filename: str = COMPLETE_FILENAME, +) -> None: + if complete_filename not in {COMPLETE_FILENAME, FISHER_SMOKE_COMPLETE_FILENAME}: + raise ValueError("completion marker is not a frozen Experiment 013 marker") resolved = Path(os.path.abspath(output_dir)) parent = resolved.parent parent.mkdir(parents=True, exist_ok=True) @@ -3541,8 +4884,12 @@ def _publish_output_directory(output_dir: Path, payloads: Mapping[str, bytes]) - for name in ordered: _atomic_publish_new(staging / name, payloads[name]) _atomic_publish_new( - staging / COMPLETE_FILENAME, - b"recurquant-experiment013-calibration-complete-v1\n", + staging / complete_filename, + ( + FISHER_SMOKE_COMPLETE_BYTES + if complete_filename == FISHER_SMOKE_COMPLETE_FILENAME + else b"recurquant-experiment013-calibration-complete-v1\n" + ), ) if resolved.exists(): raise FileExistsError(f"refusing to overwrite existing calibration output: {resolved}") @@ -3577,6 +4924,17 @@ def run_calibration( """Execute the authenticated calibration and publish one no-overwrite result set.""" started = time.perf_counter() + if not isinstance(config.require_cuda, bool) or not isinstance(config.fisher_h1_smoke, bool): + raise TypeError("calibration mode flags must be booleans") + for name, value in ( + ("prior_fisher_h1_smoke_report_bytes", config.prior_fisher_h1_smoke_report_bytes), + ( + "prior_fisher_h1_smoke_complete_bytes", + config.prior_fisher_h1_smoke_complete_bytes, + ), + ): + if value is not None and not isinstance(value, bytes): + raise TypeError(f"{name} must be bytes or None") # First executable boundary: a strict promoted identity decode. No source # adapter, model path, repository command, or output path is touched first. identity = services.backend.decode_identity(config.frozen_identity_bytes) @@ -3584,10 +4942,28 @@ def run_calibration( raise FileExistsError( f"refusing to overwrite existing calibration output: {config.output_dir.resolve()}" ) + if config.fisher_h1_smoke: + if ( + config.prior_fisher_h1_smoke_report_bytes is not None + or config.prior_fisher_h1_smoke_complete_bytes is not None + ): + raise CalibrationRunError("Fisher H=1 smoke mode forbids a prior smoke prerequisite") + elif ( + config.prior_fisher_h1_smoke_report_bytes is None + or config.prior_fisher_h1_smoke_complete_bytes is None + ): + raise CalibrationRunError( + "full calibration requires the prior Fisher H=1 smoke report and marker" + ) - source_commit = _verify_repository_commit( - config.repository_root, + # H0 remains the policy/report provenance even when the authenticated + # verifier accepts a later HEAD whose complete frozen source inventory is + # byte-identical to H0. The verifier below proves ancestry and equality; + # a raw HEAD == H0 check would incorrectly forbid committing the promoted + # identity before opening weights. + source_commit = _git_revision( config.expected_source_commit, + context="expected source commit", ) source_manifest_file_sha256 = sha256_bytes(config.repository_source_manifest_bytes) if source_manifest_file_sha256 != identity.repository_source_manifest_file_sha256: @@ -3649,24 +5025,38 @@ def run_calibration( config.expected_parquet_materialization_manifest_sha256, context="expected parquet materialization manifest SHA-256", ) - parquet_manifest_file_sha256 = sha256_bytes( - config.parquet_materialization_manifest_bytes - ) + parquet_manifest_file_sha256 = sha256_bytes(config.parquet_materialization_manifest_bytes) if ( parquet_manifest_file_sha256 != expected_parquet_manifest_sha256 - or parquet_manifest_file_sha256 - != identity.parquet_materialization_manifest_file_sha256 + or parquet_manifest_file_sha256 != identity.parquet_materialization_manifest_file_sha256 ): raise CalibrationRunError( - "parquet materialization manifest bytes differ from the frozen identity/config " - "binding" + "parquet materialization manifest bytes differ from the frozen identity/config binding" ) runtime_before_data = services.authenticate_runtime(runtime_manifest) if runtime_before_data != authenticated_runtime: raise CalibrationRunError("calibration runtime identity changed before data access") + fisher_h1_smoke_report_file_sha256: str | None = None + if not config.fisher_h1_smoke: + assert config.prior_fisher_h1_smoke_report_bytes is not None + assert config.prior_fisher_h1_smoke_complete_bytes is not None + fisher_h1_smoke_report_file_sha256 = authenticate_fisher_h1_smoke_prerequisite( + config.prior_fisher_h1_smoke_report_bytes, + config.prior_fisher_h1_smoke_complete_bytes, + identity=identity, + source_commit=source_commit, + source_manifest_sha256=source_manifest_sha256, + source_manifest_file_sha256=source_manifest_file_sha256, + model_manifest=model_manifest, + authenticated_runtime=authenticated_runtime, + ) + materialized: list[tuple[dict[str, object], tuple[int, ...]]] = [] + identity_resolver = services.identity_resolver + if identity_resolver is None: + identity_resolver = _load_identity_resolver(config.repository_root) for record in identity.records: candidate = adapter.materialize_sequence(record) materialized.append( @@ -3676,6 +5066,7 @@ def run_calibration( record, candidate, calibration_api=services.calibration_api, + identity_resolver=identity_resolver, ), ) ) @@ -3705,6 +5096,7 @@ def run_calibration( scores: list[object] = [] total_tokens = 0 total_anchors = 0 + total_fisher_boundaries = 0 try: # The adapter may call AutoModel only inside this method. The exact local # file set has already been hashed, revision checked, and source verified. @@ -3714,7 +5106,8 @@ def run_calibration( ) if authenticated_after_load != authenticated_model: raise CalibrationRunError("local model identity changed while loading weights") - for record, token_ids in materialized: + selected_materialized = materialized[:1] if config.fisher_h1_smoke else materialized + for record, token_ids in selected_materialized: captured = capture_sequence_causally( adapter, model, @@ -3724,14 +5117,20 @@ def run_calibration( calibration_api=services.calibration_api, require_cuda=config.require_cuda, distortion_function=services.distortion_function, + fisher_distortion_function=services.fisher_distortion_function, ) scores.append(services.backend.reduce_sequence(record, token_ids, captured)) total_tokens += len(token_ids) total_anchors += len(captured.anchor_positions) - result = services.backend.finalize( - scores, - identity=identity, - source_commit=source_commit, + total_fisher_boundaries += len(captured.fisher_boundary_positions) + result = ( + None + if config.fisher_h1_smoke + else services.backend.finalize( + scores, + identity=identity, + source_commit=source_commit, + ) ) # Snapshot run metadata while the authenticated model/device/observer # are still live. Cleanup follows immediately and every external @@ -3743,6 +5142,11 @@ def run_calibration( if adapter_runtime_metadata is None: raise RuntimeError("successful calibration omitted live adapter runtime metadata") + observed_fisher_steps = adapter_runtime_metadata.get("fisher_step_count") + if type(observed_fisher_steps) is not int or observed_fisher_steps != total_fisher_boundaries: + raise CalibrationRunError( + "adapter Fisher-step count differs from the frozen boundary inventory" + ) authenticated_after_run = services.authenticate_model_files(config.model_root, model_manifest) if authenticated_after_run != authenticated_model: @@ -3767,6 +5171,48 @@ def run_calibration( if final_model != authenticated_model: raise CalibrationRunError("local model identity changed before publication") + if config.fisher_h1_smoke: + if len(scores) != 1: + raise RuntimeError("Fisher H=1 smoke must cover exactly one frozen sequence") + report = _report_bytes( + status="fisher_h1_smoke_passed", + identity=identity, + source_commit=source_commit, + source_manifest_sha256=source_manifest_sha256, + source_manifest_file_sha256=source_manifest_file_sha256, + model_files=authenticated_model, + sequence_count=len(scores), + token_count=total_tokens, + post_token_anchor_count=total_anchors, + fisher_boundary_count=total_fisher_boundaries, + observed_fisher_step_count=cast(int, observed_fisher_steps), + stability={ + "checks": [], + "evaluated": False, + "passed": None, + "scope": "smoke_only", + }, + artifacts={}, + runtime=runtime, + fisher_h1_smoke_report_file_sha256=None, + ) + smoke_payloads = {FISHER_SMOKE_REPORT_FILENAME: report} + _publish_output_directory( + config.output_dir, + smoke_payloads, + complete_filename=FISHER_SMOKE_COMPLETE_FILENAME, + ) + return { + "artifact_sha256": {FISHER_SMOKE_REPORT_FILENAME: sha256_bytes(report)}, + "fisher_boundary_count": total_fisher_boundaries, + "output_dir": str(config.output_dir.resolve()), + "sequence_count": len(scores), + "status": "fisher_h1_smoke_passed", + "token_count": total_tokens, + } + + if result is None: + raise RuntimeError("full calibration omitted its finalization result") if not result.passed: report = _report_bytes( status="stability_failed", @@ -3777,10 +5223,13 @@ def run_calibration( model_files=authenticated_model, sequence_count=len(scores), token_count=total_tokens, - anchor_count=total_anchors, + post_token_anchor_count=total_anchors, + fisher_boundary_count=total_fisher_boundaries, + observed_fisher_step_count=cast(int, observed_fisher_steps), stability=result.stability, artifacts={}, runtime=runtime, + fisher_h1_smoke_report_file_sha256=(fisher_h1_smoke_report_file_sha256), ) _publish_output_directory(config.output_dir, {REPORT_FILENAME: report}) report_path = config.output_dir / REPORT_FILENAME @@ -3792,9 +5241,12 @@ def run_calibration( artifacts = result.artifacts payloads = { SCORE_FILENAME: artifacts.score, + COMPARATOR_SCORE_FILENAME: artifacts.comparator_score, SPLIT_FILENAME: artifacts.split_half, K27030_FILENAME: artifacts.static_k27030, K29334_FILENAME: artifacts.static_k29334, + MSE_K29334_FILENAME: artifacts.static_mse_k29334, + FISHER_K29334_FILENAME: artifacts.static_fisher_k29334, Q48_FILENAME: artifacts.static_q48, BINDING_FILENAME: artifacts.stage_a_binding, } @@ -3807,10 +5259,13 @@ def run_calibration( model_files=authenticated_model, sequence_count=len(scores), token_count=total_tokens, - anchor_count=total_anchors, + post_token_anchor_count=total_anchors, + fisher_boundary_count=total_fisher_boundaries, + observed_fisher_step_count=cast(int, observed_fisher_steps), stability=result.stability, artifacts=payloads, runtime=runtime, + fisher_h1_smoke_report_file_sha256=fisher_h1_smoke_report_file_sha256, ) payloads[REPORT_FILENAME] = report _publish_output_directory(config.output_dir, payloads) @@ -3829,6 +5284,7 @@ def default_services( calibration_api: ModuleType | None = None, interpreter_path: Path, package_roots: Mapping[str, Path], + git_executable_path: Path | None = None, ) -> RunnerServices: api = _AUTHENTICATED_CALIBRATION_API if calibration_api is None else calibration_api if api is None: @@ -3837,12 +5293,20 @@ def default_services( return RunnerServices( backend=backend, calibration_api=api, - verify_repository_source=verify_repository_source_manifest, + # The stdlib bootstrap authenticates and installs the resolver before + # run_calibration. Keep service construction import-free. + identity_resolver=None, + verify_repository_source=lambda manifest, root: verify_repository_source_manifest( + manifest, + root, + git_executable_path=git_executable_path, + ), validate_adapter=lambda adapter: validate_adapter_contract( adapter, calibration_api=api, ), distortion_function=compute_anchor_distortions, + fisher_distortion_function=compute_fisher_distortions, authenticate_model_files=lambda root, manifest: authenticate_local_model_files( root, manifest, @@ -3853,6 +5317,7 @@ def default_services( base_runtime_root=base_runtime_root, package_roots=package_roots, interpreter_path=interpreter_path, + git_executable_path=git_executable_path, ), ) @@ -3929,6 +5394,30 @@ def _parser() -> argparse.ArgumentParser: parser.add_argument("--repository-root", default=REPOSITORY_ROOT, type=Path) parser.add_argument("--source-commit", required=True) parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument( + "--fisher-h1-smoke", + action="store_true", + help=( + "Run exactly the first frozen calibration sequence through the full causal H=1 " + "path and publish only its authenticated smoke receipt." + ), + ) + parser.add_argument( + "--prior-fisher-h1-smoke-report", + type=Path, + help=( + "Canonical smoke report from the required earlier --fisher-h1-smoke run; " + "mandatory for full calibration and forbidden in smoke mode." + ), + ) + parser.add_argument( + "--prior-fisher-h1-smoke-complete-marker", + type=Path, + help=( + "FISHER_H1_SMOKE_COMPLETE marker paired with the prior smoke report; " + "mandatory for full calibration and forbidden in smoke mode." + ), + ) parser.add_argument( "--adapter", choices=[CANONICAL_ADAPTER_SPEC], @@ -3944,11 +5433,13 @@ def _capture_manifest_mode(arguments: Sequence[str]) -> int | None: "capture-runtime-manifest", "capture-model-manifest", "prepare-runtime", + "stage-model", }: return None command = arguments[0] parser = argparse.ArgumentParser(prog=f"{Path(__file__).name} {command}") if command == "prepare-runtime": + parser.add_argument("--git-executable", required=True, type=Path) parser.add_argument("--source-python", required=True, type=Path) parser.add_argument("--requirements", required=True, type=Path) parser.add_argument("--output-root", required=True, type=Path) @@ -3956,11 +5447,26 @@ def _capture_manifest_mode(arguments: Sequence[str]) -> int | None: "--package-root-name", default=DEFAULT_PACKAGE_RUNTIME_ROOT_NAME, ) + elif command == "stage-model": + parser.add_argument("--git-executable", required=True, type=Path) + parser.add_argument("--frozen-identity", required=True, type=Path) + parser.add_argument("--expected-frozen-identity-sha256", required=True) + parser.add_argument("--identity-commit", required=True) + parser.add_argument("--repository-root", required=True, type=Path) + parser.add_argument("--repository-source-manifest", required=True, type=Path) + parser.add_argument("--source-commit", required=True) + parser.add_argument("--model-file-manifest", required=True, type=Path) + parser.add_argument("--expected-model-file-manifest-sha256", required=True) + parser.add_argument("--hub-cache-root", required=True, type=Path) + parser.add_argument("--output-root", required=True, type=Path) + parser.add_argument("--local-files-only", action="store_true") else: parser.add_argument("--output", required=True, type=Path) if command == "capture-source-manifest": + parser.add_argument("--git-executable", required=True, type=Path) parser.add_argument("--repository-root", required=True, type=Path) if command == "capture-runtime-manifest": + parser.add_argument("--git-executable", required=True, type=Path) parser.add_argument("--base-runtime-root", required=True, type=Path) parser.add_argument("--staged-interpreter", required=True, type=Path) parser.add_argument("--package-root", required=True, action="append") @@ -3972,6 +5478,7 @@ def _capture_manifest_mode(arguments: Sequence[str]) -> int | None: args = parser.parse_args(arguments[1:]) if command == "prepare-runtime": details = prepare_calibration_runtime( + git_executable_path=args.git_executable, source_python=args.source_python, requirements_file=args.requirements, output_root=args.output_root, @@ -3979,18 +5486,44 @@ def _capture_manifest_mode(arguments: Sequence[str]) -> int | None: ) print(json.dumps(details, sort_keys=True)) return 0 + if command == "stage-model": + details = stage_identity_bound_model( + git_executable_path=args.git_executable, + frozen_identity_path=args.frozen_identity, + expected_frozen_identity_sha256=args.expected_frozen_identity_sha256, + identity_commit=args.identity_commit, + repository_root=args.repository_root, + repository_source_manifest_path=args.repository_source_manifest, + source_commit=args.source_commit, + model_file_manifest_path=args.model_file_manifest, + expected_model_file_manifest_sha256=args.expected_model_file_manifest_sha256, + hub_cache_root=args.hub_cache_root, + output_root=args.output_root, + local_files_only=args.local_files_only, + ) + print(json.dumps(details, sort_keys=True)) + return 0 status = "captured_metadata_only" details: dict[str, object] = {} if command == "capture-source-manifest": source_module = _load_source_capture_module(args.repository_root) try: - captured = source_module.capture_experiment013_source_manifest(args.repository_root) + captured = source_module.capture_experiment013_source_manifest( + args.repository_root, + git_executable=args.git_executable, + ) normalized = source_module.validate_experiment013_source_manifest(captured) payload = source_module.canonical_experiment013_source_manifest_bytes(normalized) - output = _assert_source_manifest_output_location(args.repository_root, args.output) + git_executable = _authenticate_git_executable(args.git_executable) + output = _assert_source_manifest_output_location( + args.repository_root, + args.output, + git_executable=git_executable, + ) before_publish = source_module.verify_experiment013_source_manifest( normalized, args.repository_root, + git_executable=git_executable.path, ) if before_publish != normalized: raise CalibrationRunError("source verifier changed the captured manifest") @@ -4000,6 +5533,7 @@ def _capture_manifest_mode(arguments: Sequence[str]) -> int | None: after_publish = source_module.verify_experiment013_source_manifest( normalized, args.repository_root, + git_executable=git_executable.path, ) if after_publish != normalized: raise CalibrationRunError("source identity changed after manifest publication") @@ -4020,6 +5554,7 @@ def _capture_manifest_mode(arguments: Sequence[str]) -> int | None: context="package import path", ) payload = capture_calibration_runtime_manifest( + git_executable_path=args.git_executable, base_runtime_root=args.base_runtime_root, package_roots=package_roots, package_import_paths=package_import_paths, @@ -4030,6 +5565,7 @@ def _capture_manifest_mode(arguments: Sequence[str]) -> int | None: args.model_id, args.revision, transformers_version=args.transformers_version, + token=False, ) if command != "capture-source-manifest": output = args.output.resolve() @@ -4092,6 +5628,26 @@ def _official_main( raise CalibrationRunError( f"{name} manifest bytes differ from the frozen identity bootstrap binding" ) + if args.fisher_h1_smoke: + if ( + args.prior_fisher_h1_smoke_report is not None + or args.prior_fisher_h1_smoke_complete_marker is not None + ): + raise CalibrationRunError( + "Fisher H=1 smoke mode forbids prior smoke prerequisite paths" + ) + prior_smoke_report_bytes = None + prior_smoke_complete_bytes = None + else: + if ( + args.prior_fisher_h1_smoke_report is None + or args.prior_fisher_h1_smoke_complete_marker is None + ): + raise CalibrationRunError( + "full calibration requires prior Fisher H=1 smoke report and marker paths" + ) + prior_smoke_report_bytes = args.prior_fisher_h1_smoke_report.read_bytes() + prior_smoke_complete_bytes = args.prior_fisher_h1_smoke_complete_marker.read_bytes() if ( _sha256( args.expected_runtime_manifest_sha256, @@ -4127,7 +5683,15 @@ def _official_main( requested_commit = _git_revision(args.source_commit, context="requested source commit") if requested_commit != bootstrap_source.source_commit: raise CalibrationRunError("CLI source commit differs from source-manifest commit") - _verify_repository_commit(args.repository_root, requested_commit) + source_git = bootstrap_source.manifest["git_executable"] + runtime_git = _authenticate_git_executable(runtime_context.git_executable_path) + if source_git != { + "sha256": runtime_git.sha256, + "size_bytes": runtime_git.size_bytes, + }: + raise CalibrationRunError( + "source and runtime manifests bind different Git executable bytes" + ) _AUTHENTICATED_CALIBRATION_API = _load_exact_source_module( CALIBRATION_API_MODULE, @@ -4153,6 +5717,7 @@ def _official_main( calibration_api=_AUTHENTICATED_CALIBRATION_API, interpreter_path=interpreter_path, package_roots=runtime_context.package_roots, + git_executable_path=runtime_context.git_executable_path, ) identity = services.backend.decode_identity(identity_bytes) if ( @@ -4169,7 +5734,9 @@ def _official_main( args.repository_root, ) if verified_source.get("source_commit") != requested_commit: - raise CalibrationRunError("verified source-manifest commit differs from current HEAD") + raise CalibrationRunError( + "verified source-manifest commit differs from requested frozen source commit" + ) runtime_manifest = parse_calibration_runtime_manifest(runtime_manifest_bytes) authenticated_runtime = services.authenticate_runtime(runtime_manifest) if authenticated_runtime.manifest_file_sha256 != bindings.runtime_manifest_file_sha256: @@ -4189,6 +5756,7 @@ def _official_main( }, runtime_authentication_context={ "base_runtime_root": runtime_context.base_runtime_root, + "git_executable": runtime_context.git_executable_path, "staged_interpreter": Path(interpreter_path), "package_runtime_roots": dict(runtime_context.package_roots), "package_import_paths": dict(runtime_context.package_import_paths), @@ -4217,6 +5785,9 @@ def _official_main( expected_runtime_manifest_sha256=args.expected_runtime_manifest_sha256, output_dir=args.output_dir, require_cuda=True, + fisher_h1_smoke=args.fisher_h1_smoke, + prior_fisher_h1_smoke_report_bytes=prior_smoke_report_bytes, + prior_fisher_h1_smoke_complete_bytes=prior_smoke_complete_bytes, ) result = run_calibration( config, @@ -4234,6 +5805,7 @@ def sealed_main( package_roots: Mapping[str, Path], package_import_paths: Mapping[str, str], interpreter_path: Path, + git_executable_path: Path, pycache_prefix: Path, ) -> int: """Run only after the stdlib bootstrap supplies explicit authenticated roots.""" @@ -4247,6 +5819,7 @@ def sealed_main( package_roots=package_roots, package_import_paths=package_import_paths, interpreter_path=interpreter_path, + git_executable_path=git_executable_path, pycache_prefix=pycache_prefix, ) if manifest.file_sha256 != _sha256( @@ -4272,12 +5845,13 @@ def sealed_main( base_runtime_root=runtime_context.base_runtime_root, package_roots=runtime_context.package_roots, interpreter_path=interpreter_path, + git_executable_path=git_executable_path, ) return result def main(argv: Sequence[str] | None = None) -> int: - """Expose metadata preparation only; official runs require the sealed launcher.""" + """Expose authenticated preparation only; official runs require the sealed launcher.""" arguments = list(sys.argv[1:] if argv is None else argv) capture_result = _capture_manifest_mode(arguments) diff --git a/scripts/screen_statelease_stage_a.py b/scripts/screen_statelease_stage_a.py index 5297abe..384f946 100644 --- a/scripts/screen_statelease_stage_a.py +++ b/scripts/screen_statelease_stage_a.py @@ -385,6 +385,7 @@ "src/recurquant/experiment013_parquet.py", "src/recurquant/experiment013_qwen35_adapter.py", "src/recurquant/experiment013_source.py", + "src/recurquant/experiment013_stage_a.py", "src/recurquant/finite_difference.py", "src/recurquant/fisher_sensitivity.py", "src/recurquant/horizon.py", diff --git a/scripts/screen_static_q468_stage_a.py b/scripts/screen_static_q468_stage_a.py new file mode 100644 index 0000000..5dbf2fc --- /dev/null +++ b/scripts/screen_static_q468_stage_a.py @@ -0,0 +1,4956 @@ +#!/usr/bin/env python3 +"""Authenticated one-run Experiment 013 Stage-A falsification screen. + +This evaluator is intentionally fail closed. It authenticates the promoted +resolver-v5 identity, the embedded eight-dependency calibration binding, the +split-half pass, the complete H0 source inventory, the sealed runtime, the +model tree, and the checked-in Parquet identity before reserving the one run. +Only after an empty-diff seal commit wins an atomic HEAD compare-and-swap may +the canonical materializer open the twelve Stage-A rows. + +Stage A is a falsification screen. A pass is not confirmation, a novelty or +selector-superiority result, a deployment result, or a breakthrough claim. +""" + +from __future__ import annotations + +import argparse +import base64 +import contextlib +import dataclasses +import hashlib +import importlib +import importlib.util +import json +import math +import os +import re +import shutil +import stat +import subprocess +import sys +import tempfile +import time +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from datetime import UTC, datetime +from pathlib import Path, PurePosixPath +from types import MappingProxyType, ModuleType +from typing import Any, Final, Protocol, cast + +REPOSITORY_ROOT: Final = Path(__file__).resolve().parents[1] +RUNNER_SOURCE_PATH: Final = "scripts/screen_static_q468_stage_a.py" +LAUNCHER_SOURCE_PATH: Final = "scripts/launch_static_q468_stage_a.py" +CALIBRATION_RUNNER_SOURCE_PATH: Final = "scripts/run_static_q468_calibration.py" +CAPTURE_SOURCE_PATH: Final = "scripts/capture_static_q468_identity_input.py" +RESOLVER_SOURCE_PATH: Final = "scripts/resolve_static_q468_identity.py" +SOURCE_MODULE_PATH: Final = "src/recurquant/experiment013_source.py" +STAGE_A_GATE_MODULE_PATH: Final = "src/recurquant/experiment013_stage_a.py" + +RUNNER_REVISION: Final = "experiment-013-static-q468-stage-a-runner-v3" +ATTEMPT_SCHEMA: Final = "recurquant.experiment013.stage-a-attempt.v2" +IDENTITY_ATTEMPT_LOCK_SCHEMA: Final = "recurquant.experiment013.stage-a-identity-attempt-lock.v3" +IDENTITY_ATTEMPT_LOCK_FIELDS: Final = frozenset( + { + "schema", + "runner_revision", + "created_at_utc", + "attempt_number", + "automatic_retry_authorized", + "h0_source_commit", + "h1_identity_commit", + "identity_repository_path", + "identity_file_sha256", + "one_run_seal_commit", + "one_run_seal_tree", + "one_run_marker", + "one_run_seal_message_sha256", + "calibration_binding_file_sha256", + "source_manifest_file_sha256", + "stage_a_input_bundle_manifest_file_sha256", + "execution_bindings", + "method_specs", + "expected_forward_count", + "claim_boundary", + "output_path", + "attempt_path", + "complete_path", + } +) +EXECUTION_ARTIFACT_KIND: Final = "recurquant_experiment013_stage_a_execution" +EXECUTION_ARTIFACT_SCHEMA: Final = 3 +IDENTITY_SCHEMA_VERSION: Final = 5 +BINDING_SCHEMA_VERSION: Final = 3 +ONE_RUN_MARKER: Final = "RecurQuant-One-Run: experiment013-stage-a-v1" +CLAIM_BOUNDARY: Final = ( + "Stage A is a falsification screen only. Passage is not confirmation, selector " + "superiority evidence, deployment evidence, state of the art, or a breakthrough claim." +) + +FP32_METHOD: Final = "fp32_reference" +UNIFORM_Q4_METHOD: Final = "rht_q468_uniform_q4" +UNIFORM_Q8_METHOD: Final = "rht_q468_uniform_q8" +Q48_METHOD: Final = "rht_q48_static_p14739" +STATIC_K27030_METHOD: Final = "rht_q468_static_k27030" +DYNAMIC_K27030_METHOD: Final = "rht_q468_dynamic_k27030" +MSE_K29334_METHOD: Final = "rht_q468_static_mse_k29334" +FISHER_K29334_METHOD: Final = "rht_q468_static_diag_empirical_fisher_h1_k29334" +PRIMARY_K29334_METHOD: Final = "rht_q468_static_k29334" +METHOD_ORDER: Final = ( + FP32_METHOD, + UNIFORM_Q4_METHOD, + UNIFORM_Q8_METHOD, + Q48_METHOD, + STATIC_K27030_METHOD, + DYNAMIC_K27030_METHOD, + MSE_K29334_METHOD, + FISHER_K29334_METHOD, + PRIMARY_K29334_METHOD, +) +PRESEAL_ENGINE_SMOKE_PROFILE: Final = "experiment-013-stage-a-preseal-engine-smoke-v3" +PRESEAL_ENGINE_SMOKE_PROMPT_TOKEN_COUNT: Final = 4_096 +PRESEAL_ENGINE_SMOKE_PROMPT_TOKEN_ID: Final = 1 +PRESEAL_ENGINE_SMOKE_TARGET_TOKEN_COUNT: Final = 128 +PRESEAL_ENGINE_SMOKE_TARGET: Final = (5,) * PRESEAL_ENGINE_SMOKE_TARGET_TOKEN_COUNT +EXPECTED_RECURRENT_RESIDENT_BYTES: Final = { + FP32_METHOD: 18_874_368, + UNIFORM_Q4_METHOD: 2_515_968, + UNIFORM_Q8_METHOD: 4_875_264, + Q48_METHOD: 3_454_664, + STATIC_K27030_METHOD: 3_380_928, + DYNAMIC_K27030_METHOD: 3_454_664, + MSE_K29334_METHOD: 3_454_664, + FISHER_K29334_METHOD: 3_454_664, + PRIMARY_K29334_METHOD: 3_454_664, +} +RAW_TOKEN_EVIDENCE_FIELDS: Final = frozenset( + { + "family", + "canonical_id", + "selection_rank", + "identity_record_sha256", + "method_id", + "transition_index", + "input_position", + "target_position", + "authenticated_transition_sha256", + "reference_nll", + "method_nll", + "excess_nll", + "kl", + "top1_agreement", + "local_codec_sse", + "trajectory_nmse", + "decode_model_forward_latency_ns", + "decode_cuda_diagnostic_peak_allocated_bytes", + "decode_cuda_diagnostic_peak_reserved_bytes", + "decode_logical_recurrent_resident_bytes", + "method_cumulative_cache_reported_workspace_high_water_sum_bytes", + "finite_checks", + } +) +METHOD_RUNTIME_FIELDS: Final = frozenset( + { + "family", + "canonical_id", + "selection_rank", + "identity_record_sha256", + "method_id", + "policy_file_sha256", + "policy_origin", + "prefill_diagnostics", + "max_cuda_diagnostic_peak_allocated_bytes_across_prefill_and_decode", + "max_cuda_diagnostic_peak_reserved_bytes_across_prefill_and_decode", + "wall_time_with_diagnostics_ns", + "storage", + } +) +PREFILL_DIAGNOSTIC_FIELDS: Final = frozenset( + { + "model_forward_latency_ns", + "cuda_diagnostic_peak_allocated_bytes", + "cuda_diagnostic_peak_reserved_bytes", + "logical_recurrent_resident_bytes", + "cache_reported_workspace_high_water_sum_bytes", + } +) +STORAGE_RECEIPT_FIELDS: Final = frozenset( + { + "logical_recurrent_resident_bytes", + "cache_reported_resident_bytes", + "cache_reported_expected_resident_bytes", + "raw_state_workspace_high_water_bytes", + "query_workspace_high_water_bytes", + "workspace_scope", + } +) + +EXECUTION_BINDING_FIELDS: Final = frozenset( + { + "repository_source_manifest_file_sha256", + "calibration_runtime_manifest_file_sha256", + "model_file_manifest_file_sha256", + "parquet_materialization_manifest_file_sha256", + } +) +BINDING_DEPENDENCY_NAMES: Final = frozenset( + { + "calibration_score_artifact", + "comparator_score_artifact", + "frozen_identity_artifact", + "split_half_stability_artifact", + "static_fisher_k29334_policy_artifact", + "static_k27030_policy_artifact", + "static_k29334_policy_artifact", + "static_mse_k29334_policy_artifact", + } +) +BINDING_FIELDS: Final = frozenset( + { + "calibration_identity_file_sha256", + "calibration_score_artifact_file_sha256", + "comparator_score_artifact_file_sha256", + "split_half_stability_artifact_file_sha256", + "static_fisher_k29334_policy_file_sha256", + "static_k27030_policy_file_sha256", + "static_k29334_policy_file_sha256", + "static_mse_k29334_policy_file_sha256", + } +) +REQUIRED_SOURCE_PATHS: Final = frozenset( + { + RUNNER_SOURCE_PATH, + LAUNCHER_SOURCE_PATH, + CALIBRATION_RUNNER_SOURCE_PATH, + CAPTURE_SOURCE_PATH, + RESOLVER_SOURCE_PATH, + SOURCE_MODULE_PATH, + STAGE_A_GATE_MODULE_PATH, + } +) + +OUTPUT_FILENAME: Final = "stage-a-result.json" +ATTEMPT_FILENAME: Final = "stage-a-attempt.json" +COMPLETE_FILENAME: Final = "STAGE_A_COMPLETE" +_SHA256_RE: Final = re.compile(r"[0-9a-f]{64}") +_SHA1_RE: Final = re.compile(r"[0-9a-f]{40}") +_WINDOWS_REPARSE_POINT: Final = 0x400 + + +class StageAError(RuntimeError): + """Raised when an authentication, execution, or one-run invariant fails.""" + + +class StageAStabilityFailure(StageAError): + """Raised before reservation when the frozen split-half gate did not pass.""" + + +@dataclass(frozen=True, slots=True) +class StageAConfig: + frozen_identity_path: Path + calibration_binding_path: Path + repository_source_manifest_path: Path + runtime_manifest_path: Path + model_file_manifest_path: Path + parquet_materialization_manifest_path: Path + model_root: Path + cache_root: Path + ruler_root: Path + input_bundle_root: Path + repository_root: Path + source_commit: str + identity_commit: str + output_dir: Path + expected_runtime_manifest_sha256: str + expected_model_file_manifest_sha256: str + expected_parquet_materialization_manifest_sha256: str + base_runtime_root: Path | None = None + package_roots: Mapping[str, Path] = field(default_factory=dict) + package_import_paths: Mapping[str, str] = field(default_factory=dict) + interpreter_path: Path | None = None + git_executable_path: Path | None = None + pycache_prefix: Path | None = None + + @property + def output_path(self) -> Path: + return self.output_dir / OUTPUT_FILENAME + + @property + def attempt_path(self) -> Path: + return self.output_dir / ATTEMPT_FILENAME + + @property + def complete_path(self) -> Path: + return self.output_dir / COMPLETE_FILENAME + + +@dataclass(frozen=True, slots=True) +class BootstrapIdentity: + file_sha256: str + canonical_evidence_sha256: str + execution_bindings: Mapping[str, str] + calibration_binding: Mapping[str, str] + expected_forward_count: int + + +@dataclass(frozen=True, slots=True) +class StageAMethodSpec: + method_id: str + policy: object | None + policy_file_sha256: str | None + origin: str + + +@dataclass(frozen=True, slots=True) +class AuthenticatedStageA: + bootstrap_identity: BootstrapIdentity + identity: object + binding: object + dependency_bytes: Mapping[str, bytes] + execution_artifact_bytes: Mapping[str, bytes] + source_manifest: Mapping[str, object] + source_manifest_file_sha256: str + source_commit: str + input_bundle: object | None + input_bundle_manifest_file_sha256: str | None + model_manifest: object + authenticated_model_files: object + runtime_manifest: object + authenticated_runtime: object + resolver: ModuleType + capture: ModuleType + calibration_runner: ModuleType + source_module: ModuleType + methods: tuple[StageAMethodSpec, ...] + + +@dataclass(frozen=True, slots=True) +class AttemptReservation: + receipt: Mapping[str, object] + h1_commit: str + seal_commit: str + tree: str + + +@dataclass(frozen=True, slots=True) +class ForwardObservation: + """One authenticated model forward, including private comparison logits.""" + + position: int + target_token_id: int + comparison_logits: object = field(repr=False) + target_nll: float = 0.0 + top1_token_id: int = 0 + local_codec_sse: float = 0.0 + trajectory_nmse: float = 0.0 + latency_ns: int = 0 + peak_allocated_bytes: int = 0 + peak_reserved_bytes: int = 0 + resident_bytes: int = 0 + transient_bytes: int = 0 + + +@dataclass(frozen=True, slots=True) +class _PresealSmokeSequence: + identity_record_sha256: str + target_token_ids: tuple[int, ...] + + +@dataclass(frozen=True, slots=True) +class StageAEvaluation: + examples: tuple[object, ...] + gate_rows: tuple[object, ...] + raw_rows: tuple[Mapping[str, object], ...] + forward_count: int + method_runtime: tuple[Mapping[str, object], ...] + device_runtime: Mapping[str, object] + + +class StageAEngine(Protocol): + """Model-specific surface; orchestration owns order and forward counting.""" + + def load_model(self, authenticated_model_files: object) -> object: ... + + def close_model(self, model: object) -> None: ... + + def begin_method( + self, + model: object, + method: StageAMethodSpec, + sequence: object, + ) -> object: ... + + def prefill( + self, + session: object, + *, + prompt_token_ids: tuple[int, ...], + first_target_token_id: int, + position: int, + ) -> ForwardObservation: ... + + def step( + self, + session: object, + *, + input_token_id: int, + target_token_id: int, + position: int, + ) -> ForwardObservation: ... + + def end_method(self, session: object) -> Mapping[str, object]: ... + + def runtime_snapshot(self, model: object) -> Mapping[str, object]: ... + + +@dataclass(frozen=True, slots=True) +class StageAServices: + authenticate: Callable[[StageAConfig], AuthenticatedStageA] + reauthenticate: Callable[[StageAConfig, AuthenticatedStageA, AttemptReservation | None], None] + preseal_smoke: Callable[[AuthenticatedStageA], Mapping[str, object]] + reserve: Callable[[StageAConfig, AuthenticatedStageA], AttemptReservation] + materialize: Callable[[StageAConfig, AuthenticatedStageA], object] + engine: StageAEngine + persist_receipt: Callable[ + [StageAConfig, AttemptReservation, Mapping[str, object]], AttemptReservation + ] + publish: Callable[ + [StageAConfig, AttemptReservation, bytes, Mapping[str, object]], Mapping[str, object] + ] + record_failure: Callable[[StageAConfig, AttemptReservation | None, BaseException, str], None] + + +def canonical_json_bytes(value: object) -> bytes: + try: + return ( + json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ) + + "\n" + ).encode("utf-8") + except (TypeError, ValueError) as error: + raise StageAError("value is not finite canonical JSON data") from error + + +def pretty_json_bytes(value: object) -> bytes: + try: + return (json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n").encode("utf-8") + except (TypeError, ValueError) as error: + raise StageAError("value is not finite pretty JSON data") from error + + +def sha256_bytes(value: bytes) -> str: + if not isinstance(value, bytes): + raise TypeError("SHA-256 input must be bytes") + return hashlib.sha256(value).hexdigest() + + +def _require_sha256(value: object, *, context: str) -> str: + if not isinstance(value, str) or _SHA256_RE.fullmatch(value) is None: + raise StageAError(f"{context} must be a lowercase SHA-256 digest") + return value + + +def _require_sha1(value: object, *, context: str) -> str: + if not isinstance(value, str) or _SHA1_RE.fullmatch(value) is None: + raise StageAError(f"{context} must be a lowercase SHA-1 object ID") + return value + + +def _strict_json(data: bytes, *, context: str) -> dict[str, Any]: + if not isinstance(data, bytes): + raise TypeError(f"{context} must be bytes") + + def unique(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise StageAError(f"{context} contains a duplicate key") + result[key] = value + return result + + def reject_constant(value: str) -> None: + raise StageAError(f"{context} contains a non-finite JSON constant: {value}") + + try: + value = json.loads( + data.decode("utf-8"), + object_pairs_hook=unique, + parse_constant=reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise StageAError(f"{context} is not strict UTF-8 JSON") from error + if not isinstance(value, dict): + raise StageAError(f"{context} must be a JSON object") + return value + + +def _exact_fields( + value: Mapping[str, object], expected: set[str] | frozenset[str], *, context: str +) -> None: + if set(value) != set(expected): + raise StageAError(f"{context} fields differ from the frozen schema") + + +def _safe_relative_path(value: object, *, context: str) -> str: + if not isinstance(value, str) or not value or value != value.strip(): + raise StageAError(f"{context} is not a canonical relative path") + if any(character in value for character in ("\\", "\0", "\n", "\r", ":")): + raise StageAError(f"{context} is not a safe POSIX path") + path = PurePosixPath(value) + if ( + path.is_absolute() + or path.as_posix() != value + or any(part in {"", ".", ".."} for part in path.parts) + ): + raise StageAError(f"{context} is not a canonical relative path") + return value + + +def _is_link_or_reparse(path: Path) -> bool: + try: + status = path.lstat() + except OSError as error: + raise StageAError(f"required path is unavailable: {path}") from error + return path.is_symlink() or bool( + getattr(status, "st_file_attributes", 0) & _WINDOWS_REPARSE_POINT + ) + + +def _stable_file_bytes(path: Path, *, context: str) -> bytes: + absolute = Path(os.path.abspath(path)) + if _is_link_or_reparse(absolute): + raise StageAError(f"{context} is a link or reparse point") + try: + before = absolute.stat() + data = absolute.read_bytes() + after = absolute.stat() + except OSError as error: + raise StageAError(f"cannot read {context}") from error + if ( + not stat.S_ISREG(before.st_mode) + or not stat.S_ISREG(after.st_mode) + or before.st_size != after.st_size + or before.st_mtime_ns != after.st_mtime_ns + or len(data) != after.st_size + or _is_link_or_reparse(absolute) + ): + raise StageAError(f"{context} changed while it was authenticated") + return data + + +def bootstrap_stage_a_identity(data: bytes) -> BootstrapIdentity: + """Strict stdlib-only v5 promotion check performed before dependency imports.""" + + root = _strict_json(data, context="frozen Stage-A identity") + _exact_fields(root, {"canonical_evidence_sha256", "evidence"}, context="identity wrapper") + if canonical_json_bytes(root) != data: + raise StageAError("frozen Stage-A identity is not canonical JSON") + evidence = root.get("evidence") + if not isinstance(evidence, dict): + raise StageAError("frozen Stage-A identity evidence is missing") + if ( + type(evidence.get("schema_version")) is not int + or evidence.get("schema_version") != IDENTITY_SCHEMA_VERSION + or evidence.get("identity_schema") != "recurquant.experiment013.identity-frozen.v5" + or evidence.get("status") != "frozen" + or evidence.get("phase") != "stage_a" + or evidence.get("identity_only") is not True + or evidence.get("promotion_required") is not False + ): + raise StageAError("identity is not the promoted resolver-v5 Stage-A artifact") + promotion = evidence.get("promotion") + if not isinstance(promotion, dict) or promotion.get("explicit") is not True: + raise StageAError("Stage-A identity lacks an explicit promotion") + canonical_hash = _require_sha256( + root.get("canonical_evidence_sha256"), context="identity canonical evidence SHA-256" + ) + if canonical_hash != sha256_bytes(canonical_json_bytes(evidence)): + raise StageAError("frozen Stage-A identity canonical evidence hash drifted") + execution = evidence.get("execution_bindings") + if not isinstance(execution, dict): + raise StageAError("frozen Stage-A identity execution bindings are missing") + _exact_fields(execution, EXECUTION_BINDING_FIELDS, context="identity execution bindings") + normalized_execution = { + name: _require_sha256(execution[name], context=f"execution binding {name}") + for name in sorted(EXECUTION_BINDING_FIELDS) + } + calibration = evidence.get("calibration_binding") + if not isinstance(calibration, dict): + raise StageAError("frozen Stage-A identity calibration binding is missing") + _exact_fields(calibration, BINDING_FIELDS, context="identity calibration binding") + normalized_calibration = { + name: _require_sha256(calibration[name], context=f"calibration binding {name}") + for name in sorted(BINDING_FIELDS) + } + records = evidence.get("records") + if not isinstance(records, list) or len(records) != 12: + raise StageAError("frozen Stage-A identity must contain exactly twelve records") + expected_order = [ + (family, rank) for family in ("pg19", "ruler", "humaneval_plus") for rank in range(4) + ] + observed_order: list[tuple[object, object]] = [] + continuation_counts: list[int] = [] + for index, record in enumerate(records): + if not isinstance(record, dict): + raise StageAError(f"identity records[{index}] must be an object") + observed_order.append((record.get("family"), record.get("selection_rank"))) + span = record.get("token_span") + if not isinstance(span, dict): + raise StageAError(f"identity records[{index}] token span is missing") + scored_start = span.get("scored_start") + scored_stop = span.get("scored_stop") + cache_start = span.get("cache_exposed_start") + cache_stop = span.get("cache_exposed_stop") + if any( + isinstance(value, bool) or not isinstance(value, int) or value < 0 + for value in (scored_start, scored_stop, cache_start, cache_stop) + ): + raise StageAError(f"identity records[{index}] token span is invalid") + assert isinstance(scored_start, int) and isinstance(scored_stop, int) + assert isinstance(cache_start, int) and isinstance(cache_stop, int) + continuation = scored_stop - scored_start + if continuation < 2 or cache_start != scored_start + 1 or cache_stop != scored_stop: + raise StageAError(f"identity records[{index}] cache-exposed span drifted") + continuation_counts.append(continuation) + if observed_order != expected_order: + raise StageAError("Stage-A identity record order drifted") + expected_forward_count = len(METHOD_ORDER) * sum(1 + count - 1 for count in continuation_counts) + return BootstrapIdentity( + file_sha256=sha256_bytes(data), + canonical_evidence_sha256=canonical_hash, + execution_bindings=MappingProxyType(normalized_execution), + calibration_binding=MappingProxyType(normalized_calibration), + expected_forward_count=expected_forward_count, + ) + + +def _bootstrap_source_manifest(data: bytes) -> dict[str, object]: + root = _strict_json(data, context="repository source manifest") + _exact_fields( + root, + { + "canonical_manifest_sha256", + "git_executable", + "object_format", + "paths", + "profile", + "repository_binding", + "schema", + "source_commit", + }, + context="repository source manifest", + ) + if pretty_json_bytes(root) != data: + raise StageAError("repository source manifest is not canonical pretty JSON") + if ( + root.get("schema") != "recurquant.experiment013.source-manifest.v2" + or root.get("profile") != "experiment-013-static-q468-frozen-source-v2" + or root.get("object_format") != "sha1" + ): + raise StageAError("repository source manifest object format drifted") + raw_git = root.get("git_executable") + if not isinstance(raw_git, dict): + raise StageAError("repository source Git executable identity is missing") + _exact_fields(raw_git, {"sha256", "size_bytes"}, context="source Git executable") + size_bytes = raw_git["size_bytes"] + if isinstance(size_bytes, bool) or not isinstance(size_bytes, int) or size_bytes <= 0: + raise StageAError("repository source Git executable size is invalid") + git_executable = { + "sha256": _require_sha256(raw_git["sha256"], context="source Git executable SHA-256"), + "size_bytes": size_bytes, + } + _require_sha1(root.get("source_commit"), context="source commit") + claimed = _require_sha256( + root.get("canonical_manifest_sha256"), context="source manifest self-hash" + ) + payload = dict(root) + payload.pop("canonical_manifest_sha256") + if claimed != sha256_bytes(pretty_json_bytes(payload)): + raise StageAError("repository source manifest self-hash drifted") + raw_paths = root.get("paths") + if not isinstance(raw_paths, list) or not raw_paths: + raise StageAError("repository source manifest has no path inventory") + paths: list[dict[str, object]] = [] + for index, entry in enumerate(raw_paths): + if not isinstance(entry, dict): + raise StageAError(f"source paths[{index}] must be an object") + _exact_fields( + entry, + {"git_blob_oid", "index_blob_oid", "mode", "path", "raw_sha256", "worktree_blob_oid"}, + context=f"source paths[{index}]", + ) + relative = _safe_relative_path(entry["path"], context=f"source paths[{index}].path") + identities = { + _require_sha1(entry[name], context=f"source paths[{index}].{name}") + for name in ("git_blob_oid", "index_blob_oid", "worktree_blob_oid") + } + if len(identities) != 1: + raise StageAError(f"source paths[{index}] commit/index/worktree identities differ") + if entry["mode"] not in {"100644", "100755"}: + raise StageAError(f"source paths[{index}] mode is not a regular file") + paths.append( + { + "path": relative, + "raw_sha256": _require_sha256( + entry["raw_sha256"], context=f"source paths[{index}].raw_sha256" + ), + } + ) + rendered = [cast(str, entry["path"]) for entry in paths] + if rendered != sorted(rendered) or len({item.casefold() for item in rendered}) != len(rendered): + raise StageAError("repository source path inventory is not unique and sorted") + if not set(rendered) >= REQUIRED_SOURCE_PATHS: + raise StageAError("repository source manifest omits Stage-A implementation paths") + return { + "document": root, + "entries": paths, + "git_executable": git_executable, + "source_commit": root["source_commit"], + } + + +def _source_entries(manifest: Mapping[str, object]) -> dict[str, Mapping[str, object]]: + return { + cast(str, entry["path"]): cast(Mapping[str, object], entry) + for entry in cast(Sequence[Mapping[str, object]], manifest["entries"]) + } + + +def _verify_source_bytes(manifest: Mapping[str, object], repository_root: Path) -> None: + root = Path(os.path.abspath(repository_root)) + try: + resolved_root = root.resolve(strict=True) + except OSError as error: + raise StageAError("repository root is unavailable") from error + if not resolved_root.is_dir() or _is_link_or_reparse(resolved_root): + raise StageAError("repository root is not a stable directory") + for entry in cast(Sequence[Mapping[str, object]], manifest["entries"]): + relative = cast(str, entry["path"]) + path = resolved_root.joinpath(*PurePosixPath(relative).parts) + try: + resolved = path.resolve(strict=True) + resolved.relative_to(resolved_root) + except (OSError, ValueError) as error: + raise StageAError(f"source path escapes or is unavailable: {relative}") from error + if ( + sha256_bytes(_stable_file_bytes(resolved, context=f"source file {relative}")) + != entry["raw_sha256"] + ): + raise StageAError(f"source bytes drifted: {relative}") + + +def _install_source_namespace(repository_root: Path) -> None: + existing = sys.modules.get("recurquant") + package_root = repository_root.resolve(strict=True) / "src" / "recurquant" + if existing is not None: + paths = getattr(existing, "__path__", None) + if paths is None or [str(package_root)] != list(paths): + raise StageAError("recurquant namespace was preloaded from an unauthenticated path") + return + package = ModuleType("recurquant") + package.__package__ = "recurquant" + package.__path__ = [str(package_root)] # type: ignore[attr-defined] + package.__spec__ = importlib.machinery.ModuleSpec("recurquant", loader=None, is_package=True) + package.__spec__.submodule_search_locations = [str(package_root)] + sys.modules["recurquant"] = package + + +def _load_exact_module( + name: str, + relative: str, + *, + repository_root: Path, + entries: Mapping[str, Mapping[str, object]], +) -> ModuleType: + if name in sys.modules: + module = sys.modules[name] + raw_file = getattr(module, "__file__", None) + if raw_file is None: + raise StageAError(f"module {name} was preloaded without a source file") + path = Path(raw_file).resolve(strict=True) + expected = (repository_root / PurePosixPath(relative)).resolve(strict=True) + if path != expected or sha256_bytes(path.read_bytes()) != entries[relative]["raw_sha256"]: + raise StageAError(f"module {name} was preloaded from unauthenticated bytes") + return cast(ModuleType, module) + path = (repository_root / PurePosixPath(relative)).resolve(strict=True) + if ( + sha256_bytes(_stable_file_bytes(path, context=f"module {name}")) + != entries[relative]["raw_sha256"] + ): + raise StageAError(f"module source bytes drifted before load: {relative}") + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise StageAError(f"cannot construct module spec for {relative}") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + try: + spec.loader.exec_module(module) + except BaseException: + sys.modules.pop(name, None) + raise + if ( + sha256_bytes(_stable_file_bytes(path, context=f"module {name}")) + != entries[relative]["raw_sha256"] + ): + raise StageAError(f"module source bytes drifted during load: {relative}") + return module + + +def _decode_binding_dependencies(data: bytes) -> dict[str, bytes]: + root = _strict_json(data, context="Stage-A calibration binding") + if root.get("schema_version") != BINDING_SCHEMA_VERSION: + raise StageAError("Stage-A calibration binding is not schema v3") + evidence = root.get("evidence") + if not isinstance(evidence, dict): + raise StageAError("Stage-A calibration binding evidence is missing") + encoded = evidence.get("dependencies_base64") + hashes = evidence.get("dependency_file_sha256") + if not isinstance(encoded, dict) or not isinstance(hashes, dict): + raise StageAError("Stage-A calibration binding dependencies are missing") + _exact_fields(encoded, BINDING_DEPENDENCY_NAMES, context="binding dependencies") + _exact_fields(hashes, BINDING_DEPENDENCY_NAMES, context="binding dependency hashes") + result: dict[str, bytes] = {} + for name in sorted(BINDING_DEPENDENCY_NAMES): + value = encoded[name] + if not isinstance(value, str): + raise StageAError(f"binding dependency {name} is not base64 text") + try: + decoded = base64.b64decode(value, validate=True) + except ValueError as error: + raise StageAError(f"binding dependency {name} is not canonical base64") from error + if base64.b64encode(decoded).decode("ascii") != value: + raise StageAError(f"binding dependency {name} base64 is not canonical") + expected = _require_sha256(hashes[name], context=f"binding dependency {name} hash") + if sha256_bytes(decoded) != expected: + raise StageAError(f"binding dependency {name} bytes differ from their hash") + result[name] = decoded + return result + + +def _binding_field_for_dependency(name: str) -> str: + return { + "frozen_identity_artifact": "calibration_identity_file_sha256", + "calibration_score_artifact": "calibration_score_artifact_file_sha256", + "split_half_stability_artifact": "split_half_stability_artifact_file_sha256", + "static_k27030_policy_artifact": "static_k27030_policy_file_sha256", + "static_k29334_policy_artifact": "static_k29334_policy_file_sha256", + "comparator_score_artifact": "comparator_score_artifact_file_sha256", + "static_fisher_k29334_policy_artifact": ("static_fisher_k29334_policy_file_sha256"), + "static_mse_k29334_policy_artifact": "static_mse_k29334_policy_file_sha256", + }[name] + + +def reconstruct_stage_a_methods( + *, + dependency_bytes: Mapping[str, bytes], + frozen_stage_a_identity: object, + source_commit: str, +) -> tuple[StageAMethodSpec, ...]: + """Reconstruct uniform and Q48 policies only from bound candidate scores.""" + + if set(dependency_bytes) != BINDING_DEPENDENCY_NAMES: + raise StageAError("Stage-A reconstruction requires the exact eight dependencies") + static = importlib.import_module("recurquant.static_q468") + calibration = importlib.import_module("recurquant.static_q468_calibration") + scores = calibration.deserialize_calibration_score_artifact( + dependency_bytes["calibration_score_artifact"] + ) + identity_hash = cast( + str, frozen_stage_a_identity.calibration_binding["calibration_identity_file_sha256"] + ) + tokenizer_hash = cast(str, frozen_stage_a_identity.tokenizer_manifest_sha256) + if scores.calibration_identity_sha256 != identity_hash: + raise StageAError("candidate scores differ from the Stage-A calibration identity binding") + geometry = scores.geometry + common = { + "geometry": geometry, + "calibration_manifest_sha256": scores.aggregate.sequence_score_manifest_sha256, + "identity_artifact_sha256": identity_hash, + "tokenizer_manifest_sha256": tokenizer_hash, + "source_commit": source_commit, + "calibration_scores_sha256": scores.calibration_scores_sha256, + } + uniform_q4 = static.build_static_rht_q468_policy( + scores.aggregate.d4, + scores.aggregate.d6, + scores.aggregate.d8, + marginal_steps=0, + method_id=UNIFORM_Q4_METHOD, + **common, + ) + uniform_q8 = static.build_static_rht_q468_policy( + scores.aggregate.d4, + scores.aggregate.d6, + scores.aggregate.d8, + marginal_steps=2 * geometry.total_rows, + method_id=UNIFORM_Q8_METHOD, + **common, + ) + q48_scores_hash = static.static_q48_distortion_sha256( + scores.aggregate.d4, + scores.aggregate.d8, + geometry=geometry, + ) + q48 = static.build_static_rht_q48_policy( + scores.aggregate.d4, + scores.aggregate.d8, + geometry=geometry, + promoted_rows=static.FROZEN_STATIC_Q48_PROMOTIONS, + calibration_manifest_sha256=scores.aggregate.sequence_score_manifest_sha256, + identity_artifact_sha256=identity_hash, + tokenizer_manifest_sha256=tokenizer_hash, + source_commit=source_commit, + calibration_scores_sha256=q48_scores_hash, + method_id=Q48_METHOD, + ) + supplied = { + STATIC_K27030_METHOD: dependency_bytes["static_k27030_policy_artifact"], + MSE_K29334_METHOD: dependency_bytes["static_mse_k29334_policy_artifact"], + FISHER_K29334_METHOD: dependency_bytes["static_fisher_k29334_policy_artifact"], + PRIMARY_K29334_METHOD: dependency_bytes["static_k29334_policy_artifact"], + } + decoded: dict[str, object] = {} + for method_id, payload in supplied.items(): + policy = static.deserialize_static_rht_q468_policy(payload) + if policy.method_id != method_id or policy.source_commit != source_commit: + raise StageAError(f"bound policy identity drifted for {method_id}") + decoded[method_id] = policy + policies: dict[str, object | None] = { + FP32_METHOD: None, + UNIFORM_Q4_METHOD: uniform_q4, + UNIFORM_Q8_METHOD: uniform_q8, + Q48_METHOD: q48, + STATIC_K27030_METHOD: decoded[STATIC_K27030_METHOD], + DYNAMIC_K27030_METHOD: None, + MSE_K29334_METHOD: decoded[MSE_K29334_METHOD], + FISHER_K29334_METHOD: decoded[FISHER_K29334_METHOD], + PRIMARY_K29334_METHOD: decoded[PRIMARY_K29334_METHOD], + } + origins = { + FP32_METHOD: "runtime_reference", + UNIFORM_Q4_METHOD: "reconstructed_candidate_scores_k0", + UNIFORM_Q8_METHOD: f"reconstructed_candidate_scores_k{2 * geometry.total_rows}", + Q48_METHOD: "reconstructed_candidate_scores_p14739", + STATIC_K27030_METHOD: "embedded_binding_v3", + DYNAMIC_K27030_METHOD: "frozen_runtime_baseline", + MSE_K29334_METHOD: "embedded_binding_v3", + FISHER_K29334_METHOD: "embedded_binding_v3", + PRIMARY_K29334_METHOD: "embedded_binding_v3", + } + result: list[StageAMethodSpec] = [] + for method_id in METHOD_ORDER: + policy = policies[method_id] + if method_id in {UNIFORM_Q4_METHOD, UNIFORM_Q8_METHOD}: + serialized = static.serialize_static_rht_q468_policy(policy) + digest = sha256_bytes(serialized) + elif method_id == Q48_METHOD: + serialized = static.serialize_static_rht_q48_policy(policy) + digest = sha256_bytes(serialized) + elif method_id in supplied: + digest = sha256_bytes(supplied[method_id]) + else: + digest = None + result.append(StageAMethodSpec(method_id, policy, digest, origins[method_id])) + if tuple(item.method_id for item in result) != METHOD_ORDER: + raise RuntimeError("internal Stage-A method order drifted") + return tuple(result) + + +def _git_environment() -> dict[str, str]: + inherited = {key.upper(): (key, value) for key, value in os.environ.items()} + environment = { + inherited[name][0]: inherited[name][1] + for name in ("SYSTEMROOT", "WINDIR", "COMSPEC") + if name in inherited + } + environment.update( + { + "GIT_NO_REPLACE_OBJECTS": "1", + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_COUNT": "2", + "GIT_CONFIG_KEY_0": "core.hooksPath", + "GIT_CONFIG_VALUE_0": os.devnull, + "GIT_CONFIG_KEY_1": "core.fsmonitor", + "GIT_CONFIG_VALUE_1": "false", + "GIT_AUTHOR_NAME": "RecurQuant Experiment 013", + "GIT_AUTHOR_EMAIL": "experiment013@invalid", + "GIT_COMMITTER_NAME": "RecurQuant Experiment 013", + "GIT_COMMITTER_EMAIL": "experiment013@invalid", + "LC_ALL": "C", + "LANG": "C", + } + ) + return environment + + +def _authenticated_git_executable(path: Path | None) -> Path: + selected: str | os.PathLike[str] + if path is None: + discovered = shutil.which("git") + if discovered is None: + raise StageAError("Git executable is unavailable") + selected = discovered + else: + selected = path + try: + resolved = Path(selected).resolve(strict=True) + except OSError as error: + raise StageAError("Git executable is unavailable") from error + if resolved.name.casefold() == "git.exe" and resolved.parent.name.casefold() == "cmd": + try: + resolved = (resolved.parent.parent / "mingw64" / "bin" / "git.exe").resolve(strict=True) + except OSError as error: + raise StageAError( + "Git-for-Windows cmd shim has no canonical mingw64 executable" + ) from error + current = Path(resolved.anchor) + for part in resolved.parts[1:]: + current /= part + if _is_link_or_reparse(current): + raise StageAError("Git executable traverses a link or reparse point") + if not resolved.is_file() or _is_link_or_reparse(resolved): + raise StageAError("Git executable must be a regular non-link file") + return resolved + + +def _git_process( + git_executable_path: Path | None, + root: Path, + *arguments: str, + input_bytes: bytes | None = None, +) -> subprocess.CompletedProcess[bytes]: + git_executable = _authenticated_git_executable(git_executable_path) + try: + return subprocess.run( + [str(git_executable), *arguments], + cwd=root, + input=input_bytes, + capture_output=True, + check=False, + env=_git_environment(), + timeout=30, + ) + except (OSError, subprocess.TimeoutExpired) as error: + raise StageAError(f"git {arguments[0]} could not be executed") from error + + +def _git( + git_executable_path: Path | None, + root: Path, + *arguments: str, + input_bytes: bytes | None = None, +) -> str: + result = _git_process(git_executable_path, root, *arguments, input_bytes=input_bytes) + if result.returncode != 0: + detail = result.stderr.decode("utf-8", errors="replace").strip() + raise StageAError(f"git {' '.join(arguments)} failed" + (f": {detail}" if detail else "")) + try: + return result.stdout.decode("utf-8").strip() + except UnicodeDecodeError as error: + raise StageAError("Git returned non-UTF-8 output") from error + + +def _identity_attempt_lock_path( + root: Path, + identity_file_sha256: str, + *, + git_executable_path: Path | None, +) -> Path: + identity_hash = _require_sha256( + identity_file_sha256, + context="identity-scoped attempt lock identity SHA-256", + ) + raw_common = _git( + git_executable_path, + root, + "rev-parse", + "--path-format=absolute", + "--git-common-dir", + ) + common = Path(raw_common) + if not common.is_absolute(): + raise StageAError("Git common directory is not absolute") + common = Path(os.path.abspath(common)) + if not common.is_dir() or _is_link_or_reparse(common): + raise StageAError("Git common directory is not a stable directory") + parent = common + for name in ("recurquant", "experiment013-stage-a"): + child = parent / name + try: + child.mkdir() + except FileExistsError: + pass + except OSError as error: + raise StageAError("cannot create the identity-scoped attempt-lock directory") from error + if not child.is_dir() or _is_link_or_reparse(child): + raise StageAError("identity-scoped attempt-lock directory is unsafe") + parent = child + return parent / f"{identity_hash}.attempt.json" + + +def _reject_prior_identity_seal( + root: Path, + *, + identity_file_sha256: str, + git_executable_path: Path | None, +) -> None: + identity_line = f"Stage-A-Identity: {identity_file_sha256}" + prior = _git( + git_executable_path, + root, + "log", + "--all", + "--reflog", + "--format=%H", + "--fixed-strings", + "--all-match", + f"--grep={ONE_RUN_MARKER}", + f"--grep={identity_line}", + ) + if prior: + raise StageAError("this Stage-A identity already has a one-run seal in Git history") + + +def _assert_tracked_identity_bytes(config: StageAConfig, identity_bytes: bytes) -> str: + root = config.repository_root.resolve(strict=True) + identity = config.frozen_identity_path.resolve(strict=True) + try: + relative = identity.relative_to(root).as_posix() + except ValueError as error: + raise StageAError( + "frozen Stage-A identity must be tracked inside the repository" + ) from error + relative = _safe_relative_path(relative, context="tracked Stage-A identity path") + head = _require_sha1( + _git(config.git_executable_path, root, "rev-parse", "HEAD"), + context="repository HEAD", + ) + identity_commit = _require_sha1(config.identity_commit, context="identity commit") + if head != identity_commit: + raise StageAError("HEAD must equal the explicit Stage-A identity authorization commit") + show = _git_process( + config.git_executable_path, + root, + "show", + f"{identity_commit}:{relative}", + ) + if show.returncode != 0 or show.stdout != identity_bytes: + raise StageAError("tracked identity bytes differ from the identity authorization commit") + index = _git_process(config.git_executable_path, root, "show", f":{relative}") + if index.returncode != 0 or index.stdout != identity_bytes: + raise StageAError("tracked identity bytes differ from the Git index") + if _stable_file_bytes(identity, context="tracked Stage-A identity") != identity_bytes: + raise StageAError("tracked identity bytes differ from the worktree") + return relative + + +def _assert_output_paths_isolated(config: StageAConfig) -> None: + """Require a stable output directory and ignore repository-local evidence.""" + + root = config.repository_root.resolve(strict=True) + output_candidate = Path(os.path.abspath(config.output_dir)) + + def nested(path: Path, possible_parent: Path) -> bool: + try: + path.relative_to(possible_parent) + except ValueError: + return False + return True + + output_paths = ( + output_candidate, + Path(os.path.abspath(config.output_path)), + Path(os.path.abspath(config.attempt_path)), + Path(os.path.abspath(config.complete_path)), + ) + protected_roots = { + "input bundle": Path(os.path.abspath(config.input_bundle_root)), + "model": Path(os.path.abspath(config.model_root)), + **( + {"base runtime": Path(os.path.abspath(config.base_runtime_root))} + if config.base_runtime_root is not None + else {} + ), + **{ + f"package runtime {name}": Path(os.path.abspath(path)) + for name, path in config.package_roots.items() + }, + } + for protected_name, protected_root in protected_roots.items(): + for path in output_paths: + if nested(path, protected_root) or nested(protected_root, path): + raise StageAError( + f"Stage-A {protected_name} and output evidence paths must not overlap" + ) + output_dir = _safe_directory(config.output_dir, create=True) + for path in (config.output_path, config.attempt_path, config.complete_path): + absolute = Path(os.path.abspath(path)) + if absolute.parent != output_dir: + raise StageAError("Stage-A output paths escaped the authenticated output directory") + try: + relative = absolute.relative_to(root).as_posix() + except ValueError: + continue + relative = _safe_relative_path(relative, context="repository-local Stage-A output path") + ignored = _git_process( + config.git_executable_path, + root, + "check-ignore", + "--quiet", + "--no-index", + "--", + relative, + ) + if ignored.returncode != 0: + raise StageAError( + "repository-local Stage-A outputs must be ignored before one-run reservation" + ) + + +def _safe_directory(path: Path, *, create: bool) -> Path: + """Validate every directory component without resolving through links/reparse points.""" + + absolute = Path(os.path.abspath(path)) + if not absolute.is_absolute() or not absolute.anchor: + raise StageAError("Stage-A directory is not absolute") + current = Path(absolute.anchor) + components = absolute.parts[1:] + for component in components: + current = current / component + if not os.path.lexists(current): + if not create: + raise StageAError(f"required Stage-A directory is absent: {current}") + try: + current.mkdir() + except FileExistsError: + pass + except OSError as error: + raise StageAError(f"cannot create Stage-A directory: {current}") from error + if _is_link_or_reparse(current): + raise StageAError(f"Stage-A directory is a link or reparse point: {current}") + try: + status = current.stat() + except OSError as error: + raise StageAError(f"cannot authenticate Stage-A directory: {current}") from error + if not stat.S_ISDIR(status.st_mode): + raise StageAError(f"Stage-A directory component is not a directory: {current}") + return absolute + + +def _exclusive_write(path: Path, payload: bytes) -> None: + absolute = Path(os.path.abspath(path)) + parent = _safe_directory(absolute.parent, create=True) + if absolute.parent != parent: + raise StageAError("exclusive write path escaped its authenticated parent") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + descriptor = os.open(absolute, flags, 0o600) + try: + with os.fdopen(descriptor, "wb", closefd=True) as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + _safe_directory(parent, create=False) + if _is_link_or_reparse(absolute) or not absolute.is_file(): + raise StageAError("exclusive write did not create a stable regular file") + except BaseException: + with contextlib.suppress(OSError): + absolute.unlink() + raise + + +def _atomic_replace_owned(path: Path, payload: bytes) -> None: + absolute = Path(os.path.abspath(path)) + parent = _safe_directory(absolute.parent, create=False) + if not absolute.is_file() or _is_link_or_reparse(absolute): + raise StageAError("owned receipt disappeared or became unsafe") + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{absolute.name}.", suffix=".tmp", dir=parent + ) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + _safe_directory(parent, create=False) + os.replace(temporary, absolute) + _safe_directory(parent, create=False) + if _is_link_or_reparse(absolute) or not absolute.is_file(): + raise StageAError("owned receipt replacement became unsafe") + finally: + with contextlib.suppress(OSError): + temporary.unlink() + + +def _atomic_publish_new(path: Path, payload: bytes) -> None: + absolute = Path(os.path.abspath(path)) + parent = _safe_directory(absolute.parent, create=True) + descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{absolute.name}.", suffix=".tmp", dir=parent + ) + temporary = Path(temporary_name) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + try: + _safe_directory(parent, create=False) + os.link(temporary, absolute) + except FileExistsError as error: + raise StageAError(f"refusing to overwrite published output: {absolute}") from error + temporary.unlink() + _safe_directory(parent, create=False) + if _is_link_or_reparse(absolute) or not absolute.is_file(): + raise StageAError("published output became unsafe") + finally: + with contextlib.suppress(OSError): + temporary.unlink() + + +def _seal_message_values( + *, + identity_file_sha256: str, + calibration_binding_file_sha256: str, + source_manifest_file_sha256: str, + input_bundle_manifest_file_sha256: str, + expected_forward_count: int, +) -> str: + return ( + "Experiment 013 Stage-A one-run seal\n\n" + f"{ONE_RUN_MARKER}\n" + f"Stage-A-Identity: {identity_file_sha256}\n" + f"Calibration-Binding: {calibration_binding_file_sha256}\n" + f"Source-Manifest: {source_manifest_file_sha256}\n" + f"Stage-A-Input-Bundle: {input_bundle_manifest_file_sha256}\n" + f"Expected-Forwards: {expected_forward_count}\n" + ) + + +def _seal_message(authenticated: AuthenticatedStageA) -> str: + bundle_hash = _require_sha256( + authenticated.input_bundle_manifest_file_sha256, + context="authenticated Stage-A input bundle manifest SHA-256", + ) + return _seal_message_values( + identity_file_sha256=authenticated.bootstrap_identity.file_sha256, + calibration_binding_file_sha256=authenticated.binding.file_sha256, + source_manifest_file_sha256=authenticated.source_manifest_file_sha256, + input_bundle_manifest_file_sha256=bundle_hash, + expected_forward_count=authenticated.bootstrap_identity.expected_forward_count, + ) + + +def _method_spec_receipts( + methods: Sequence[StageAMethodSpec], +) -> list[dict[str, object]]: + if tuple(method.method_id for method in methods) != METHOD_ORDER: + raise StageAError("Stage-A method specifications are incomplete or reordered") + receipts: list[dict[str, object]] = [] + for method in methods: + if method.method_id in {FP32_METHOD, DYNAMIC_K27030_METHOD}: + if method.policy_file_sha256 is not None: + raise StageAError(f"{method.method_id} must not bind a static policy file") + else: + _require_sha256( + method.policy_file_sha256, + context=f"{method.method_id} policy file SHA-256", + ) + receipts.append( + { + "method_id": method.method_id, + "policy_file_sha256": method.policy_file_sha256, + "policy_origin": _bounded_text( + method.origin, + context=f"{method.method_id} policy origin", + ), + } + ) + return receipts + + +def reserve_one_run(config: StageAConfig, authenticated: AuthenticatedStageA) -> AttemptReservation: + _assert_output_paths_isolated(config) + if config.output_path.exists() or config.complete_path.exists() or config.attempt_path.exists(): + raise StageAError("Stage-A output or attempt already exists; automatic retry is forbidden") + identity_bytes = _stable_file_bytes(config.frozen_identity_path, context="frozen identity") + identity_path = _assert_tracked_identity_bytes(config, identity_bytes) + identity_file_sha256 = authenticated.bootstrap_identity.file_sha256 + _reject_prior_identity_seal( + config.repository_root, + identity_file_sha256=identity_file_sha256, + git_executable_path=config.git_executable_path, + ) + h1 = _require_sha1( + _git(config.git_executable_path, config.repository_root, "rev-parse", "HEAD"), + context="H1", + ) + tree = _require_sha1( + _git( + config.git_executable_path, + config.repository_root, + "show", + "-s", + "--format=%T", + h1, + ), + context="H1 tree", + ) + message = _seal_message(authenticated) + seal = _require_sha1( + _git( + config.git_executable_path, + config.repository_root, + "commit-tree", + tree, + "-p", + h1, + input_bytes=message.encode("utf-8"), + ), + context="one-run seal commit", + ) + global_lock = _identity_attempt_lock_path( + config.repository_root, + identity_file_sha256, + git_executable_path=config.git_executable_path, + ) + created_at_utc = datetime.now(UTC).isoformat() + method_specs = _method_spec_receipts(authenticated.methods) + global_lock_document = { + "schema": IDENTITY_ATTEMPT_LOCK_SCHEMA, + "runner_revision": RUNNER_REVISION, + "created_at_utc": created_at_utc, + "attempt_number": 1, + "automatic_retry_authorized": False, + "h0_source_commit": authenticated.source_commit, + "h1_identity_commit": h1, + "identity_repository_path": identity_path, + "identity_file_sha256": identity_file_sha256, + "one_run_seal_commit": seal, + "one_run_seal_tree": tree, + "one_run_marker": ONE_RUN_MARKER, + "one_run_seal_message_sha256": sha256_bytes(message.encode("utf-8")), + "calibration_binding_file_sha256": authenticated.binding.file_sha256, + "source_manifest_file_sha256": authenticated.source_manifest_file_sha256, + "stage_a_input_bundle_manifest_file_sha256": _require_sha256( + authenticated.input_bundle_manifest_file_sha256, + context="authenticated Stage-A input bundle manifest SHA-256", + ), + "execution_bindings": dict(authenticated.bootstrap_identity.execution_bindings), + "method_specs": method_specs, + "expected_forward_count": authenticated.bootstrap_identity.expected_forward_count, + "claim_boundary": CLAIM_BOUNDARY, + "output_path": str(Path(os.path.abspath(config.output_path))), + "attempt_path": str(Path(os.path.abspath(config.attempt_path))), + "complete_path": str(Path(os.path.abspath(config.complete_path))), + } + global_lock_bytes = canonical_json_bytes(global_lock_document) + try: + _exclusive_write(global_lock, global_lock_bytes) + except FileExistsError as error: + raise StageAError( + "this Stage-A identity already has a consumed identity-scoped attempt lock" + ) from error + if _stable_file_bytes(global_lock, context="identity-scoped attempt lock") != ( + global_lock_bytes + ): + raise StageAError("identity-scoped attempt lock changed after exclusive creation") + prepared: dict[str, object] = { + "schema": ATTEMPT_SCHEMA, + "status": "prepared_before_head_cas", + "runner_revision": RUNNER_REVISION, + "created_at_utc": created_at_utc, + "attempt_number": 1, + "h0_source_commit": authenticated.source_commit, + "h1_identity_commit": h1, + "identity_repository_path": identity_path, + "one_run_seal_commit": seal, + "one_run_seal_tree": tree, + "one_run_marker": ONE_RUN_MARKER, + "one_run_seal_message_sha256": sha256_bytes(message.encode("utf-8")), + "identity_file_sha256": identity_file_sha256, + "identity_scoped_attempt_lock_file_sha256": sha256_bytes(global_lock_bytes), + "calibration_binding_file_sha256": authenticated.binding.file_sha256, + "source_manifest_file_sha256": authenticated.source_manifest_file_sha256, + "stage_a_input_bundle_manifest_file_sha256": _require_sha256( + authenticated.input_bundle_manifest_file_sha256, + context="authenticated Stage-A input bundle manifest SHA-256", + ), + "execution_bindings": dict(authenticated.bootstrap_identity.execution_bindings), + "method_specs": method_specs, + "expected_forward_count": authenticated.bootstrap_identity.expected_forward_count, + "observed_forward_count": 0, + "content_materialized": False, + "model_load_count": 0, + "evaluation_complete": False, + "result_available": False, + "automatic_retry_authorized": False, + "claim_boundary": CLAIM_BOUNDARY, + } + prepared_bytes = canonical_json_bytes(prepared) + _exclusive_write(config.attempt_path, prepared_bytes) + if ( + _stable_file_bytes(config.attempt_path, context="prepared attempt receipt") + != prepared_bytes + ): + raise StageAError("prepared attempt receipt changed before HEAD CAS") + cas = _git_process( + config.git_executable_path, + config.repository_root, + "update-ref", + "HEAD", + seal, + h1, + ) + if cas.returncode != 0: + detail = cas.stderr.decode("utf-8", errors="replace").strip() + raise StageAError( + "one-run HEAD compare-and-swap failed" + (f": {detail}" if detail else "") + ) + if ( + _git( + config.git_executable_path, + config.repository_root, + "rev-parse", + "HEAD", + ) + != seal + ): + raise StageAError("one-run HEAD compare-and-swap did not reach the seal") + reserved = {**prepared, "status": "reserved_before_stage_a_content_access"} + _atomic_replace_owned(config.attempt_path, canonical_json_bytes(reserved)) + return AttemptReservation(reserved, h1, seal, tree) + + +def persist_receipt( + config: StageAConfig, + reservation: AttemptReservation, + updates: Mapping[str, object], +) -> AttemptReservation: + disk = _strict_json( + _stable_file_bytes(config.attempt_path, context="Stage-A attempt receipt"), + context="Stage-A attempt receipt", + ) + if disk != dict(reservation.receipt): + raise StageAError("Stage-A attempt receipt changed outside the evaluator") + updated = {**disk, **dict(updates), "automatic_retry_authorized": False} + payload = canonical_json_bytes(updated) + _atomic_replace_owned(config.attempt_path, payload) + if _stable_file_bytes(config.attempt_path, context="updated attempt receipt") != payload: + raise StageAError("Stage-A attempt receipt persistence drifted") + return dataclasses.replace(reservation, receipt=MappingProxyType(updated)) + + +def _transition_hash( + *, + identity_record_sha256: str, + method_id: str, + transition_index: int, + input_position: int, + target_position: int, +) -> str: + return sha256_bytes( + b"recurquant.experiment013.stage-a-transition.v2\0" + + canonical_json_bytes( + { + "identity_record_sha256": identity_record_sha256, + "input_position": input_position, + "method_id": method_id, + "target_position": target_position, + "transition_index": transition_index, + } + ) + ) + + +def _finite_observation( + observation: ForwardObservation, + *, + expected_position: int, + expected_target: int, +) -> ForwardObservation: + if not isinstance(observation, ForwardObservation): + raise StageAError("Stage-A engine returned a non-observation") + if observation.position != expected_position or observation.target_token_id != expected_target: + raise StageAError("Stage-A forward position or target identity drifted") + torch = importlib.import_module("torch") + logits = observation.comparison_logits + if ( + not isinstance(logits, torch.Tensor) + or logits.ndim != 1 + or logits.numel() == 0 + or logits.device.type != "cpu" + or logits.dtype != torch.float32 + or not logits.is_contiguous() + or logits.requires_grad + or not torch.isfinite(logits).all().item() + ): + raise StageAError( + "Stage-A comparison logits must be a finite contiguous CPU float32 tensor" + ) + values = ( + observation.target_nll, + observation.local_codec_sse, + observation.trajectory_nmse, + ) + if ( + any(not math.isfinite(value) for value in values) + or observation.target_nll < 0.0 + or observation.local_codec_sse < 0.0 + or observation.trajectory_nmse < 0.0 + or any( + isinstance(value, bool) or not isinstance(value, int) or value < 0 + for value in ( + observation.latency_ns, + observation.peak_allocated_bytes, + observation.peak_reserved_bytes, + observation.resident_bytes, + observation.transient_bytes, + ) + ) + or isinstance(observation.top1_token_id, bool) + or not isinstance(observation.top1_token_id, int) + or observation.top1_token_id < 0 + or observation.top1_token_id >= logits.numel() + or isinstance(observation.target_token_id, bool) + or not isinstance(observation.target_token_id, int) + or observation.target_token_id < 0 + or observation.target_token_id >= logits.numel() + ): + raise StageAError("Stage-A forward observation contains invalid or non-finite evidence") + log_probabilities = torch.log_softmax(logits, dim=-1) + expected_nll = -float(log_probabilities[expected_target].item()) + expected_top1 = int(torch.argmax(logits).item()) + if observation.target_nll != expected_nll or observation.top1_token_id != expected_top1: + raise StageAError("Stage-A NLL or top-1 evidence differs from its comparison logits") + return observation + + +def _kl(reference: object, candidate: object) -> float: + """Reviewed FP32 token KL equation over compact CPU logits.""" + + torch = importlib.import_module("torch") + if ( + not isinstance(reference, torch.Tensor) + or not isinstance(candidate, torch.Tensor) + or reference.ndim != 1 + or candidate.ndim != 1 + or reference.shape != candidate.shape + or reference.dtype != torch.float32 + or candidate.dtype != torch.float32 + or reference.device.type != "cpu" + or candidate.device.type != "cpu" + or not reference.is_contiguous() + or not candidate.is_contiguous() + ): + raise StageAError("Stage-A reference and candidate vocabulary dimensions differ") + if not torch.isfinite(reference).all().item() or not torch.isfinite(candidate).all().item(): + raise StageAError("Stage-A comparison logits are non-finite") + reference_log_probabilities = torch.log_softmax(reference, dim=-1) + candidate_log_probabilities = torch.log_softmax(candidate, dim=-1) + value = float( + ( + reference_log_probabilities.exp() + * (reference_log_probabilities - candidate_log_probabilities) + ) + .sum(dim=-1) + .item() + ) + if not math.isfinite(value): + raise StageAError("Stage-A KL became non-finite") + if value < -1.0e-6: + raise StageAError("Stage-A KL is materially negative") + return max(0.0, value) + + +def _validated_device_runtime(value: Mapping[str, object]) -> Mapping[str, object]: + expected = { + "attention_implementation", + "capability", + "cuda_runtime", + "device_index", + "model_class", + "model_config_class", + "model_parameter_dtype", + "name", + "torch_version", + "total_memory_bytes", + } + _exact_fields(value, expected, context="Stage-A device runtime") + capability = value.get("capability") + if ( + not isinstance(capability, list) + or len(capability) != 2 + or any( + isinstance(item, bool) or not isinstance(item, int) or item < 0 for item in capability + ) + ): + raise StageAError("Stage-A CUDA capability is invalid") + for field_name in ( + "attention_implementation", + "cuda_runtime", + "model_class", + "model_config_class", + "model_parameter_dtype", + "name", + "torch_version", + ): + item = value.get(field_name) + if ( + not isinstance(item, str) + or not item + or len(item) > 256 + or any(ord(character) < 32 for character in item) + ): + raise StageAError(f"Stage-A device runtime {field_name} is invalid") + for field_name in ("device_index", "total_memory_bytes"): + item = value.get(field_name) + if isinstance(item, bool) or not isinstance(item, int) or item < 0: + raise StageAError(f"Stage-A device runtime {field_name} is invalid") + if ( + value.get("attention_implementation") != "eager" + or value.get("model_class") != "Qwen3_5ForCausalLM" + or value.get("model_config_class") != "Qwen3_5TextConfig" + or value.get("model_parameter_dtype") != "torch.bfloat16" + ): + raise StageAError( + "Stage-A model class, config, eager attention, or BF16 dtype contract drifted" + ) + return MappingProxyType({**dict(value), "capability": list(capability)}) + + +def _validated_authenticated_runtime( + value: Mapping[str, object], + *, + expected_manifest_file_sha256: str, +) -> Mapping[str, object]: + if not isinstance(value, Mapping): + raise StageAError("Stage-A authenticated runtime evidence is missing") + _exact_fields( + value, + { + "base_runtime_file_count", + "distribution_count", + "distributions", + "file_count", + "git_executable_absolute_path_sha256", + "git_executable_sha256", + "git_executable_size_bytes", + "interpreter_sha256", + "machine_name", + "manifest_file_sha256", + "package_root_count", + "python_cache_tag", + "python_implementation", + "python_version", + }, + context="Stage-A authenticated runtime evidence", + ) + manifest_hash = _require_sha256( + expected_manifest_file_sha256, + context="expected Stage-A runtime manifest file SHA-256", + ) + if value.get("manifest_file_sha256") != manifest_hash: + raise StageAError("Stage-A authenticated runtime differs from its execution binding") + for name in ( + "git_executable_absolute_path_sha256", + "git_executable_sha256", + "interpreter_sha256", + ): + _require_sha256(value.get(name), context=f"Stage-A authenticated runtime {name}") + for name in ( + "base_runtime_file_count", + "distribution_count", + "file_count", + "git_executable_size_bytes", + "package_root_count", + ): + item = value.get(name) + if isinstance(item, bool) or not isinstance(item, int) or item < 0: + raise StageAError(f"Stage-A authenticated runtime {name} is invalid") + for name in ( + "machine_name", + "python_cache_tag", + "python_implementation", + "python_version", + ): + _bounded_text(value.get(name), context=f"Stage-A authenticated runtime {name}") + distributions = value.get("distributions") + if not isinstance(distributions, list) or any( + not isinstance(item, list) + or len(item) != 2 + or any( + not isinstance(part, str) + or not part + or len(part) > 512 + or part != part.strip() + or any(ord(character) < 32 or ord(character) == 127 for character in part) + for part in item + ) + for item in distributions + ): + raise StageAError("Stage-A authenticated distributions are invalid") + if value.get("distribution_count") != len(distributions): + raise StageAError("Stage-A authenticated distribution count drifted") + return MappingProxyType( + {**dict(value), "distributions": [list(item) for item in distributions]} + ) + + +def _authenticated_runtime_record( + runtime: object, + *, + expected_manifest_file_sha256: str, +) -> Mapping[str, object]: + return _validated_authenticated_runtime( + { + "base_runtime_file_count": getattr(runtime, "base_runtime_file_count", None), + "distribution_count": getattr(runtime, "distribution_count", None), + "distributions": [list(item) for item in getattr(runtime, "distributions", ())], + "file_count": getattr(runtime, "file_count", None), + "git_executable_absolute_path_sha256": getattr( + runtime, + "git_executable_absolute_path_sha256", + None, + ), + "git_executable_sha256": getattr(runtime, "git_executable_sha256", None), + "git_executable_size_bytes": getattr( + runtime, + "git_executable_size_bytes", + None, + ), + "interpreter_sha256": getattr(runtime, "interpreter_sha256", None), + "machine_name": getattr(runtime, "machine_name", None), + "manifest_file_sha256": getattr(runtime, "manifest_file_sha256", None), + "package_root_count": getattr(runtime, "package_root_count", None), + "python_cache_tag": getattr(runtime, "python_cache_tag", None), + "python_implementation": getattr(runtime, "python_implementation", None), + "python_version": getattr(runtime, "python_version", None), + }, + expected_manifest_file_sha256=expected_manifest_file_sha256, + ) + + +def _sequence_fields( + sequence: object, +) -> tuple[Mapping[str, object], tuple[int, ...], tuple[int, ...]]: + record = getattr(sequence, "identity_record", None) + prompt = getattr(sequence, "prompt_token_ids", None) + target = getattr(sequence, "target_token_ids", None) + if ( + not isinstance(record, Mapping) + or not isinstance(prompt, tuple) + or not isinstance(target, tuple) + ): + raise StageAError("Stage-A materializer returned an invalid sequence") + if not prompt or len(target) < 2: + raise StageAError("Stage-A sequence does not satisfy prompt/continuation bounds") + span = record.get("token_span") + if not isinstance(span, Mapping): + raise StageAError("Stage-A identity record has no token span") + expected_span = { + "prefill_start": 0, + "prefill_stop": len(prompt), + "scored_start": len(prompt), + "scored_stop": len(prompt) + len(target), + "cache_exposed_start": len(prompt) + 1, + "cache_exposed_stop": len(prompt) + len(target), + } + if dict(span) != expected_span: + raise StageAError("Stage-A materialized token span differs from the identity") + return record, prompt, target + + +def _storage_receipt(method_id: str, summary: Mapping[str, object]) -> Mapping[str, object]: + if method_id not in METHOD_ORDER: + raise StageAError("Stage-A storage receipt method is unknown") + expected_resident = EXPECTED_RECURRENT_RESIDENT_BYTES[method_id] + reported_resident = summary.get("resident_bytes", expected_resident) + reported_expected = summary.get("expected_resident_bytes", expected_resident) + raw_workspace = summary.get("raw_state_workspace_peak_bytes", 0) + query_workspace = summary.get("query_workspace_peak_bytes", 0) + for name, value in ( + ("cache-reported resident bytes", reported_resident), + ("cache-reported expected resident bytes", reported_expected), + ("raw-state workspace high-water bytes", raw_workspace), + ("query workspace high-water bytes", query_workspace), + ): + _nonnegative_int(value, context=f"Stage-A {name}") + if reported_resident != expected_resident or reported_expected != expected_resident: + raise StageAError("Stage-A cache storage differs from the frozen resident-byte ledger") + return MappingProxyType( + { + "logical_recurrent_resident_bytes": expected_resident, + "cache_reported_resident_bytes": reported_resident, + "cache_reported_expected_resident_bytes": reported_expected, + "raw_state_workspace_high_water_bytes": raw_workspace, + "query_workspace_high_water_bytes": query_workspace, + "workspace_scope": "method_lifetime_high_water_since_cache_creation", + } + ) + + +def evaluate_materialized_stage_a( + authenticated: AuthenticatedStageA, + materialization: object, + engine: StageAEngine, + model: object, +) -> StageAEvaluation: + """Execute the fixed 9-method grid and independently count every forward.""" + + gate = importlib.import_module("recurquant.experiment013_stage_a") + if tuple(gate.STAGE_A_METHOD_ORDER) != METHOD_ORDER: + raise StageAError("Stage-A gate method order differs from the runner") + if tuple(method.method_id for method in authenticated.methods) != METHOD_ORDER: + raise StageAError("authenticated Stage-A methods are missing, duplicated, or reordered") + sequences = getattr(materialization, "sequences", None) + if not isinstance(sequences, tuple) or len(sequences) != 12: + raise StageAError("Stage-A materialization must contain exactly twelve sequences") + examples: list[object] = [] + gate_rows: list[object] = [] + raw_rows: list[Mapping[str, object]] = [] + method_runtime: list[Mapping[str, object]] = [] + forwards = 0 + for sequence in sequences: + record, prompt_ids, target_ids = _sequence_fields(sequence) + identity_hash = _require_sha256( + record.get("identity_record_sha256"), context="Stage-A identity record SHA-256" + ) + family = record.get("family") + canonical_id = record.get("canonical_id") + rank = record.get("selection_rank") + if family not in {"pg19", "ruler", "humaneval_plus"}: + raise StageAError("Stage-A sequence family drifted") + if not isinstance(canonical_id, str) or not canonical_id: + raise StageAError("Stage-A canonical ID is invalid") + if isinstance(rank, bool) or not isinstance(rank, int) or rank not in range(4): + raise StageAError("Stage-A selection rank is invalid") + examples.append( + gate.StageAExample( + family=family, + canonical_id=canonical_id, + selection_rank=rank, + continuation_token_count=len(target_ids), + identity_record_sha256=identity_hash, + ) + ) + references: list[ForwardObservation] = [] + for method in authenticated.methods: + session = engine.begin_method(model, method, sequence) + method_started = time.perf_counter_ns() + try: + prefill_position = len(prompt_ids) - 1 + prefill = _finite_observation( + engine.prefill( + session, + prompt_token_ids=prompt_ids, + first_target_token_id=target_ids[0], + position=prefill_position, + ), + expected_position=prefill_position, + expected_target=target_ids[0], + ) + forwards += 1 + current_rows: list[ForwardObservation] = [] + for transition_index in range(len(target_ids) - 1): + input_position = len(prompt_ids) + transition_index + target_position = input_position + 1 + observation = _finite_observation( + engine.step( + session, + input_token_id=target_ids[transition_index], + target_token_id=target_ids[transition_index + 1], + position=input_position, + ), + expected_position=input_position, + expected_target=target_ids[transition_index + 1], + ) + forwards += 1 + current_rows.append(observation) + if method.method_id == FP32_METHOD: + reference = observation + references.append(observation) + kl = 0.0 + agreement = True + else: + if transition_index >= len(references): + raise StageAError("candidate method ran before its FP32 reference") + reference = references[transition_index] + kl = _kl(reference.comparison_logits, observation.comparison_logits) + agreement = observation.top1_token_id == reference.top1_token_id + transition_hash = _transition_hash( + identity_record_sha256=identity_hash, + method_id=method.method_id, + transition_index=transition_index, + input_position=input_position, + target_position=target_position, + ) + gate_rows.append( + gate.StageATokenRow( + family=family, + canonical_id=canonical_id, + selection_rank=rank, + identity_record_sha256=identity_hash, + method_id=method.method_id, + transition_index=transition_index, + reference_nll=reference.target_nll, + method_nll=observation.target_nll, + kl=kl, + top1_agreement=agreement, + ) + ) + raw_rows.append( + MappingProxyType( + { + "family": family, + "canonical_id": canonical_id, + "selection_rank": rank, + "identity_record_sha256": identity_hash, + "method_id": method.method_id, + "transition_index": transition_index, + "input_position": input_position, + "target_position": target_position, + "authenticated_transition_sha256": transition_hash, + "reference_nll": reference.target_nll, + "method_nll": observation.target_nll, + "excess_nll": observation.target_nll - reference.target_nll, + "kl": kl, + "top1_agreement": agreement, + "local_codec_sse": observation.local_codec_sse, + "trajectory_nmse": observation.trajectory_nmse, + "decode_model_forward_latency_ns": observation.latency_ns, + "decode_cuda_diagnostic_peak_allocated_bytes": ( + observation.peak_allocated_bytes + ), + "decode_cuda_diagnostic_peak_reserved_bytes": ( + observation.peak_reserved_bytes + ), + "decode_logical_recurrent_resident_bytes": ( + observation.resident_bytes + ), + "method_cumulative_cache_reported_workspace_high_water_sum_bytes": ( + observation.transient_bytes + ), + "finite_checks": { + "comparison_logits": True, + "nll": True, + "kl": True, + "local_codec_sse": True, + "trajectory_nmse": True, + }, + } + ) + ) + if len(current_rows) != len(target_ids) - 1: + raise StageAError("Stage-A method omitted a cache-exposed transition") + except BaseException as error: + try: + engine.end_method(session) + except BaseException as cleanup_error: + error.add_note(f"Stage-A cache observer cleanup also failed: {cleanup_error!r}") + raise + else: + summary = dict(engine.end_method(session)) + method_runtime.append( + MappingProxyType( + { + "family": family, + "canonical_id": canonical_id, + "selection_rank": rank, + "identity_record_sha256": identity_hash, + "method_id": method.method_id, + "policy_file_sha256": method.policy_file_sha256, + "policy_origin": method.origin, + "prefill_diagnostics": { + "model_forward_latency_ns": prefill.latency_ns, + "cuda_diagnostic_peak_allocated_bytes": (prefill.peak_allocated_bytes), + "cuda_diagnostic_peak_reserved_bytes": prefill.peak_reserved_bytes, + "logical_recurrent_resident_bytes": prefill.resident_bytes, + "cache_reported_workspace_high_water_sum_bytes": ( + prefill.transient_bytes + ), + }, + "max_cuda_diagnostic_peak_allocated_bytes_across_prefill_and_decode": max( + [prefill.peak_allocated_bytes] + + [row.peak_allocated_bytes for row in current_rows] + ), + "max_cuda_diagnostic_peak_reserved_bytes_across_prefill_and_decode": max( + [prefill.peak_reserved_bytes] + + [row.peak_reserved_bytes for row in current_rows] + ), + "wall_time_with_diagnostics_ns": (time.perf_counter_ns() - method_started), + "storage": dict(_storage_receipt(method.method_id, summary)), + } + ) + ) + if len(references) != len(target_ids) - 1: + raise StageAError("FP32 reference trace is incomplete") + references.clear() + if forwards != authenticated.bootstrap_identity.expected_forward_count: + raise StageAError( + "Stage-A forward count differs from 9*sum(1+m-1): " + f"expected {authenticated.bootstrap_identity.expected_forward_count}, " + f"observed {forwards}" + ) + device_runtime = _validated_device_runtime(dict(engine.runtime_snapshot(model))) + return StageAEvaluation( + examples=tuple(examples), + gate_rows=tuple(gate_rows), + raw_rows=tuple(raw_rows), + forward_count=forwards, + method_runtime=tuple(method_runtime), + device_runtime=device_runtime, + ) + + +def _expected_execution_contract(expected_forward_count: int) -> dict[str, object]: + if ( + isinstance(expected_forward_count, bool) + or not isinstance(expected_forward_count, int) + or expected_forward_count <= 0 + ): + raise StageAError("Stage-A expected forward count is invalid") + return { + "method_order": list(METHOD_ORDER), + "model_load_count": 1, + "forward_formula": "9*sum(1+m-1)", + "expected_forward_count": expected_forward_count, + "observed_forward_count": expected_forward_count, + "one_prefill_per_method_example": True, + "one_token_transitions_per_method_example": True, + "trajectory_nmse": ( + "per-token mean of per-layer FP64 NMSE against the matched FP32 recurrent trajectory" + ), + "measurement_scope": { + "decode_logical_recurrent_resident_bytes": ( + "logical persistent recurrent-state bytes exposed after each scored decode; " + "FP32 is the exact 18,874,368-byte tensor ledger and packed methods use " + "checkpoint bytes" + ), + "method_cumulative_cache_reported_workspace_high_water_sum_bytes": ( + "sum of independently tracked cache workspace high-water marks accumulated " + "since method start, including prefill; not per-decode transient memory and " + "not a simultaneous allocator peak" + ), + "decode_cuda_diagnostic_peak_bytes": ( + "CUDA allocator peak since the per-forward reset for a scored decode, " + "including observer/cache and diagnostic state materialization; not " + "deployment HBM" + ), + "decode_model_forward_latency_ns": ( + "synchronized scored-decode model forward including observer/cache update, " + "excluding post-forward trajectory diagnostics" + ), + "prefill_diagnostics": ( + "separate synchronized 4096-or-identity-length prefill forward and CUDA " + "allocator diagnostics; no prefill quality metric is scored" + ), + "method_runtime": ( + "wall time includes prefill, scored decodes, cache observers, and trajectory " + "diagnostics; max CUDA fields cover both prefill and scored decodes" + ), + }, + "automatic_retry_authorized": False, + } + + +def _materialization_receipt(materialization: object) -> dict[str, object]: + receipt = { + "sequence_count": 12, + "capture_input_sha256": getattr(materialization, "capture_input_sha256", None), + "token_sequence_manifest_sha256": getattr( + materialization, + "token_sequence_manifest_sha256", + None, + ), + "tokenizer_manifest_sha256": getattr( + materialization, + "tokenizer_manifest_sha256", + None, + ), + } + for name in ( + "capture_input_sha256", + "token_sequence_manifest_sha256", + "tokenizer_manifest_sha256", + ): + _require_sha256(receipt[name], context=f"Stage-A materialization {name}") + return receipt + + +def build_execution_artifact( + authenticated: AuthenticatedStageA, + materialization: object, + evaluation: StageAEvaluation, + reservation: AttemptReservation, +) -> bytes: + gate = importlib.import_module("recurquant.experiment013_stage_a") + gate_bytes = gate.build_stage_a_evidence_artifact( + evaluation.examples, + evaluation.gate_rows, + stage_a_identity_file_sha256=authenticated.bootstrap_identity.file_sha256, + stage_a_calibration_binding_file_sha256=authenticated.binding.file_sha256, + ) + verified_gate = gate.deserialize_stage_a_evidence_artifact( + gate_bytes, + expected_stage_a_identity_file_sha256=authenticated.bootstrap_identity.file_sha256, + expected_stage_a_calibration_binding_file_sha256=authenticated.binding.file_sha256, + ) + receipt = reservation.receipt + expected_runtime_manifest_hash = _require_sha256( + authenticated.bootstrap_identity.execution_bindings.get( + "calibration_runtime_manifest_file_sha256" + ), + context="Stage-A runtime execution binding", + ) + live_runtime_record = _authenticated_runtime_record( + authenticated.authenticated_runtime, + expected_manifest_file_sha256=expected_runtime_manifest_hash, + ) + receipt_runtime = receipt.get("post_load_authenticated_runtime") + if not isinstance(receipt_runtime, Mapping): + raise StageAError("Stage-A attempt receipt does not bind the post-load runtime") + runtime_record = _validated_authenticated_runtime( + receipt_runtime, + expected_manifest_file_sha256=expected_runtime_manifest_hash, + ) + if dict(runtime_record) != dict(live_runtime_record): + raise StageAError("Stage-A authenticated runtime drifted after its post-load receipt") + receipt_device = receipt.get("post_load_device_runtime") + if not isinstance(receipt_device, Mapping): + raise StageAError("Stage-A attempt receipt does not bind the post-load device") + device_record = _validated_device_runtime(receipt_device) + if dict(device_record) != dict(evaluation.device_runtime): + raise StageAError("Stage-A device runtime drifted after its post-load receipt") + raw_smoke = receipt.get("preseal_engine_smoke") + if not isinstance(raw_smoke, Mapping): + raise StageAError("Stage-A attempt receipt does not bind the pre-seal engine smoke") + preseal_smoke = _validated_preseal_engine_smoke(raw_smoke) + preseal_smoke_sha256 = sha256_bytes(canonical_json_bytes(dict(preseal_smoke))) + if preseal_smoke_sha256 != _require_sha256( + receipt.get("preseal_engine_smoke_sha256"), + context="attempt pre-seal engine smoke SHA-256", + ): + raise StageAError("Stage-A pre-seal engine smoke receipt hash drifted") + one_run = { + "attempt_schema": receipt.get("schema"), + "automatic_retry_authorized": receipt.get("automatic_retry_authorized"), + "h0_source_commit": receipt.get("h0_source_commit"), + "h1_identity_commit": reservation.h1_commit, + "identity_scoped_attempt_lock_file_sha256": receipt.get( + "identity_scoped_attempt_lock_file_sha256" + ), + "one_run_marker": receipt.get("one_run_marker"), + "one_run_seal_commit": reservation.seal_commit, + "one_run_seal_message_sha256": receipt.get("one_run_seal_message_sha256"), + "one_run_seal_tree": reservation.tree, + "preseal_engine_smoke_sha256": preseal_smoke_sha256, + "stage_a_input_bundle_manifest_file_sha256": _require_sha256( + authenticated.input_bundle_manifest_file_sha256, + context="authenticated Stage-A input bundle manifest SHA-256", + ), + } + evidence = { + "artifact_revision": RUNNER_REVISION, + "claim_boundary": CLAIM_BOUNDARY, + "dependencies": { + "stage_a_identity_file_sha256": authenticated.bootstrap_identity.file_sha256, + "stage_a_calibration_binding_file_sha256": authenticated.binding.file_sha256, + "repository_source_manifest_file_sha256": authenticated.source_manifest_file_sha256, + "stage_a_input_bundle_manifest_file_sha256": _require_sha256( + authenticated.input_bundle_manifest_file_sha256, + context="authenticated Stage-A input bundle manifest SHA-256", + ), + "execution_bindings": dict(authenticated.bootstrap_identity.execution_bindings), + "method_specs": _method_spec_receipts(authenticated.methods), + }, + "execution_contract": _expected_execution_contract(evaluation.forward_count), + "one_run": one_run, + "preseal_engine_smoke": dict(preseal_smoke), + "materialization": _materialization_receipt(materialization), + "runtime": { + "authenticated_runtime": dict(runtime_record), + "device": dict(device_record), + }, + "method_runtime": [dict(row) for row in evaluation.method_runtime], + "raw_token_evidence": [dict(row) for row in evaluation.raw_rows], + "stage_a_gate_artifact": _strict_json(gate_bytes, context="Stage-A gate artifact"), + "stage_a_gate_file_sha256": verified_gate.file_sha256, + "stage_a_passed": verified_gate.passed, + } + document = { + "artifact_kind": EXECUTION_ARTIFACT_KIND, + "schema_version": EXECUTION_ARTIFACT_SCHEMA, + "canonical_evidence_sha256": sha256_bytes(canonical_json_bytes(evidence)), + "evidence": evidence, + } + payload = canonical_json_bytes(document) + verify_execution_artifact( + payload, + expected_identity_file_sha256=authenticated.bootstrap_identity.file_sha256, + expected_calibration_binding_file_sha256=authenticated.binding.file_sha256, + expected_h1_commit=reservation.h1_commit, + expected_seal_commit=reservation.seal_commit, + expected_source_commit=authenticated.source_commit, + expected_source_manifest_file_sha256=authenticated.source_manifest_file_sha256, + expected_input_bundle_manifest_file_sha256=_require_sha256( + authenticated.input_bundle_manifest_file_sha256, + context="authenticated Stage-A input bundle manifest SHA-256", + ), + expected_execution_bindings=authenticated.bootstrap_identity.execution_bindings, + expected_method_specs=_method_spec_receipts(authenticated.methods), + expected_materialization=_materialization_receipt(materialization), + expected_seal_tree=reservation.tree, + expected_seal_message_sha256=_require_sha256( + receipt.get("one_run_seal_message_sha256"), + context="attempt one-run seal message SHA-256", + ), + expected_attempt_lock_file_sha256=_require_sha256( + receipt.get("identity_scoped_attempt_lock_file_sha256"), + context="attempt identity-scoped lock SHA-256", + ), + expected_authenticated_runtime=runtime_record, + expected_device_runtime=device_record, + expected_forward_count=authenticated.bootstrap_identity.expected_forward_count, + ) + return payload + + +def _nonnegative_int(value: object, *, context: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise StageAError(f"{context} is not a nonnegative integer") + return value + + +def _finite_number( + value: object, + *, + context: str, + nonnegative: bool, +) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise StageAError(f"{context} is not numeric") + result = float(value) + if not math.isfinite(result) or (nonnegative and result < 0.0): + raise StageAError(f"{context} is invalid or non-finite") + return result + + +def _bounded_text(value: object, *, context: str) -> str: + if ( + not isinstance(value, str) + or not value + or len(value) > 512 + or value != value.strip() + or any(ord(character) < 32 or ord(character) == 127 for character in value) + ): + raise StageAError(f"{context} is invalid") + return value + + +def verify_execution_artifact( + data: bytes, + *, + expected_identity_file_sha256: str, + expected_calibration_binding_file_sha256: str, + expected_h1_commit: str, + expected_seal_commit: str, + expected_source_commit: str, + expected_source_manifest_file_sha256: str, + expected_input_bundle_manifest_file_sha256: str, + expected_execution_bindings: Mapping[str, str], + expected_method_specs: Sequence[Mapping[str, object]], + expected_materialization: Mapping[str, object], + expected_seal_tree: str, + expected_seal_message_sha256: str, + expected_attempt_lock_file_sha256: str, + expected_authenticated_runtime: Mapping[str, object], + expected_device_runtime: Mapping[str, object], + expected_forward_count: int, +) -> Mapping[str, object]: + identity_hash = _require_sha256( + expected_identity_file_sha256, + context="expected Stage-A identity file SHA-256", + ) + binding_hash = _require_sha256( + expected_calibration_binding_file_sha256, + context="expected Stage-A calibration binding file SHA-256", + ) + h1 = _require_sha1(expected_h1_commit, context="expected Stage-A H1 commit") + seal = _require_sha1(expected_seal_commit, context="expected Stage-A seal commit") + source_commit = _require_sha1( + expected_source_commit, + context="expected Stage-A H0 source commit", + ) + source_manifest_hash = _require_sha256( + expected_source_manifest_file_sha256, + context="expected Stage-A source manifest file SHA-256", + ) + input_bundle_manifest_hash = _require_sha256( + expected_input_bundle_manifest_file_sha256, + context="expected Stage-A input bundle manifest file SHA-256", + ) + seal_tree = _require_sha1(expected_seal_tree, context="expected Stage-A seal tree") + seal_message_hash = _require_sha256( + expected_seal_message_sha256, + context="expected Stage-A seal message SHA-256", + ) + attempt_lock_hash = _require_sha256( + expected_attempt_lock_file_sha256, + context="expected Stage-A attempt lock SHA-256", + ) + expected_bindings = dict(expected_execution_bindings) + _exact_fields( + expected_bindings, + EXECUTION_BINDING_FIELDS, + context="expected Stage-A execution bindings", + ) + for name, digest in expected_bindings.items(): + _require_sha256(digest, context=f"expected Stage-A execution binding {name}") + trusted_runtime = _validated_authenticated_runtime( + expected_authenticated_runtime, + expected_manifest_file_sha256=expected_bindings["calibration_runtime_manifest_file_sha256"], + ) + trusted_device = _validated_device_runtime(expected_device_runtime) + if isinstance(expected_method_specs, (str, bytes, bytearray)): + raise StageAError("expected Stage-A method specifications are invalid") + expected_specs: list[dict[str, object]] = [] + if len(expected_method_specs) != len(METHOD_ORDER): + raise StageAError("expected Stage-A method specifications are incomplete") + for method_id, raw_spec in zip(METHOD_ORDER, expected_method_specs, strict=True): + if not isinstance(raw_spec, Mapping): + raise StageAError("expected Stage-A method specification is invalid") + _exact_fields( + raw_spec, + {"method_id", "policy_file_sha256", "policy_origin"}, + context=f"expected Stage-A method specification {method_id}", + ) + if raw_spec.get("method_id") != method_id: + raise StageAError("expected Stage-A method specifications are reordered") + policy_hash = raw_spec.get("policy_file_sha256") + if method_id in {FP32_METHOD, DYNAMIC_K27030_METHOD}: + if policy_hash is not None: + raise StageAError(f"expected {method_id} policy hash must be null") + else: + _require_sha256(policy_hash, context=f"expected {method_id} policy SHA-256") + _bounded_text(raw_spec.get("policy_origin"), context=f"expected {method_id} origin") + expected_specs.append(dict(raw_spec)) + if ( + isinstance(expected_forward_count, bool) + or not isinstance(expected_forward_count, int) + or expected_forward_count <= 0 + ): + raise StageAError("expected Stage-A forward count is invalid") + expected_materialization_receipt = dict(expected_materialization) + _exact_fields( + expected_materialization_receipt, + { + "sequence_count", + "capture_input_sha256", + "token_sequence_manifest_sha256", + "tokenizer_manifest_sha256", + }, + context="expected Stage-A materialization receipt", + ) + if expected_materialization_receipt.get("sequence_count") != 12: + raise StageAError("expected Stage-A materialization count drifted") + for name in ( + "capture_input_sha256", + "token_sequence_manifest_sha256", + "tokenizer_manifest_sha256", + ): + _require_sha256( + expected_materialization_receipt.get(name), + context=f"expected Stage-A materialization {name}", + ) + root = _strict_json(data, context="Stage-A execution artifact") + _exact_fields( + root, + {"artifact_kind", "schema_version", "canonical_evidence_sha256", "evidence"}, + context="Stage-A execution artifact", + ) + if canonical_json_bytes(root) != data: + raise StageAError("Stage-A execution artifact is not canonical JSON") + if ( + root["artifact_kind"] != EXECUTION_ARTIFACT_KIND + or root["schema_version"] != EXECUTION_ARTIFACT_SCHEMA + ): + raise StageAError("Stage-A execution artifact kind or schema drifted") + evidence = root.get("evidence") + if not isinstance(evidence, Mapping): + raise StageAError("Stage-A execution evidence is missing") + _exact_fields( + evidence, + { + "artifact_revision", + "claim_boundary", + "dependencies", + "execution_contract", + "materialization", + "method_runtime", + "one_run", + "preseal_engine_smoke", + "raw_token_evidence", + "runtime", + "stage_a_gate_artifact", + "stage_a_gate_file_sha256", + "stage_a_passed", + }, + context="Stage-A execution evidence", + ) + if root.get("canonical_evidence_sha256") != sha256_bytes(canonical_json_bytes(evidence)): + raise StageAError("Stage-A execution evidence SHA-256 drifted") + if ( + evidence.get("artifact_revision") != RUNNER_REVISION + or evidence.get("claim_boundary") != CLAIM_BOUNDARY + ): + raise StageAError("Stage-A execution artifact revision or claim boundary drifted") + + dependencies = evidence.get("dependencies") + if not isinstance(dependencies, Mapping): + raise StageAError("Stage-A execution dependencies are missing") + _exact_fields( + dependencies, + { + "stage_a_identity_file_sha256", + "stage_a_calibration_binding_file_sha256", + "repository_source_manifest_file_sha256", + "stage_a_input_bundle_manifest_file_sha256", + "execution_bindings", + "method_specs", + }, + context="Stage-A execution dependencies", + ) + if ( + dependencies.get("stage_a_identity_file_sha256") != identity_hash + or dependencies.get("stage_a_calibration_binding_file_sha256") != binding_hash + ): + raise StageAError("Stage-A result identity or calibration binding drifted") + if dependencies.get("repository_source_manifest_file_sha256") != source_manifest_hash: + raise StageAError("Stage-A result source manifest identity drifted") + if dependencies.get("stage_a_input_bundle_manifest_file_sha256") != input_bundle_manifest_hash: + raise StageAError("Stage-A result input bundle identity drifted") + execution_bindings = dependencies.get("execution_bindings") + if not isinstance(execution_bindings, Mapping): + raise StageAError("Stage-A result execution bindings are missing") + _exact_fields( + execution_bindings, + EXECUTION_BINDING_FIELDS, + context="Stage-A result execution bindings", + ) + for name, digest in execution_bindings.items(): + _require_sha256(digest, context=f"Stage-A result execution binding {name}") + if dict(execution_bindings) != expected_bindings: + raise StageAError("Stage-A result execution bindings drifted") + method_specs = dependencies.get("method_specs") + if not isinstance(method_specs, list) or method_specs != expected_specs: + raise StageAError("Stage-A result method specifications drifted") + + one_run = evidence.get("one_run") + if not isinstance(one_run, Mapping): + raise StageAError("Stage-A result one-run identity is missing") + _exact_fields( + one_run, + { + "attempt_schema", + "automatic_retry_authorized", + "h0_source_commit", + "h1_identity_commit", + "identity_scoped_attempt_lock_file_sha256", + "one_run_marker", + "one_run_seal_commit", + "one_run_seal_message_sha256", + "one_run_seal_tree", + "preseal_engine_smoke_sha256", + "stage_a_input_bundle_manifest_file_sha256", + }, + context="Stage-A result one-run identity", + ) + if ( + one_run.get("attempt_schema") != ATTEMPT_SCHEMA + or one_run.get("automatic_retry_authorized") is not False + or one_run.get("one_run_marker") != ONE_RUN_MARKER + or one_run.get("h0_source_commit") != source_commit + or one_run.get("h1_identity_commit") != h1 + or one_run.get("one_run_seal_commit") != seal + or one_run.get("one_run_seal_tree") != seal_tree + or one_run.get("one_run_seal_message_sha256") != seal_message_hash + or one_run.get("identity_scoped_attempt_lock_file_sha256") != attempt_lock_hash + or one_run.get("stage_a_input_bundle_manifest_file_sha256") != input_bundle_manifest_hash + ): + raise StageAError("Stage-A result one-run identity drifted") + preseal_smoke = evidence.get("preseal_engine_smoke") + if not isinstance(preseal_smoke, Mapping): + raise StageAError("Stage-A result pre-seal engine smoke is missing") + _validated_preseal_engine_smoke(preseal_smoke) + if sha256_bytes(canonical_json_bytes(dict(preseal_smoke))) != _require_sha256( + one_run.get("preseal_engine_smoke_sha256"), + context="Stage-A result pre-seal engine smoke SHA-256", + ): + raise StageAError("Stage-A result pre-seal engine smoke hash drifted") + + execution_contract = evidence.get("execution_contract") + if not isinstance(execution_contract, Mapping) or dict(execution_contract) != ( + _expected_execution_contract(expected_forward_count) + ): + raise StageAError("Stage-A execution contract drifted") + materialization = evidence.get("materialization") + if not isinstance(materialization, Mapping): + raise StageAError("Stage-A materialization receipt is missing") + _exact_fields( + materialization, + { + "sequence_count", + "capture_input_sha256", + "token_sequence_manifest_sha256", + "tokenizer_manifest_sha256", + }, + context="Stage-A materialization receipt", + ) + if materialization.get("sequence_count") != 12: + raise StageAError("Stage-A materialization sequence count drifted") + for name in ( + "capture_input_sha256", + "token_sequence_manifest_sha256", + "tokenizer_manifest_sha256", + ): + _require_sha256( + materialization.get(name), + context=f"Stage-A materialization {name}", + ) + if dict(materialization) != expected_materialization_receipt: + raise StageAError("Stage-A materialization receipt drifted") + + runtime = evidence.get("runtime") + if not isinstance(runtime, Mapping): + raise StageAError("Stage-A result runtime evidence is missing") + _exact_fields(runtime, {"authenticated_runtime", "device"}, context="Stage-A runtime") + authenticated_runtime = runtime.get("authenticated_runtime") + if not isinstance(authenticated_runtime, Mapping): + raise StageAError("Stage-A authenticated runtime evidence is missing") + observed_runtime = _validated_authenticated_runtime( + authenticated_runtime, + expected_manifest_file_sha256=expected_bindings["calibration_runtime_manifest_file_sha256"], + ) + if dict(observed_runtime) != dict(trusted_runtime): + raise StageAError("Stage-A authenticated runtime evidence drifted") + device = runtime.get("device") + if not isinstance(device, Mapping): + raise StageAError("Stage-A device evidence is missing") + observed_device = _validated_device_runtime(device) + if dict(observed_device) != dict(trusted_device): + raise StageAError("Stage-A device runtime evidence drifted") + + rows = evidence.get("raw_token_evidence") + if not isinstance(rows, list) or not rows: + raise StageAError("Stage-A result has no raw transition evidence") + gate_artifact = evidence.get("stage_a_gate_artifact") + if not isinstance(gate_artifact, Mapping): + raise StageAError("Stage-A gate artifact is missing") + gate = importlib.import_module("recurquant.experiment013_stage_a") + gate_bytes = gate.canonical_json_bytes(dict(gate_artifact)) + if sha256_bytes(gate_bytes) != _require_sha256( + evidence.get("stage_a_gate_file_sha256"), + context="Stage-A gate artifact file SHA-256", + ): + raise StageAError("Stage-A gate artifact file identity drifted") + try: + verified_gate = gate.deserialize_stage_a_evidence_artifact( + gate_bytes, + expected_stage_a_identity_file_sha256=identity_hash, + expected_stage_a_calibration_binding_file_sha256=binding_hash, + ) + except (TypeError, ValueError) as error: + raise StageAError("Stage-A gate artifact reconstruction failed") from error + if ( + not isinstance(evidence.get("stage_a_passed"), bool) + or evidence.get("stage_a_passed") is not verified_gate.passed + ): + raise StageAError("Stage-A result passage scalar is invalid") + gate_evidence = verified_gate.evidence + gate_examples = gate_evidence.get("examples") + gate_rows = gate_evidence.get("token_rows") + if ( + not isinstance(gate_examples, tuple) + or len(gate_examples) != 12 + or not isinstance(gate_rows, tuple) + or len(gate_rows) != len(rows) + ): + raise StageAError("Stage-A outer and gate row counts differ") + derived_forward_count = 0 + for example in gate_examples: + if not isinstance(example, Mapping): + raise StageAError("Stage-A gate example is invalid") + continuation_count = _nonnegative_int( + example.get("continuation_token_count"), + context="Stage-A gate continuation token count", + ) + if continuation_count < 2: + raise StageAError("Stage-A gate continuation is too short") + derived_forward_count += len(METHOD_ORDER) * continuation_count + if derived_forward_count != expected_forward_count: + raise StageAError("Stage-A gate examples differ from the authenticated forward count") + shared_fields = ( + "canonical_id", + "family", + "identity_record_sha256", + "kl", + "method_id", + "method_nll", + "reference_nll", + "selection_rank", + "top1_agreement", + "transition_index", + ) + position_by_transition: dict[tuple[object, ...], tuple[int, int]] = {} + first_input_by_method_example: dict[tuple[object, ...], int] = {} + rows_by_method_example: dict[tuple[object, ...], list[Mapping[str, object]]] = {} + finite_checks = { + "comparison_logits": True, + "nll": True, + "kl": True, + "local_codec_sse": True, + "trajectory_nmse": True, + } + for row_index, (outer_row, gate_row) in enumerate(zip(rows, gate_rows, strict=True)): + if not isinstance(outer_row, Mapping) or not isinstance(gate_row, Mapping): + raise StageAError("Stage-A transition evidence is invalid") + if any( + name in outer_row + for name in ( + "input_token_id", + "input_token_ids_sha256", + "target_token_id", + "target_token_ids_sha256", + ) + ): + raise StageAError("Stage-A result exposes low-entropy per-token evidence") + _exact_fields( + outer_row, + RAW_TOKEN_EVIDENCE_FIELDS, + context=f"Stage-A raw transition evidence {row_index}", + ) + if any(outer_row.get(name) != gate_row.get(name) for name in shared_fields): + raise StageAError("Stage-A outer transition evidence differs from the gate") + method_id = outer_row.get("method_id") + if method_id not in METHOD_ORDER: + raise StageAError("Stage-A raw transition method is unknown") + transition_index = _nonnegative_int( + outer_row.get("transition_index"), + context="Stage-A transition index", + ) + input_position = _nonnegative_int( + outer_row.get("input_position"), + context="Stage-A input position", + ) + target_position = _nonnegative_int( + outer_row.get("target_position"), + context="Stage-A target position", + ) + if target_position != input_position + 1: + raise StageAError("Stage-A transition positions are not causal") + example_key = ( + outer_row.get("family"), + outer_row.get("canonical_id"), + outer_row.get("selection_rank"), + outer_row.get("identity_record_sha256"), + ) + method_example_key = (*example_key, method_id) + first_input = first_input_by_method_example.setdefault( + method_example_key, + input_position - transition_index, + ) + if first_input < 1 or input_position != first_input + transition_index: + raise StageAError("Stage-A transition positions are incomplete or reordered") + transition_key = (*example_key, transition_index) + reference_positions = position_by_transition.setdefault( + transition_key, + (input_position, target_position), + ) + if reference_positions != (input_position, target_position): + raise StageAError("Stage-A methods used different causal positions") + if outer_row.get("authenticated_transition_sha256") != _transition_hash( + identity_record_sha256=_require_sha256( + outer_row.get("identity_record_sha256"), + context="Stage-A raw identity record SHA-256", + ), + method_id=cast(str, method_id), + transition_index=transition_index, + input_position=input_position, + target_position=target_position, + ): + raise StageAError("Stage-A transition commitment drifted") + reference_nll = _finite_number( + outer_row.get("reference_nll"), + context="Stage-A reference NLL", + nonnegative=True, + ) + method_nll = _finite_number( + outer_row.get("method_nll"), + context="Stage-A method NLL", + nonnegative=True, + ) + excess_nll = _finite_number( + outer_row.get("excess_nll"), + context="Stage-A excess NLL", + nonnegative=False, + ) + if excess_nll != method_nll - reference_nll: + raise StageAError("Stage-A excess NLL differs from its component values") + for name in ("kl", "local_codec_sse", "trajectory_nmse"): + _finite_number( + outer_row.get(name), + context=f"Stage-A {name}", + nonnegative=True, + ) + if not isinstance(outer_row.get("top1_agreement"), bool): + raise StageAError("Stage-A top-1 agreement is not boolean") + for name in ( + "decode_model_forward_latency_ns", + "decode_cuda_diagnostic_peak_allocated_bytes", + "decode_cuda_diagnostic_peak_reserved_bytes", + "decode_logical_recurrent_resident_bytes", + "method_cumulative_cache_reported_workspace_high_water_sum_bytes", + ): + _nonnegative_int(outer_row.get(name), context=f"Stage-A {name}") + if ( + outer_row.get("decode_logical_recurrent_resident_bytes") + != (EXPECTED_RECURRENT_RESIDENT_BYTES[cast(str, method_id)]) + ): + raise StageAError("Stage-A recurrent resident-byte ledger drifted") + checks = outer_row.get("finite_checks") + if not isinstance(checks, Mapping) or dict(checks) != finite_checks: + raise StageAError("Stage-A finite-check receipt drifted") + rows_by_method_example.setdefault(method_example_key, []).append(outer_row) + + method_runtime = evidence.get("method_runtime") + expected_runtime_count = len(gate_examples) * len(METHOD_ORDER) + if not isinstance(method_runtime, list) or len(method_runtime) != expected_runtime_count: + raise StageAError("Stage-A method runtime inventory is incomplete") + runtime_index = 0 + for example in gate_examples: + continuation_count = cast(int, example["continuation_token_count"]) + example_key = ( + example["family"], + example["canonical_id"], + example["selection_rank"], + example["identity_record_sha256"], + ) + for method_id in METHOD_ORDER: + runtime_row = method_runtime[runtime_index] + runtime_index += 1 + if not isinstance(runtime_row, Mapping): + raise StageAError("Stage-A method runtime row is invalid") + _exact_fields( + runtime_row, + METHOD_RUNTIME_FIELDS, + context=f"Stage-A method runtime row {runtime_index - 1}", + ) + if ( + tuple( + runtime_row.get(name) + for name in ( + "family", + "canonical_id", + "selection_rank", + "identity_record_sha256", + ) + ) + != example_key + or runtime_row.get("method_id") != method_id + ): + raise StageAError("Stage-A method runtime rows are reordered or misbound") + policy_hash = runtime_row.get("policy_file_sha256") + if policy_hash is not None: + _require_sha256(policy_hash, context="Stage-A method policy file SHA-256") + policy_origin = _bounded_text( + runtime_row.get("policy_origin"), + context="Stage-A policy origin", + ) + expected_spec = expected_specs[METHOD_ORDER.index(method_id)] + if ( + policy_hash != expected_spec["policy_file_sha256"] + or policy_origin != expected_spec["policy_origin"] + ): + raise StageAError("Stage-A method runtime policy identity drifted") + _nonnegative_int( + runtime_row.get("wall_time_with_diagnostics_ns"), + context="Stage-A method wall time", + ) + prefill = runtime_row.get("prefill_diagnostics") + if not isinstance(prefill, Mapping): + raise StageAError("Stage-A prefill diagnostics are missing") + _exact_fields( + prefill, + PREFILL_DIAGNOSTIC_FIELDS, + context="Stage-A prefill diagnostics", + ) + for name in PREFILL_DIAGNOSTIC_FIELDS: + _nonnegative_int(prefill.get(name), context=f"Stage-A prefill {name}") + expected_resident = EXPECTED_RECURRENT_RESIDENT_BYTES[method_id] + if prefill.get("logical_recurrent_resident_bytes") != expected_resident: + raise StageAError("Stage-A prefill resident-byte ledger drifted") + group = rows_by_method_example.get((*example_key, method_id), []) + if len(group) != continuation_count - 1: + raise StageAError("Stage-A method runtime has an incomplete decode trajectory") + max_allocated = _nonnegative_int( + runtime_row.get( + "max_cuda_diagnostic_peak_allocated_bytes_across_prefill_and_decode" + ), + context="Stage-A method maximum allocated bytes", + ) + max_reserved = _nonnegative_int( + runtime_row.get( + "max_cuda_diagnostic_peak_reserved_bytes_across_prefill_and_decode" + ), + context="Stage-A method maximum reserved bytes", + ) + expected_max_allocated = max( + cast(int, prefill["cuda_diagnostic_peak_allocated_bytes"]), + *(cast(int, row["decode_cuda_diagnostic_peak_allocated_bytes"]) for row in group), + ) + expected_max_reserved = max( + cast(int, prefill["cuda_diagnostic_peak_reserved_bytes"]), + *(cast(int, row["decode_cuda_diagnostic_peak_reserved_bytes"]) for row in group), + ) + if max_allocated != expected_max_allocated or max_reserved != expected_max_reserved: + raise StageAError("Stage-A method CUDA maximum diagnostics drifted") + storage = runtime_row.get("storage") + if not isinstance(storage, Mapping): + raise StageAError("Stage-A method storage receipt is missing") + _exact_fields( + storage, + STORAGE_RECEIPT_FIELDS, + context="Stage-A method storage receipt", + ) + for name in STORAGE_RECEIPT_FIELDS - {"workspace_scope"}: + _nonnegative_int(storage.get(name), context=f"Stage-A storage {name}") + if ( + storage.get("logical_recurrent_resident_bytes") != expected_resident + or storage.get("cache_reported_resident_bytes") != expected_resident + or storage.get("cache_reported_expected_resident_bytes") != expected_resident + or storage.get("workspace_scope") + != "method_lifetime_high_water_since_cache_creation" + ): + raise StageAError("Stage-A method storage ledger drifted") + final_workspace = cast( + int, + group[-1]["method_cumulative_cache_reported_workspace_high_water_sum_bytes"], + ) + if final_workspace != cast(int, storage["raw_state_workspace_high_water_bytes"]) + cast( + int, storage["query_workspace_high_water_bytes"] + ): + raise StageAError("Stage-A method workspace high-water receipt drifted") + return MappingProxyType(dict(root)) + + +def _completion_marker_bytes( + config: StageAConfig, + *, + result_file_sha256: str, +) -> bytes: + result_hash = _require_sha256( + result_file_sha256, + context="completion marker result SHA-256", + ) + attempt_bytes = _stable_file_bytes( + config.attempt_path, + context="completed Stage-A attempt receipt", + ) + return canonical_json_bytes( + { + "attempt_file_sha256": sha256_bytes(attempt_bytes), + "result_file_sha256": result_hash, + "status": "complete", + } + ) + + +def publish_result( + config: StageAConfig, + reservation: AttemptReservation, + payload: bytes, + summary: Mapping[str, object], +) -> Mapping[str, object]: + verify_execution_artifact( + payload, + expected_identity_file_sha256=_require_sha256( + reservation.receipt.get("identity_file_sha256"), + context="attempt identity file SHA-256", + ), + expected_calibration_binding_file_sha256=_require_sha256( + reservation.receipt.get("calibration_binding_file_sha256"), + context="attempt calibration binding file SHA-256", + ), + expected_h1_commit=reservation.h1_commit, + expected_seal_commit=reservation.seal_commit, + expected_source_commit=_require_sha1( + reservation.receipt.get("h0_source_commit"), + context="attempt H0 source commit", + ), + expected_source_manifest_file_sha256=_require_sha256( + reservation.receipt.get("source_manifest_file_sha256"), + context="attempt source manifest file SHA-256", + ), + expected_input_bundle_manifest_file_sha256=_require_sha256( + reservation.receipt.get("stage_a_input_bundle_manifest_file_sha256"), + context="attempt Stage-A input bundle manifest SHA-256", + ), + expected_execution_bindings=cast( + Mapping[str, str], + reservation.receipt.get("execution_bindings"), + ), + expected_method_specs=cast( + Sequence[Mapping[str, object]], + reservation.receipt.get("method_specs"), + ), + expected_materialization={ + "sequence_count": 12, + "capture_input_sha256": reservation.receipt.get("capture_input_sha256"), + "token_sequence_manifest_sha256": reservation.receipt.get( + "token_sequence_manifest_sha256" + ), + "tokenizer_manifest_sha256": reservation.receipt.get("tokenizer_manifest_sha256"), + }, + expected_seal_tree=reservation.tree, + expected_seal_message_sha256=_require_sha256( + reservation.receipt.get("one_run_seal_message_sha256"), + context="attempt seal message SHA-256", + ), + expected_attempt_lock_file_sha256=_require_sha256( + reservation.receipt.get("identity_scoped_attempt_lock_file_sha256"), + context="attempt identity-scoped lock SHA-256", + ), + expected_authenticated_runtime=cast( + Mapping[str, object], + reservation.receipt.get("post_load_authenticated_runtime"), + ), + expected_device_runtime=cast( + Mapping[str, object], + reservation.receipt.get("post_load_device_runtime"), + ), + expected_forward_count=_nonnegative_int( + reservation.receipt.get("expected_forward_count"), + context="attempt expected forward count", + ), + ) + file_hash = sha256_bytes(payload) + document = _strict_json(payload, context="prepared Stage-A result") + canonical_hash = _require_sha256( + document.get("canonical_evidence_sha256"), context="result canonical evidence SHA-256" + ) + prepared = persist_receipt( + config, + reservation, + { + "status": "result_prepared_before_atomic_publication", + "observed_forward_count": reservation.receipt["expected_forward_count"], + "content_materialized": True, + "model_load_count": 1, + "evaluation_complete": True, + "result_available": False, + "result_file_sha256": file_hash, + "result_canonical_evidence_sha256": canonical_hash, + "stage_a_passed": bool(summary["stage_a_passed"]), + }, + ) + _atomic_publish_new(config.output_path, payload) + if _stable_file_bytes(config.output_path, context="published Stage-A result") != payload: + raise StageAError("published Stage-A output changed on disk") + completed = persist_receipt( + config, + prepared, + { + "status": "completed_with_authenticated_stage_a_result", + "result_available": True, + "completed_at_utc": datetime.now(UTC).isoformat(), + }, + ) + marker = _completion_marker_bytes(config, result_file_sha256=file_hash) + _atomic_publish_new(config.complete_path, marker) + return { + "status": completed.receipt["status"], + "output": str(config.output_path.resolve()), + "artifact_file_sha256": file_hash, + "canonical_evidence_sha256": canonical_hash, + "stage_a_passed": summary["stage_a_passed"], + "claim_boundary": CLAIM_BOUNDARY, + } + + +def record_failure( + config: StageAConfig, + reservation: AttemptReservation | None, + error: BaseException, + phase: str, +) -> None: + if reservation is None or not config.attempt_path.is_file(): + return + try: + disk = _strict_json( + _stable_file_bytes(config.attempt_path, context="Stage-A attempt receipt"), + context="Stage-A attempt receipt", + ) + for field in ( + "schema", + "h1_identity_commit", + "one_run_seal_commit", + "identity_file_sha256", + "identity_scoped_attempt_lock_file_sha256", + ): + if disk.get(field) != reservation.receipt.get(field): + raise StageAError("Stage-A attempt identity changed before failure recording") + current = dataclasses.replace(reservation, receipt=MappingProxyType(disk)) + detail_hash = sha256_bytes(str(error).encode("utf-8", errors="replace")) + persist_receipt( + config, + current, + { + "status": "consumed_attempt_failed_no_automatic_retry", + "failure_phase": phase, + "failure_type": type(error).__name__, + "failure_detail_sha256": detail_hash, + "failure_detail_recorded": False, + "result_available": config.output_path.is_file(), + }, + ) + except BaseException as receipt_error: + raise StageAError("could not preserve an authenticated failure receipt") from receipt_error + + +def _validated_preseal_engine_smoke(value: Mapping[str, object]) -> Mapping[str, object]: + _exact_fields( + value, + { + "profile", + "passed", + "stage_a_content_accessed", + "input_profile", + "prompt_token_count", + "continuation_token_count", + "model_load_count", + "method_order", + "forward_count", + "method_receipts", + "device", + }, + context="pre-seal engine smoke", + ) + if ( + value.get("profile") != PRESEAL_ENGINE_SMOKE_PROFILE + or value.get("passed") is not True + or value.get("stage_a_content_accessed") is not False + or value.get("input_profile") != "fixed_public_synthetic_4096_plus_128_v3" + or value.get("prompt_token_count") != PRESEAL_ENGINE_SMOKE_PROMPT_TOKEN_COUNT + or value.get("continuation_token_count") != PRESEAL_ENGINE_SMOKE_TARGET_TOKEN_COUNT + or value.get("model_load_count") != 1 + or value.get("method_order") != list(METHOD_ORDER) + or value.get("forward_count") != len(METHOD_ORDER) * len(PRESEAL_ENGINE_SMOKE_TARGET) + ): + raise StageAError("pre-seal engine smoke contract drifted") + receipts = value.get("method_receipts") + if not isinstance(receipts, list) or len(receipts) != len(METHOD_ORDER): + raise StageAError("pre-seal engine smoke method receipts are incomplete") + for method_id, receipt in zip(METHOD_ORDER, receipts, strict=True): + if not isinstance(receipt, Mapping): + raise StageAError("pre-seal engine smoke method receipt is invalid") + _exact_fields( + receipt, + { + "method_id", + "forward_count", + "logical_recurrent_resident_bytes", + "equal_byte_observer_required", + }, + context=f"pre-seal engine smoke {method_id}", + ) + if ( + receipt.get("method_id") != method_id + or receipt.get("forward_count") != len(PRESEAL_ENGINE_SMOKE_TARGET) + or receipt.get("logical_recurrent_resident_bytes") + != EXPECTED_RECURRENT_RESIDENT_BYTES[method_id] + or receipt.get("equal_byte_observer_required") is not (method_id != FP32_METHOD) + ): + raise StageAError(f"pre-seal engine smoke {method_id} receipt drifted") + device = value.get("device") + if not isinstance(device, Mapping): + raise StageAError("pre-seal engine smoke device receipt is missing") + _validated_device_runtime(device) + return MappingProxyType( + _strict_json(canonical_json_bytes(value), context="pre-seal engine smoke") + ) + + +def run_stage_a(config: StageAConfig, services: StageAServices) -> Mapping[str, object]: + """Order auth -> seal -> materialize -> load once -> evaluate -> publish.""" + + reservation: AttemptReservation | None = None + phase = "metadata_authentication" + authenticated = services.authenticate(config) + try: + phase = "preseal_engine_smoke" + smoke = _validated_preseal_engine_smoke(dict(services.preseal_smoke(authenticated))) + phase = "post_preseal_engine_smoke_reauthentication" + services.reauthenticate(config, authenticated, None) + phase = "one_run_reservation" + reservation = services.reserve(config, authenticated) + smoke_bytes = canonical_json_bytes(dict(smoke)) + reservation = services.persist_receipt( + config, + reservation, + { + "status": "preseal_engine_smoke_bound_before_materialization", + "preseal_engine_smoke": dict(smoke), + "preseal_engine_smoke_sha256": sha256_bytes(smoke_bytes), + }, + ) + phase = "pre_materialization_reauthentication" + services.reauthenticate(config, authenticated, reservation) + reservation = services.persist_receipt( + config, + reservation, + {"status": "stage_a_materialization_entered"}, + ) + phase = "stage_a_materialization" + materialization = services.materialize(config, authenticated) + if getattr(materialization, "frozen_identity_file_sha256", None) != ( + authenticated.bootstrap_identity.file_sha256 + ): + raise StageAError("materializer returned a different frozen Stage-A identity") + if ( + getattr(materialization, "calibration_binding_file_sha256", None) + != authenticated.binding.file_sha256 + ): + raise StageAError("materializer returned a different calibration binding") + reservation = services.persist_receipt( + config, + reservation, + { + "status": "stage_a_content_materialized_before_model_load", + "content_materialized": True, + "capture_input_sha256": materialization.capture_input_sha256, + "token_sequence_manifest_sha256": materialization.token_sequence_manifest_sha256, + "tokenizer_manifest_sha256": materialization.tokenizer_manifest_sha256, + }, + ) + phase = "pre_model_reauthentication" + services.reauthenticate(config, authenticated, reservation) + phase = "model_load" + model = services.engine.load_model(authenticated.authenticated_model_files) + try: + phase = "post_model_load_reauthentication" + services.reauthenticate(config, authenticated, reservation) + runtime_manifest_hash = _require_sha256( + authenticated.bootstrap_identity.execution_bindings.get( + "calibration_runtime_manifest_file_sha256" + ), + context="Stage-A runtime execution binding", + ) + post_load_authenticated_runtime = _authenticated_runtime_record( + authenticated.authenticated_runtime, + expected_manifest_file_sha256=runtime_manifest_hash, + ) + post_load_device_runtime = _validated_device_runtime( + dict(services.engine.runtime_snapshot(model)) + ) + phase = "post_model_load_receipt" + reservation = services.persist_receipt( + config, + reservation, + { + "status": "model_loaded_once_before_evaluation", + "model_load_count": 1, + "post_load_authenticated_runtime": dict(post_load_authenticated_runtime), + "post_load_device_runtime": dict(post_load_device_runtime), + }, + ) + phase = "evaluation" + evaluation = evaluate_materialized_stage_a( + authenticated, + materialization, + services.engine, + model, + ) + phase = "post_evaluation_device_reauthentication" + if dict(evaluation.device_runtime) != dict(post_load_device_runtime): + raise StageAError( + "Stage-A device runtime drifted between model load and evaluation end" + ) + finally: + services.engine.close_model(model) + reservation = services.persist_receipt( + config, + reservation, + { + "status": "evaluation_returned_before_result_build", + "observed_forward_count": evaluation.forward_count, + "evaluation_complete": True, + }, + ) + phase = "post_evaluation_reauthentication" + services.reauthenticate(config, authenticated, reservation) + phase = "artifact_build" + payload = build_execution_artifact( + authenticated, + materialization, + evaluation, + reservation, + ) + document = _strict_json(payload, context="Stage-A execution artifact") + evidence = cast(Mapping[str, object], document["evidence"]) + phase = "pre_publication_reauthentication" + services.reauthenticate(config, authenticated, reservation) + phase = "result_publication" + return services.publish( + config, + reservation, + payload, + {"stage_a_passed": evidence["stage_a_passed"]}, + ) + except BaseException as error: + services.record_failure(config, reservation, error, phase) + raise + + +def _runtime_context(config: StageAConfig) -> dict[str, object]: + if ( + config.base_runtime_root is None + or config.git_executable_path is None + or config.interpreter_path is None + ): + raise StageAError("sealed runner did not receive its runtime authentication context") + return { + "base_runtime_root": config.base_runtime_root, + "git_executable": config.git_executable_path, + "staged_interpreter": config.interpreter_path, + "package_runtime_roots": dict(config.package_roots), + "package_import_paths": dict(config.package_import_paths), + } + + +def _read_execution_artifacts(config: StageAConfig) -> dict[str, bytes]: + paths = { + "repository_source_manifest_file_sha256": config.repository_source_manifest_path, + "calibration_runtime_manifest_file_sha256": config.runtime_manifest_path, + "model_file_manifest_file_sha256": config.model_file_manifest_path, + "parquet_materialization_manifest_file_sha256": ( + config.parquet_materialization_manifest_path + ), + } + return { + name: _stable_file_bytes(path, context=f"execution artifact {name}") + for name, path in paths.items() + } + + +def authenticate_production( + config: StageAConfig, + *, + require_input_bundle: bool = True, +) -> AuthenticatedStageA: + _assert_output_paths_isolated(config) + identity_bytes = _stable_file_bytes(config.frozen_identity_path, context="frozen identity") + bootstrap = bootstrap_stage_a_identity(identity_bytes) + binding_bytes = _stable_file_bytes( + config.calibration_binding_path, context="Stage-A calibration binding" + ) + execution = _read_execution_artifacts(config) + if {name: sha256_bytes(data) for name, data in execution.items()} != dict( + bootstrap.execution_bindings + ): + raise StageAError("one or more execution artifacts differ from identity v5") + expected_cli = { + "calibration_runtime_manifest_file_sha256": _require_sha256( + config.expected_runtime_manifest_sha256, context="expected runtime manifest SHA-256" + ), + "model_file_manifest_file_sha256": _require_sha256( + config.expected_model_file_manifest_sha256, context="expected model manifest SHA-256" + ), + "parquet_materialization_manifest_file_sha256": _require_sha256( + config.expected_parquet_materialization_manifest_sha256, + context="expected Parquet manifest SHA-256", + ), + } + if any(bootstrap.execution_bindings[name] != digest for name, digest in expected_cli.items()): + raise StageAError("explicit expected execution-artifact digest differs from identity v5") + source_bootstrap = _bootstrap_source_manifest( + execution["repository_source_manifest_file_sha256"] + ) + if source_bootstrap["source_commit"] != _require_sha1( + config.source_commit, context="explicit H0 source commit" + ): + raise StageAError("explicit H0 differs from the authenticated source manifest") + _verify_source_bytes(source_bootstrap, config.repository_root) + entries = _source_entries(source_bootstrap) + _install_source_namespace(config.repository_root) + source_module = _load_exact_module( + "recurquant.experiment013_source", + SOURCE_MODULE_PATH, + repository_root=config.repository_root, + entries=entries, + ) + calibration_runner = _load_exact_module( + "_recurquant_experiment013_calibration_runner_for_stage_a", + CALIBRATION_RUNNER_SOURCE_PATH, + repository_root=config.repository_root, + entries=entries, + ) + resolver = _load_exact_module( + "recurquant_experiment013_identity_resolver", + RESOLVER_SOURCE_PATH, + repository_root=config.repository_root, + entries=entries, + ) + normalized_source = source_module.validate_experiment013_source_manifest( + source_bootstrap["document"] + ) + if ( + source_module.verify_experiment013_source_manifest( + normalized_source, + config.repository_root, + git_executable=config.git_executable_path, + ) + != normalized_source + ): + raise StageAError("source verifier returned a different H0 manifest") + _assert_tracked_identity_bytes(config, identity_bytes) + binding = resolver.deserialize_stage_a_calibration_binding_artifact(binding_bytes) + dependencies = _decode_binding_dependencies(binding_bytes) + if any( + sha256_bytes(dependencies[name]) + != bootstrap.calibration_binding[_binding_field_for_dependency(name)] + for name in BINDING_DEPENDENCY_NAMES + ): + raise StageAError("identity v5 eight-field binding differs from embedded dependencies") + identity = resolver.deserialize_frozen_stage_a_identity_artifact( + identity_bytes, + calibration_binding_artifact=binding_bytes, + expected_file_sha256=bootstrap.file_sha256, + ) + calibration = importlib.import_module("recurquant.static_q468_calibration") + split = calibration.deserialize_frozen_split_half_stability_artifact( + dependencies["split_half_stability_artifact"] + ) + if split.stability.passed is not True: + raise StageAStabilityFailure("frozen split-half stability gate did not pass") + runtime_manifest = calibration_runner.parse_calibration_runtime_manifest( + execution["calibration_runtime_manifest_file_sha256"] + ) + if source_bootstrap["git_executable"] != { + "sha256": runtime_manifest.git_executable_sha256, + "size_bytes": runtime_manifest.git_executable_size_bytes, + }: + raise StageAError("source and runtime manifests bind different Git executable bytes") + authenticated_runtime = calibration_runner.authenticate_calibration_runtime( + runtime_manifest, + base_runtime_root=config.base_runtime_root, + package_roots=config.package_roots, + interpreter_path=config.interpreter_path, + git_executable_path=config.git_executable_path, + ) + model_manifest = calibration_runner.parse_model_file_manifest( + execution["model_file_manifest_file_sha256"] + ) + authenticated_model = calibration_runner.authenticate_local_model_files( + config.model_root, + model_manifest, + ) + capture = _load_exact_module( + "_recurquant_experiment013_stage_a_capture", + CAPTURE_SOURCE_PATH, + repository_root=config.repository_root, + entries=entries, + ) + input_bundle = None + input_bundle_manifest_file_sha256 = None + if require_input_bundle: + input_bundle = capture.authenticate_stage_a_input_bundle( + config.input_bundle_root, + frozen_stage_a_identity_artifact=identity_bytes, + calibration_binding_artifact=binding_bytes, + execution_binding_artifacts=execution, + ) + input_bundle_manifest_file_sha256 = _require_sha256( + getattr(input_bundle, "manifest_file_sha256", None), + context="authenticated Stage-A input bundle manifest SHA-256", + ) + methods = reconstruct_stage_a_methods( + dependency_bytes=dependencies, + frozen_stage_a_identity=identity, + source_commit=cast(str, source_bootstrap["source_commit"]), + ) + return AuthenticatedStageA( + bootstrap_identity=bootstrap, + identity=identity, + binding=binding, + dependency_bytes=MappingProxyType(dependencies), + execution_artifact_bytes=MappingProxyType(execution), + source_manifest=MappingProxyType(normalized_source), + source_manifest_file_sha256=sha256_bytes( + execution["repository_source_manifest_file_sha256"] + ), + source_commit=cast(str, source_bootstrap["source_commit"]), + input_bundle=input_bundle, + input_bundle_manifest_file_sha256=input_bundle_manifest_file_sha256, + model_manifest=model_manifest, + authenticated_model_files=authenticated_model, + runtime_manifest=runtime_manifest, + authenticated_runtime=authenticated_runtime, + resolver=resolver, + capture=capture, + calibration_runner=calibration_runner, + source_module=source_module, + methods=methods, + ) + + +def reauthenticate_production( + config: StageAConfig, + previous: AuthenticatedStageA, + reservation: AttemptReservation | None, +) -> None: + execution = _read_execution_artifacts(config) + if execution != dict(previous.execution_artifact_bytes): + raise StageAError("execution artifact bytes changed during Stage A") + identity_bytes = _stable_file_bytes(config.frozen_identity_path, context="frozen identity") + if sha256_bytes(identity_bytes) != previous.bootstrap_identity.file_sha256: + raise StageAError("frozen Stage-A identity changed during evaluation") + if reservation is None: + _assert_tracked_identity_bytes(config, identity_bytes) + else: + _assert_tracked_identity_bytes_after_seal( + config, + identity_bytes, + previous, + reservation, + ) + if previous.source_module.verify_experiment013_source_manifest( + previous.source_manifest, + config.repository_root, + git_executable=config.git_executable_path, + ) != dict(previous.source_manifest): + raise StageAError("H0 source identity changed during Stage A") + runtime = previous.calibration_runner.authenticate_calibration_runtime( + previous.runtime_manifest, + base_runtime_root=config.base_runtime_root, + package_roots=config.package_roots, + interpreter_path=config.interpreter_path, + git_executable_path=config.git_executable_path, + ) + if runtime != previous.authenticated_runtime: + raise StageAError("sealed runtime changed during Stage A") + model = previous.calibration_runner.authenticate_local_model_files( + config.model_root, previous.model_manifest + ) + if model != previous.authenticated_model_files: + raise StageAError("authenticated model files changed during Stage A") + previous.resolver.deserialize_stage_a_calibration_binding_artifact( + _stable_file_bytes(config.calibration_binding_path, context="calibration binding"), + expected_file_sha256=previous.binding.file_sha256, + ) + if previous.input_bundle is None or previous.input_bundle_manifest_file_sha256 is None: + raise StageAError("authenticated Stage-A input bundle is unavailable") + bundle = previous.capture.authenticate_stage_a_input_bundle( + config.input_bundle_root, + frozen_stage_a_identity_artifact=identity_bytes, + calibration_binding_artifact=_stable_file_bytes( + config.calibration_binding_path, + context="calibration binding for Stage-A input bundle", + ), + execution_binding_artifacts=execution, + ) + if getattr(bundle, "manifest_file_sha256", None) != ( + previous.input_bundle_manifest_file_sha256 + ): + raise StageAError("Stage-A input bundle manifest changed during Stage A") + + +def _assert_tracked_identity_bytes_after_seal( + config: StageAConfig, + identity_bytes: bytes, + authenticated: AuthenticatedStageA, + reservation: AttemptReservation, +) -> None: + head = _require_sha1( + _git(config.git_executable_path, config.repository_root, "rev-parse", "HEAD"), + context="HEAD", + ) + if head != reservation.seal_commit: + raise StageAError("HEAD differs from the reserved one-run seal commit") + parents = _git( + config.git_executable_path, + config.repository_root, + "show", + "-s", + "--format=%P", + head, + ).split() + if ( + len(parents) != 1 + or parents[0] != config.identity_commit + or parents[0] != reservation.h1_commit + ): + raise StageAError("HEAD is not the one-run seal child of the identity commit") + identity = config.frozen_identity_path.resolve(strict=True) + relative = identity.relative_to(config.repository_root.resolve(strict=True)).as_posix() + commit = _git_process( + config.git_executable_path, + config.repository_root, + "show", + f"{head}:{relative}", + ) + index = _git_process( + config.git_executable_path, + config.repository_root, + "show", + f":{relative}", + ) + if commit.returncode != 0 or index.returncode != 0: + raise StageAError("sealed identity disappeared from Git") + if commit.stdout != identity_bytes or index.stdout != identity_bytes: + raise StageAError("sealed identity differs across HEAD, index, and worktree") + if _git( + config.git_executable_path, + config.repository_root, + "show", + "-s", + "--format=%T", + head, + ) != _git( + config.git_executable_path, + config.repository_root, + "show", + "-s", + "--format=%T", + config.identity_commit, + ): + raise StageAError("one-run seal is not an empty-diff commit") + expected_message = _seal_message(authenticated).encode("utf-8") + commit = _git_process( + config.git_executable_path, + config.repository_root, + "cat-file", + "commit", + head, + ) + if commit.returncode != 0: + raise StageAError("cannot read the reserved one-run seal commit") + _headers, separator, message = commit.stdout.partition(b"\n\n") + if ( + not separator + or message != expected_message + or sha256_bytes(expected_message) != reservation.receipt.get("one_run_seal_message_sha256") + ): + raise StageAError("reserved one-run seal message drifted") + if authenticated.source_commit != config.source_commit: + raise StageAError("authenticated H0 source commit changed") + + +def materialize_production(config: StageAConfig, authenticated: AuthenticatedStageA) -> object: + if authenticated.input_bundle is None: + raise StageAError("authenticated Stage-A input bundle is unavailable") + source = authenticated.capture.StagedCaptureSource(authenticated.input_bundle) + materialization = authenticated.capture.materialize_stage_a_identity_sequences( + source=source, + frozen_stage_a_identity_artifact=_stable_file_bytes( + config.frozen_identity_path, context="frozen Stage-A identity" + ), + calibration_binding_artifact=_stable_file_bytes( + config.calibration_binding_path, context="Stage-A calibration binding" + ), + expected_frozen_stage_a_identity_file_sha256=(authenticated.bootstrap_identity.file_sha256), + execution_binding_artifacts=dict(authenticated.execution_artifact_bytes), + runtime_authentication_context=_runtime_context(config), + ) + return materialization + + +def prepare_inputs_production(config: StageAConfig) -> Mapping[str, object]: + """Stage opaque public bytes before the one-run seal in this network-only child.""" + + authenticated = authenticate_production(config, require_input_bundle=False) + identity_bytes = _stable_file_bytes(config.frozen_identity_path, context="frozen identity") + binding_bytes = _stable_file_bytes( + config.calibration_binding_path, + context="Stage-A calibration binding", + ) + bundle = authenticated.capture.stage_stage_a_input_bundle( + bundle_root=config.input_bundle_root, + cache_dir=config.cache_root, + ruler_receipt_dir=config.ruler_root, + frozen_stage_a_identity_artifact=identity_bytes, + calibration_binding_artifact=binding_bytes, + execution_binding_artifacts=dict(authenticated.execution_artifact_bytes), + runtime_authentication_context=_runtime_context(config), + ) + return MappingProxyType( + { + "status": "stage_a_input_bundle_prepared", + "stage_a_input_bundle_manifest_file_sha256": _require_sha256( + getattr(bundle, "manifest_file_sha256", None), + context="prepared Stage-A input bundle manifest SHA-256", + ), + "identity_file_sha256": authenticated.bootstrap_identity.file_sha256, + "calibration_binding_file_sha256": authenticated.binding.file_sha256, + "source_commit": authenticated.source_commit, + "content_materialized": False, + "model_instantiated": False, + "one_run_reserved": False, + } + ) + + +class TorchStageAEngine: + """Reviewed Qwen3.5 execution path; all imports and weights are late.""" + + def __init__(self) -> None: + self._torch: Any = None + self._model: object | None = None + self._reference_states: dict[str, list[dict[int, object]]] = {} + + @staticmethod + def _assert_loaded_model_contract( + model: object, + *, + torch: object, + transformers: object, + device: object, + ) -> None: + if type(model) is not getattr(transformers, "Qwen3_5ForCausalLM", None): + raise StageAError("loaded model is not the pinned Qwen3.5 causal-LM class") + model_config = getattr(model, "config", None) + if type(model_config) is not getattr(transformers, "Qwen3_5TextConfig", None): + raise StageAError("loaded model config is not the pinned Qwen3.5 text config class") + if ( + getattr(model_config, "_attn_implementation", None) != "eager" + or getattr(model_config, "_attn_implementation_internal", None) != "eager" + ): + raise StageAError("loaded Stage-A model did not retain eager attention") + parameters = tuple(model.parameters()) + if not parameters: + raise StageAError("loaded Stage-A model has no parameters") + floating_parameter_count = 0 + for index, parameter in enumerate(parameters): + if getattr(parameter, "device", None) != device: + raise StageAError(f"loaded Stage-A parameter {index} is not on the one CUDA device") + is_floating_point = getattr(parameter, "is_floating_point", None) + if not callable(is_floating_point): + raise StageAError(f"loaded Stage-A parameter {index} has no dtype contract") + if is_floating_point(): + floating_parameter_count += 1 + if getattr(parameter, "dtype", None) != getattr(torch, "bfloat16", None): + raise StageAError(f"loaded Stage-A floating parameter {index} is not BF16") + if floating_parameter_count == 0: + raise StageAError("loaded Stage-A model has no floating parameters") + + def load_model(self, authenticated_model_files: object) -> object: + if self._model is not None: + raise StageAError("Stage-A model may be loaded exactly once") + torch = importlib.import_module("torch") + transformers = importlib.import_module("transformers") + if not torch.cuda.is_available(): + raise StageAError("official Stage A requires CUDA") + root = getattr(authenticated_model_files, "model_root", None) + if not isinstance(root, Path): + raise StageAError("authenticated model root is unavailable") + config = transformers.Qwen3_5TextConfig.from_pretrained( + str(root), local_files_only=True, trust_remote_code=False + ) + loaded = transformers.Qwen3_5ForCausalLM.from_pretrained( + str(root), + config=config, + dtype=torch.bfloat16, + attn_implementation="eager", + low_cpu_mem_usage=True, + use_safetensors=True, + weights_only=True, + local_files_only=True, + trust_remote_code=False, + output_loading_info=True, + ) + if type(loaded) is not tuple or len(loaded) != 2: + raise StageAError("Transformers did not return model loading diagnostics") + model, diagnostics = loaded + if not isinstance(diagnostics, dict) or any(diagnostics.values()): + raise StageAError("authenticated model load reported missing or unexpected weights") + device = torch.device("cuda", torch.cuda.current_device()) + model = model.to(device) + model.eval() + model.requires_grad_(False) + self._assert_loaded_model_contract( + model, + torch=torch, + transformers=transformers, + device=device, + ) + self._torch = torch + self._model = model + return model + + def close_model(self, model: object) -> None: + if model is not self._model: + raise StageAError("attempted to close a different Stage-A model") + self._model = None + self._reference_states.clear() + with contextlib.suppress(Exception): + self._torch.cuda.empty_cache() + + def runtime_snapshot(self, model: object) -> Mapping[str, object]: + if model is not self._model: + raise StageAError("cannot snapshot a different Stage-A model") + torch = self._torch + parameter = next(model.parameters()) + device = parameter.device + if device.type != "cuda" or device.index is None: + raise StageAError("official Stage A model is not on one explicit CUDA device") + torch.cuda.synchronize(device) + properties = torch.cuda.get_device_properties(device) + cuda_runtime = torch.version.cuda + if not isinstance(cuda_runtime, str) or not cuda_runtime: + raise StageAError("Stage-A Torch runtime has no CUDA version") + return _validated_device_runtime( + { + "attention_implementation": str(model.config._attn_implementation), + "capability": list(torch.cuda.get_device_capability(device)), + "cuda_runtime": cuda_runtime, + "device_index": int(device.index), + "model_class": type(model).__name__, + "model_config_class": type(model.config).__name__, + "model_parameter_dtype": str(parameter.dtype), + "name": str(torch.cuda.get_device_name(device)), + "torch_version": str(torch.__version__), + "total_memory_bytes": int(properties.total_memory), + } + ) + + def _cache(self, model: object, method: StageAMethodSpec) -> tuple[object, object | None]: + transformers = importlib.import_module("transformers") + if method.method_id == FP32_METHOD: + return transformers.DynamicCache(config=model.config), None + cache_module = importlib.import_module("recurquant.static_q468_cache") + if method.method_id == DYNAMIC_K27030_METHOD: + cache = cache_module.create_qwen35_dynamic_q468_baseline_cache( + model, record_evidence=True + ) + else: + if method.policy is None or method.policy_file_sha256 is None: + raise StageAError(f"static method {method.method_id} has no authenticated policy") + cache = cache_module.create_qwen35_static_rht_cache( + model, + policy=method.policy, + expected_policy_sha256=method.policy.policy_sha256, + record_evidence=True, + ) + observer_module = importlib.import_module("recurquant.statelease_equal_byte_cache") + observer = observer_module.Qwen35EqualByteObserver(model, caches=[cache]) + return cache, observer + + def begin_method(self, model: object, method: StageAMethodSpec, sequence: object) -> object: + if model is not self._model: + raise StageAError("Stage-A engine received a different model") + identity = getattr(sequence, "identity_record_sha256", None) + target_ids = getattr(sequence, "target_token_ids", None) + if not isinstance(identity, str) or _SHA256_RE.fullmatch(identity) is None: + raise StageAError("Stage-A sequence has no authenticated identity hash") + if not isinstance(target_ids, tuple) or len(target_ids) < 2: + raise StageAError("Stage-A sequence has no valid continuation") + expected_steps = len(target_ids) - 1 + if method.method_id == FP32_METHOD: + if identity in self._reference_states: + raise StageAError("FP32 trajectory identity was reused") + self._reference_states[identity] = [] + else: + reference = self._reference_states.get(identity) + if reference is None or len(reference) != expected_steps: + raise StageAError("candidate method has no complete matched FP32 trajectory") + device = next(model.parameters()).device + self._torch.cuda.synchronize(device) + self._torch.cuda.empty_cache() + cache, observer = self._cache(model, method) + if observer is not None: + enter = getattr(observer, "__enter__", None) + if not callable(enter) or enter() is not observer: + raise StageAError("Stage-A equal-byte observer did not install exactly once") + return { + "model": model, + "method": method, + "cache": cache, + "observer": observer, + "identity_record_sha256": identity, + "expected_steps": expected_steps, + "step_index": 0, + } + + def _recurrent_states(self, cache: object, *, packed: bool) -> dict[int, object]: + torch = self._torch + static = importlib.import_module("recurquant.static_q468") + expected_layers = tuple(static.FROZEN_RECURRENT_LAYER_INDICES) + geometry = static.FROZEN_QWEN35_STATIC_Q468_GEOMETRY + expected_shape = (1, geometry.heads, geometry.key_rows, geometry.value_width) + states: dict[int, object] = {} + if packed: + if self._model is None: + raise StageAError("packed recurrent-state capture has no loaded model") + expected_device = next(self._model.parameters()).device + checkpoint = getattr(cache, "checkpoint", None) + materialize_all = getattr(checkpoint, "materialize", None) + if callable(materialize_all): + raw_states = materialize_all() + if not isinstance(raw_states, Mapping): + raise StageAError("packed cache materialization returned an invalid layer map") + source = raw_states.items() + else: + materialize_one = getattr(cache, "materialize_recurrent_state", None) + if not callable(materialize_one): + raise StageAError("packed cache cannot materialize its recurrent state") + source = ( + (layer_index, materialize_one(layer_index)) for layer_index in expected_layers + ) + for raw_layer_index, tensor in source: + layer_index = int(raw_layer_index) + if layer_index not in expected_layers or layer_index in states: + raise StageAError("packed cache recurrent layer inventory drifted") + if not isinstance(tensor, torch.Tensor) or not tensor.is_floating_point(): + raise StageAError("packed cache materialized a non-floating recurrent state") + if tensor.dtype is not torch.float32: + raise StageAError("packed cache materialized a non-FP32 recurrent state") + if tensor.device != expected_device: + raise StageAError( + "packed cache recurrent state is not on the model CUDA device" + ) + if tuple(tensor.shape) != expected_shape: + raise StageAError("packed cache recurrent-state geometry drifted") + states[layer_index] = tensor.detach().to(device="cpu", dtype=torch.float32).clone() + else: + cache_module = importlib.import_module("recurquant.cache") + if self._model is None: + raise StageAError("FP32 recurrent-state capture has no loaded model") + expected_device = next(self._model.parameters()).device + fp32_resident_bytes = 0 + for state in cache_module.iter_recurrent_states(cache): + if state.layer_index not in expected_layers or state.state_index != 0: + raise StageAError("FP32 cache recurrent layer/state inventory drifted") + if state.layer_index in states: + raise StageAError("FP32 cache exposed a duplicate recurrent layer") + tensor = state.tensor + if tensor.dtype is not torch.float32: + raise StageAError("FP32 reference cache exposed a non-FP32 recurrent state") + if tensor.device != expected_device: + raise StageAError("FP32 reference cache state is not on the model CUDA device") + if tuple(tensor.shape) != expected_shape: + raise StageAError("FP32 reference cache recurrent-state geometry drifted") + fp32_resident_bytes += int(tensor.numel()) * int(tensor.element_size()) + states[state.layer_index] = ( + tensor.detach().to(device="cpu", dtype=torch.float32).clone() + ) + if fp32_resident_bytes != EXPECTED_RECURRENT_RESIDENT_BYTES[FP32_METHOD]: + raise StageAError("FP32 reference cache byte ledger drifted") + if tuple(sorted(states)) != tuple(sorted(expected_layers)): + raise StageAError("recurrent trajectory omitted or added a frozen layer") + return states + + @staticmethod + def _cache_length(cache: object) -> int: + get_seq_length = getattr(cache, "get_seq_length", None) + if not callable(get_seq_length): + raise StageAError("Stage-A cache does not expose get_seq_length") + length = get_seq_length() + if isinstance(length, bool) or not isinstance(length, int) or length < 0: + raise StageAError("Stage-A cache returned an invalid sequence length") + return length + + def _trajectory_nmse(self, session: dict[str, object]) -> float: + torch = self._torch + method = cast(StageAMethodSpec, session["method"]) + identity = cast(str, session["identity_record_sha256"]) + step_index = cast(int, session["step_index"]) + expected_steps = cast(int, session["expected_steps"]) + if step_index >= expected_steps: + raise StageAError("Stage-A trajectory received an extra scored transition") + states = self._recurrent_states( + session["cache"], + packed=method.method_id != FP32_METHOD, + ) + if method.method_id == FP32_METHOD: + self._reference_states[identity].append(states) + return 0.0 + reference_trace = self._reference_states.get(identity) + if reference_trace is None or len(reference_trace) != expected_steps: + raise StageAError("matched FP32 trajectory changed before candidate comparison") + reference = reference_trace[step_index] + if set(reference) != set(states): + raise StageAError("candidate trajectory layer inventory differs from FP32") + layer_values: list[float] = [] + for layer_index in sorted(reference): + reference_tensor = reference[layer_index] + candidate_tensor = states[layer_index] + if not isinstance(reference_tensor, torch.Tensor) or not isinstance( + candidate_tensor, torch.Tensor + ): + raise StageAError("trajectory state is not a tensor") + if reference_tensor.shape != candidate_tensor.shape: + raise StageAError("candidate trajectory shape differs from matched FP32") + reference_fp64 = reference_tensor.to(dtype=torch.float64) + candidate_fp64 = candidate_tensor.to(dtype=torch.float64) + if ( + not torch.isfinite(reference_fp64).all().item() + or not torch.isfinite(candidate_fp64).all().item() + ): + raise StageAError("trajectory state contains non-finite values") + numerator = (candidate_fp64 - reference_fp64).square().sum(dtype=torch.float64) + denominator = reference_fp64.square().sum(dtype=torch.float64) + 1.0e-12 + value = float((numerator / denominator).item()) + if not math.isfinite(value) or value < 0.0: + raise StageAError("trajectory NMSE is invalid") + layer_values.append(value) + if not layer_values: + raise StageAError("trajectory NMSE has no recurrent layers") + return math.fsum(layer_values) / len(layer_values) + + def _forward( + self, + session: dict[str, object], + token_ids: tuple[int, ...], + *, + target_token_id: int, + position: int, + scored: bool, + ) -> ForwardObservation: + torch = self._torch + model = session["model"] + cache = session["cache"] + if ( + isinstance(position, bool) + or not isinstance(position, int) + or position < 0 + or isinstance(target_token_id, bool) + or not isinstance(target_token_id, int) + or target_token_id < 0 + or not token_ids + or any( + isinstance(token, bool) or not isinstance(token, int) or token < 0 + for token in token_ids + ) + ): + raise StageAError("Stage-A forward position or input token IDs are invalid") + expected_before = position + 1 - len(token_ids) + if expected_before < 0: + raise StageAError("Stage-A forward token span starts before position zero") + if self._cache_length(cache) != expected_before: + raise StageAError("Stage-A cache length drifted before the causal forward") + device = next(model.parameters()).device + torch.cuda.synchronize(device) + torch.cuda.reset_peak_memory_stats(device) + started = time.perf_counter_ns() + input_ids = torch.tensor([list(token_ids)], dtype=torch.long, device=device) + cache_position = torch.arange( + expected_before, + position + 1, + dtype=torch.long, + device=device, + ) + position_ids = cache_position.unsqueeze(0) + with torch.inference_mode(): + output = model( + input_ids=input_ids, + position_ids=position_ids, + cache_position=cache_position, + past_key_values=cache, + use_cache=True, + logits_to_keep=1, + ) + if getattr(output, "past_key_values", None) is not cache: + raise StageAError("Qwen3.5 returned a different Stage-A cache object") + if self._cache_length(cache) != position + 1: + raise StageAError("Stage-A cache did not advance by the exact causal token count") + torch.cuda.synchronize(device) + latency = time.perf_counter_ns() - started + logits = output.logits[0, -1].detach().to(device="cpu", dtype=torch.float32).contiguous() + log_probabilities_tensor = torch.log_softmax(logits, dim=-1) + if target_token_id >= log_probabilities_tensor.numel(): + raise StageAError("target token is outside the authenticated model vocabulary") + target_nll = -float(log_probabilities_tensor[target_token_id].item()) + top1 = int(torch.argmax(log_probabilities_tensor).item()) + evidence = getattr(cache, "last_evidence", None) + static = importlib.import_module("recurquant.static_q468") + local_sse = ( + 0.0 + if evidence is None + else float(getattr(evidence, "mean_squared_error", 0.0)) + * int(static.FROZEN_QWEN35_STATIC_Q468_GEOMETRY.state_elements) + ) + trajectory_nmse = self._trajectory_nmse(session) if scored else 0.0 + if scored: + session["step_index"] = cast(int, session["step_index"]) + 1 + storage_method = getattr(cache, "storage_summary", None) + storage = {} if not callable(storage_method) else dict(storage_method()) + method = cast(StageAMethodSpec, session["method"]) + if method.method_id == FP32_METHOD: + resident = EXPECTED_RECURRENT_RESIDENT_BYTES[FP32_METHOD] + else: + resident = int(storage.get("resident_bytes", 0)) + transient = int(storage.get("raw_state_workspace_peak_bytes", 0)) + int( + storage.get("query_workspace_peak_bytes", 0) + ) + return ForwardObservation( + position=position, + target_token_id=target_token_id, + comparison_logits=logits, + target_nll=target_nll, + top1_token_id=top1, + local_codec_sse=local_sse, + trajectory_nmse=trajectory_nmse, + latency_ns=latency, + peak_allocated_bytes=int(torch.cuda.max_memory_allocated(device)), + peak_reserved_bytes=int(torch.cuda.max_memory_reserved(device)), + resident_bytes=resident, + transient_bytes=transient, + ) + + def prefill( + self, + session: object, + *, + prompt_token_ids: tuple[int, ...], + first_target_token_id: int, + position: int, + ) -> ForwardObservation: + return self._forward( + cast(dict[str, object], session), + prompt_token_ids, + target_token_id=first_target_token_id, + position=position, + scored=False, + ) + + def step( + self, + session: object, + *, + input_token_id: int, + target_token_id: int, + position: int, + ) -> ForwardObservation: + return self._forward( + cast(dict[str, object], session), + (input_token_id,), + target_token_id=target_token_id, + position=position, + scored=True, + ) + + def end_method(self, session: object) -> Mapping[str, object]: + values = cast(dict[str, object], session) + observer = values.get("observer") + try: + step_index = cast(int, values["step_index"]) + expected_steps = cast(int, values["expected_steps"]) + if step_index != expected_steps: + raise StageAError("Stage-A method did not complete its exact scored trajectory") + method = cast(StageAMethodSpec, values["method"]) + identity = cast(str, values["identity_record_sha256"]) + if method.method_id == PRIMARY_K29334_METHOD: + reference = self._reference_states.pop(identity, None) + if reference is None or len(reference) != expected_steps: + raise StageAError("completed method grid has no exact FP32 trajectory") + cache = values["cache"] + summary = getattr(cache, "storage_summary", None) + storage = {} if not callable(summary) else dict(summary()) + if method.method_id == FP32_METHOD: + storage.update( + { + "resident_bytes": EXPECTED_RECURRENT_RESIDENT_BYTES[FP32_METHOD], + "resident_byte_scope": "logical_fp32_recurrent_state_only", + } + ) + return storage + finally: + try: + if observer is not None: + remove = getattr(observer, "remove", None) + if not callable(remove): + raise StageAError("Stage-A equal-byte observer cannot be removed") + remove() + finally: + # The caller keeps the session variable alive until the next method + # assignment. Clear ownership here so the completed cache/model can + # be released before the next method resets allocator diagnostics. + values.clear() + + +def run_preseal_engine_smoke(authenticated: AuthenticatedStageA) -> Mapping[str, object]: + """Exercise all nine real cache paths on fixed public tokens before the one-run seal.""" + + if tuple(method.method_id for method in authenticated.methods) != METHOD_ORDER: + raise StageAError("pre-seal smoke method order differs from the frozen grid") + engine = TorchStageAEngine() + sequence = _PresealSmokeSequence( + identity_record_sha256=sha256_bytes( + b"recurquant.experiment013.stage-a-preseal-engine-smoke.v1" + ), + target_token_ids=PRESEAL_ENGINE_SMOKE_TARGET, + ) + prompt_token_ids = (PRESEAL_ENGINE_SMOKE_PROMPT_TOKEN_ID,) * ( + PRESEAL_ENGINE_SMOKE_PROMPT_TOKEN_COUNT + ) + model = engine.load_model(authenticated.authenticated_model_files) + reference_logits: list[object] = [] + method_receipts: list[dict[str, object]] = [] + total_forwards = 0 + try: + for method in authenticated.methods: + session = engine.begin_method(model, method, sequence) + try: + prefill = _finite_observation( + engine.prefill( + session, + prompt_token_ids=prompt_token_ids, + first_target_token_id=PRESEAL_ENGINE_SMOKE_TARGET[0], + position=PRESEAL_ENGINE_SMOKE_PROMPT_TOKEN_COUNT - 1, + ), + expected_position=PRESEAL_ENGINE_SMOKE_PROMPT_TOKEN_COUNT - 1, + expected_target=PRESEAL_ENGINE_SMOKE_TARGET[0], + ) + del prefill + total_forwards += 1 + current_logits: list[object] = [] + for transition_index in range(len(PRESEAL_ENGINE_SMOKE_TARGET) - 1): + position = PRESEAL_ENGINE_SMOKE_PROMPT_TOKEN_COUNT + transition_index + observation = _finite_observation( + engine.step( + session, + input_token_id=PRESEAL_ENGINE_SMOKE_TARGET[transition_index], + target_token_id=PRESEAL_ENGINE_SMOKE_TARGET[transition_index + 1], + position=position, + ), + expected_position=position, + expected_target=PRESEAL_ENGINE_SMOKE_TARGET[transition_index + 1], + ) + total_forwards += 1 + current_logits.append(observation.comparison_logits) + if method.method_id == FP32_METHOD: + reference_logits.append(observation.comparison_logits) + else: + if transition_index >= len(reference_logits): + raise StageAError("pre-seal cache path ran before the FP32 reference") + _kl(reference_logits[transition_index], observation.comparison_logits) + except BaseException as error: + try: + engine.end_method(session) + except BaseException as cleanup_error: + error.add_note( + f"pre-seal cache observer cleanup also failed: {cleanup_error!r}" + ) + raise + else: + summary = dict(engine.end_method(session)) + resident = summary.get("resident_bytes") + expected_resident = EXPECTED_RECURRENT_RESIDENT_BYTES[method.method_id] + if ( + isinstance(resident, bool) + or not isinstance(resident, int) + or (resident != expected_resident) + ): + raise StageAError( + f"pre-seal {method.method_id} resident bytes differ from the frozen ledger" + ) + method_receipts.append( + { + "method_id": method.method_id, + "forward_count": len(PRESEAL_ENGINE_SMOKE_TARGET), + "logical_recurrent_resident_bytes": resident, + "equal_byte_observer_required": method.method_id != FP32_METHOD, + } + ) + current_logits.clear() + device = dict(engine.runtime_snapshot(model)) + finally: + reference_logits.clear() + engine.close_model(model) + del model + engine._torch.cuda.synchronize() + engine._torch.cuda.empty_cache() + expected_forwards = len(METHOD_ORDER) * len(PRESEAL_ENGINE_SMOKE_TARGET) + if total_forwards != expected_forwards: + raise StageAError("pre-seal engine smoke forward count drifted") + report = { + "profile": PRESEAL_ENGINE_SMOKE_PROFILE, + "passed": True, + "stage_a_content_accessed": False, + "input_profile": "fixed_public_synthetic_4096_plus_128_v3", + "prompt_token_count": PRESEAL_ENGINE_SMOKE_PROMPT_TOKEN_COUNT, + "continuation_token_count": PRESEAL_ENGINE_SMOKE_TARGET_TOKEN_COUNT, + "model_load_count": 1, + "method_order": list(METHOD_ORDER), + "forward_count": total_forwards, + "method_receipts": method_receipts, + "device": device, + } + return MappingProxyType(_strict_json(canonical_json_bytes(report), context="pre-seal smoke")) + + +def default_services() -> StageAServices: + return StageAServices( + authenticate=authenticate_production, + reauthenticate=reauthenticate_production, + preseal_smoke=run_preseal_engine_smoke, + reserve=reserve_one_run, + materialize=materialize_production, + engine=TorchStageAEngine(), + persist_receipt=persist_receipt, + publish=publish_result, + record_failure=record_failure, + ) + + +def _authenticate_recovery_boundary( + config: StageAConfig, + receipt: Mapping[str, object], + *, + allow_pre_cas_head: bool = False, +) -> tuple[BootstrapIdentity, str, str]: + identity_bytes = _stable_file_bytes( + config.frozen_identity_path, + context="recovery frozen Stage-A identity", + ) + bootstrap = bootstrap_stage_a_identity(identity_bytes) + if bootstrap.file_sha256 != _require_sha256( + receipt.get("identity_file_sha256"), + context="recovery receipt identity file SHA-256", + ): + raise StageAError("recovery identity differs from the durable receipt") + if ( + receipt.get("runner_revision") != RUNNER_REVISION + or receipt.get("attempt_number") != 1 + or receipt.get("one_run_marker") != ONE_RUN_MARKER + or receipt.get("automatic_retry_authorized") is not False + or receipt.get("claim_boundary") != CLAIM_BOUNDARY + or receipt.get("execution_bindings") != dict(bootstrap.execution_bindings) + or receipt.get("expected_forward_count") != bootstrap.expected_forward_count + ): + raise StageAError("recovery receipt contract differs from the frozen identity") + h1 = _require_sha1(config.identity_commit, context="recovery identity commit") + if h1 != _require_sha1( + receipt.get("h1_identity_commit"), + context="recovery receipt H1 commit", + ): + raise StageAError("recovery H1 differs from the durable receipt") + seal = _require_sha1( + receipt.get("one_run_seal_commit"), + context="recovery receipt seal commit", + ) + head = _require_sha1( + _git(config.git_executable_path, config.repository_root, "rev-parse", "HEAD"), + context="HEAD", + ) + if head != seal and not (allow_pre_cas_head and head == h1): + raise StageAError("recovery requires HEAD to remain at the authenticated one-run seal") + parents = _git( + config.git_executable_path, + config.repository_root, + "show", + "-s", + "--format=%P", + seal, + ).split() + if parents != [h1]: + raise StageAError("recovery seal is not the exact child of H1") + seal_tree = _require_sha1( + _git( + config.git_executable_path, + config.repository_root, + "show", + "-s", + "--format=%T", + seal, + ), + context="recovery seal tree", + ) + h1_tree = _require_sha1( + _git( + config.git_executable_path, + config.repository_root, + "show", + "-s", + "--format=%T", + h1, + ), + context="recovery H1 tree", + ) + if seal_tree != h1_tree or seal_tree != _require_sha1( + receipt.get("one_run_seal_tree"), + context="recovery receipt seal tree", + ): + raise StageAError("recovery seal is not the recorded empty-diff commit") + + binding_bytes = _stable_file_bytes( + config.calibration_binding_path, + context="recovery Stage-A calibration binding", + ) + binding_hash = sha256_bytes(binding_bytes) + if binding_hash != _require_sha256( + receipt.get("calibration_binding_file_sha256"), + context="recovery receipt calibration binding SHA-256", + ): + raise StageAError("recovery calibration binding differs from the durable receipt") + execution = _read_execution_artifacts(config) + observed_execution = {name: sha256_bytes(data) for name, data in execution.items()} + if observed_execution != dict(bootstrap.execution_bindings): + raise StageAError("recovery execution artifacts differ from the frozen identity") + source_hash = sha256_bytes(execution["repository_source_manifest_file_sha256"]) + if source_hash != _require_sha256( + receipt.get("source_manifest_file_sha256"), + context="recovery receipt source manifest SHA-256", + ): + raise StageAError("recovery source manifest differs from the durable receipt") + input_bundle_hash = _require_sha256( + receipt.get("stage_a_input_bundle_manifest_file_sha256"), + context="recovery receipt Stage-A input bundle manifest SHA-256", + ) + source = _bootstrap_source_manifest(execution["repository_source_manifest_file_sha256"]) + source_commit = _require_sha1(source.get("source_commit"), context="recovery source commit") + if source_commit != _require_sha1( + config.source_commit, context="recovery CLI H0 commit" + ) or source_commit != _require_sha1( + receipt.get("h0_source_commit"), context="recovery receipt H0 commit" + ): + raise StageAError("recovery H0 source identity drifted") + _verify_source_bytes(source, config.repository_root) + + identity = config.frozen_identity_path.resolve(strict=True) + root = config.repository_root.resolve(strict=True) + try: + relative = identity.relative_to(root).as_posix() + except ValueError as error: + raise StageAError("recovery identity is outside the repository") from error + relative = _safe_relative_path(relative, context="recovery identity repository path") + committed = _git_process( + config.git_executable_path, + root, + "show", + f"{seal}:{relative}", + ) + indexed = _git_process(config.git_executable_path, root, "show", f":{relative}") + if ( + committed.returncode != 0 + or indexed.returncode != 0 + or committed.stdout != identity_bytes + or indexed.stdout != identity_bytes + ): + raise StageAError("recovery identity differs across seal, index, and worktree") + + expected_message = _seal_message_values( + identity_file_sha256=bootstrap.file_sha256, + calibration_binding_file_sha256=binding_hash, + source_manifest_file_sha256=source_hash, + input_bundle_manifest_file_sha256=input_bundle_hash, + expected_forward_count=bootstrap.expected_forward_count, + ).encode("utf-8") + commit_object = _git_process( + config.git_executable_path, + root, + "cat-file", + "commit", + seal, + ) + if commit_object.returncode != 0: + raise StageAError("recovery cannot read the seal commit object") + _headers, separator, message = commit_object.stdout.partition(b"\n\n") + if ( + not separator + or message != expected_message + or sha256_bytes(expected_message) != (receipt.get("one_run_seal_message_sha256")) + ): + raise StageAError("recovery one-run seal message drifted") + + lock_path = _identity_attempt_lock_path( + root, + bootstrap.file_sha256, + git_executable_path=config.git_executable_path, + ) + lock_bytes = _stable_file_bytes(lock_path, context="recovery identity-scoped attempt lock") + if sha256_bytes(lock_bytes) != _require_sha256( + receipt.get("identity_scoped_attempt_lock_file_sha256"), + context="recovery receipt attempt-lock SHA-256", + ): + raise StageAError("recovery identity-scoped attempt lock drifted") + lock = _strict_json(lock_bytes, context="recovery identity-scoped attempt lock") + _exact_fields( + lock, + IDENTITY_ATTEMPT_LOCK_FIELDS, + context="recovery identity-scoped attempt lock", + ) + expected_lock = { + "schema": IDENTITY_ATTEMPT_LOCK_SCHEMA, + "runner_revision": RUNNER_REVISION, + "created_at_utc": receipt.get("created_at_utc"), + "attempt_number": 1, + "automatic_retry_authorized": False, + "h0_source_commit": source_commit, + "h1_identity_commit": h1, + "identity_repository_path": relative, + "identity_file_sha256": bootstrap.file_sha256, + "one_run_seal_commit": seal, + "one_run_seal_tree": seal_tree, + "one_run_marker": ONE_RUN_MARKER, + "one_run_seal_message_sha256": sha256_bytes(expected_message), + "calibration_binding_file_sha256": binding_hash, + "source_manifest_file_sha256": source_hash, + "stage_a_input_bundle_manifest_file_sha256": input_bundle_hash, + "execution_bindings": dict(bootstrap.execution_bindings), + "method_specs": receipt.get("method_specs"), + "expected_forward_count": bootstrap.expected_forward_count, + "claim_boundary": CLAIM_BOUNDARY, + "output_path": str(Path(os.path.abspath(config.output_path))), + "attempt_path": str(Path(os.path.abspath(config.attempt_path))), + "complete_path": str(Path(os.path.abspath(config.complete_path))), + } + if lock != expected_lock: + raise StageAError("recovery identity-scoped attempt lock semantics drifted") + return bootstrap, binding_hash, seal + + +def _recover_lock_only_attempt(config: StageAConfig) -> None: + """Materialize an administrative no-retry receipt after lock-before-receipt failure.""" + + identity_bytes = _stable_file_bytes( + config.frozen_identity_path, + context="lock-only recovery frozen Stage-A identity", + ) + bootstrap = bootstrap_stage_a_identity(identity_bytes) + lock_path = _identity_attempt_lock_path( + config.repository_root, + bootstrap.file_sha256, + git_executable_path=config.git_executable_path, + ) + if not lock_path.is_file(): + raise StageAError("there is no Stage-A attempt receipt or identity-scoped lock to recover") + lock_bytes = _stable_file_bytes( + lock_path, + context="lock-only recovery identity-scoped attempt lock", + ) + lock = _strict_json(lock_bytes, context="lock-only recovery attempt lock") + if canonical_json_bytes(lock) != lock_bytes: + raise StageAError("lock-only recovery attempt lock is not canonical JSON") + _exact_fields( + lock, + IDENTITY_ATTEMPT_LOCK_FIELDS, + context="lock-only recovery attempt lock", + ) + if ( + lock.get("schema") != IDENTITY_ATTEMPT_LOCK_SCHEMA + or lock.get("identity_file_sha256") != bootstrap.file_sha256 + or lock.get("output_path") != str(Path(os.path.abspath(config.output_path))) + or lock.get("attempt_path") != str(Path(os.path.abspath(config.attempt_path))) + or lock.get("complete_path") != str(Path(os.path.abspath(config.complete_path))) + or lock.get("automatic_retry_authorized") is not False + ): + raise StageAError("lock-only recovery attempt lock identity drifted") + if config.output_path.exists() or config.complete_path.exists(): + raise StageAError("lock-only recovery found output without an attempt receipt") + receipt: dict[str, object] = { + "schema": ATTEMPT_SCHEMA, + "status": "prepared_before_head_cas", + "runner_revision": lock.get("runner_revision"), + "created_at_utc": lock.get("created_at_utc"), + "attempt_number": lock.get("attempt_number"), + "h0_source_commit": lock.get("h0_source_commit"), + "h1_identity_commit": lock.get("h1_identity_commit"), + "identity_repository_path": lock.get("identity_repository_path"), + "one_run_seal_commit": lock.get("one_run_seal_commit"), + "one_run_seal_tree": lock.get("one_run_seal_tree"), + "one_run_marker": lock.get("one_run_marker"), + "one_run_seal_message_sha256": lock.get("one_run_seal_message_sha256"), + "identity_file_sha256": lock.get("identity_file_sha256"), + "identity_scoped_attempt_lock_file_sha256": sha256_bytes(lock_bytes), + "calibration_binding_file_sha256": lock.get("calibration_binding_file_sha256"), + "source_manifest_file_sha256": lock.get("source_manifest_file_sha256"), + "stage_a_input_bundle_manifest_file_sha256": lock.get( + "stage_a_input_bundle_manifest_file_sha256" + ), + "execution_bindings": lock.get("execution_bindings"), + "method_specs": lock.get("method_specs"), + "expected_forward_count": lock.get("expected_forward_count"), + "observed_forward_count": 0, + "content_materialized": False, + "model_load_count": 0, + "evaluation_complete": False, + "result_available": False, + "automatic_retry_authorized": False, + "claim_boundary": lock.get("claim_boundary"), + "lock_only_recovered_at_utc": datetime.now(UTC).isoformat(), + } + _authenticate_recovery_boundary( + config, + receipt, + allow_pre_cas_head=True, + ) + payload = canonical_json_bytes(receipt) + _exclusive_write(config.attempt_path, payload) + if ( + _stable_file_bytes( + config.attempt_path, + context="lock-only recovered Stage-A attempt receipt", + ) + != payload + ): + raise StageAError("lock-only recovered Stage-A receipt changed after publication") + + +def recover_interrupted(config: StageAConfig) -> Mapping[str, object]: + """Reconcile durable receipts without re-entering content or model evaluation.""" + + output_dir = _safe_directory(config.output_dir, create=False) + if any( + Path(os.path.abspath(path)).parent != output_dir + for path in (config.output_path, config.attempt_path, config.complete_path) + ): + raise StageAError("Stage-A recovery paths escaped the authenticated output directory") + if not config.attempt_path.is_file(): + _recover_lock_only_attempt(config) + receipt = _strict_json( + _stable_file_bytes(config.attempt_path, context="Stage-A attempt receipt"), + context="Stage-A attempt receipt", + ) + if ( + receipt.get("schema") != ATTEMPT_SCHEMA + or receipt.get("automatic_retry_authorized") is not False + ): + raise StageAError("Stage-A attempt receipt identity drifted") + status = receipt.get("status") + seal = _require_sha1(receipt.get("one_run_seal_commit"), context="receipt seal commit") + h1 = _require_sha1(receipt.get("h1_identity_commit"), context="receipt H1 commit") + head = _require_sha1( + _git(config.git_executable_path, config.repository_root, "rev-parse", "HEAD"), + context="HEAD", + ) + pre_cas_statuses = { + "prepared_before_head_cas", + "pre_cas_attempt_receipt_present_no_automatic_retry", + } + pre_cas = status in pre_cas_statuses and head == h1 and not config.output_path.exists() + bootstrap, binding_hash, authenticated_seal = _authenticate_recovery_boundary( + config, + receipt, + allow_pre_cas_head=pre_cas, + ) + if seal != authenticated_seal: + raise StageAError("recovery seal authentication returned a different commit") + if config.output_path.is_file(): + payload = _stable_file_bytes(config.output_path, context="published Stage-A result") + result_file_sha256 = _require_sha256( + receipt.get("result_file_sha256"), + context="receipt result file SHA-256", + ) + if sha256_bytes(payload) != result_file_sha256: + raise StageAError("published Stage-A result differs from its receipt") + result = _strict_json(payload, context="published Stage-A result") + if canonical_json_bytes(result) != payload: + raise StageAError("published Stage-A result is not canonical JSON") + verify_execution_artifact( + payload, + expected_identity_file_sha256=bootstrap.file_sha256, + expected_calibration_binding_file_sha256=binding_hash, + expected_h1_commit=h1, + expected_seal_commit=authenticated_seal, + expected_source_commit=_require_sha1( + receipt.get("h0_source_commit"), + context="recovery receipt H0 source commit", + ), + expected_source_manifest_file_sha256=_require_sha256( + receipt.get("source_manifest_file_sha256"), + context="recovery receipt source manifest SHA-256", + ), + expected_input_bundle_manifest_file_sha256=_require_sha256( + receipt.get("stage_a_input_bundle_manifest_file_sha256"), + context="recovery receipt Stage-A input bundle manifest SHA-256", + ), + expected_execution_bindings=cast( + Mapping[str, str], + receipt.get("execution_bindings"), + ), + expected_method_specs=cast( + Sequence[Mapping[str, object]], + receipt.get("method_specs"), + ), + expected_materialization={ + "sequence_count": 12, + "capture_input_sha256": receipt.get("capture_input_sha256"), + "token_sequence_manifest_sha256": receipt.get("token_sequence_manifest_sha256"), + "tokenizer_manifest_sha256": receipt.get("tokenizer_manifest_sha256"), + }, + expected_seal_tree=_require_sha1( + receipt.get("one_run_seal_tree"), + context="recovery receipt seal tree", + ), + expected_seal_message_sha256=_require_sha256( + receipt.get("one_run_seal_message_sha256"), + context="recovery receipt seal message SHA-256", + ), + expected_attempt_lock_file_sha256=_require_sha256( + receipt.get("identity_scoped_attempt_lock_file_sha256"), + context="recovery receipt attempt-lock SHA-256", + ), + expected_authenticated_runtime=cast( + Mapping[str, object], + receipt.get("post_load_authenticated_runtime"), + ), + expected_device_runtime=cast( + Mapping[str, object], + receipt.get("post_load_device_runtime"), + ), + expected_forward_count=_nonnegative_int( + receipt.get("expected_forward_count"), + context="recovery receipt expected forward count", + ), + ) + evidence = result.get("evidence") + if not isinstance(evidence, Mapping): + raise StageAError("published Stage-A result evidence is missing") + result_smoke = evidence.get("preseal_engine_smoke") + receipt_smoke = receipt.get("preseal_engine_smoke") + if ( + not isinstance(result_smoke, Mapping) + or not isinstance(receipt_smoke, Mapping) + or dict(result_smoke) != dict(receipt_smoke) + or sha256_bytes(canonical_json_bytes(dict(result_smoke))) + != _require_sha256( + receipt.get("preseal_engine_smoke_sha256"), + context="recovery receipt pre-seal smoke SHA-256", + ) + ): + raise StageAError("published Stage-A pre-seal smoke differs from the receipt") + canonical_evidence_sha256 = _require_sha256( + result.get("canonical_evidence_sha256"), + context="published result canonical evidence SHA-256", + ) + if canonical_evidence_sha256 != receipt.get("result_canonical_evidence_sha256"): + raise StageAError("published Stage-A result canonical identity differs from receipt") + completed_statuses = { + "completed_with_authenticated_stage_a_result", + "completed_result_published_receipt_recovered", + } + if status not in completed_statuses: + updated = { + **receipt, + "status": "completed_result_published_receipt_recovered", + "result_available": True, + "recovered_at_utc": datetime.now(UTC).isoformat(), + } + _atomic_replace_owned(config.attempt_path, canonical_json_bytes(updated)) + status = updated["status"] + elif receipt.get("result_available") is not True: + raise StageAError("completed Stage-A receipt does not mark the result available") + marker = _completion_marker_bytes( + config, + result_file_sha256=result_file_sha256, + ) + if config.complete_path.exists(): + if ( + not config.complete_path.is_file() + or _stable_file_bytes( + config.complete_path, + context="Stage-A completion marker", + ) + != marker + ): + raise StageAError("Stage-A completion marker differs from recovered evidence") + else: + _atomic_publish_new(config.complete_path, marker) + if ( + _stable_file_bytes( + config.complete_path, + context="recovered Stage-A completion marker", + ) + != marker + ): + raise StageAError("recovered Stage-A completion marker changed after publication") + return { + "status": status, + "result_available": True, + "completion_marker_available": True, + "automatic_retry_authorized": False, + } + if config.complete_path.exists(): + raise StageAError("Stage-A completion marker exists without its published result") + completed_statuses = { + "completed_with_authenticated_stage_a_result", + "completed_result_published_receipt_recovered", + } + if status in completed_statuses or receipt.get("result_available") is True: + raise StageAError("completed Stage-A evidence is missing its published result") + stable_no_result_statuses = { + "consumed_attempt_interrupted_no_result", + "consumed_attempt_failed_no_automatic_retry", + "pre_cas_attempt_receipt_present_no_automatic_retry", + } + if head == seal: + state = ( + status + if status + in stable_no_result_statuses - {"pre_cas_attempt_receipt_present_no_automatic_retry"} + else "consumed_attempt_interrupted_no_result" + ) + elif head == h1 and status in pre_cas_statuses: + state = "pre_cas_attempt_receipt_present_no_automatic_retry" + else: + raise StageAError("receipt, HEAD, and one-run seal cannot be reconciled") + if state == status: + return { + "status": state, + "result_available": False, + "automatic_retry_authorized": False, + } + updated = { + **receipt, + "status": state, + "result_available": False, + "automatic_retry_authorized": False, + "recovered_at_utc": datetime.now(UTC).isoformat(), + } + _atomic_replace_owned(config.attempt_path, canonical_json_bytes(updated)) + return {"status": state, "result_available": False, "automatic_retry_authorized": False} + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "mode", + choices=("prepare-inputs", "preflight", "execute", "recover"), + ) + parser.add_argument("--frozen-identity", required=True, type=Path) + parser.add_argument("--stage-a-calibration-binding", required=True, type=Path) + parser.add_argument("--repository-source-manifest", required=True, type=Path) + parser.add_argument("--runtime-manifest", required=True, type=Path) + parser.add_argument("--expected-runtime-manifest-sha256", required=True) + parser.add_argument("--model-file-manifest", required=True, type=Path) + parser.add_argument("--expected-model-file-manifest-sha256", required=True) + parser.add_argument("--parquet-materialization-manifest", required=True, type=Path) + parser.add_argument("--expected-parquet-materialization-manifest-sha256", required=True) + parser.add_argument("--model-root", required=True, type=Path) + parser.add_argument("--cache-root", required=True, type=Path) + parser.add_argument("--ruler-root", required=True, type=Path) + parser.add_argument("--input-bundle-root", required=True, type=Path) + parser.add_argument("--repository-root", required=True, type=Path) + parser.add_argument("--source-commit", required=True) + parser.add_argument("--identity-commit", required=True) + parser.add_argument("--output-dir", required=True, type=Path) + return parser + + +def _config_from_args( + args: argparse.Namespace, + *, + base_runtime_root: Path | None, + package_roots: Mapping[str, Path], + package_import_paths: Mapping[str, str], + interpreter_path: Path | None, + git_executable_path: Path | None, + pycache_prefix: Path | None, +) -> StageAConfig: + return StageAConfig( + frozen_identity_path=args.frozen_identity, + calibration_binding_path=args.stage_a_calibration_binding, + repository_source_manifest_path=args.repository_source_manifest, + runtime_manifest_path=args.runtime_manifest, + model_file_manifest_path=args.model_file_manifest, + parquet_materialization_manifest_path=args.parquet_materialization_manifest, + model_root=args.model_root, + cache_root=args.cache_root, + ruler_root=args.ruler_root, + input_bundle_root=args.input_bundle_root, + repository_root=args.repository_root, + source_commit=args.source_commit, + identity_commit=args.identity_commit, + output_dir=args.output_dir, + expected_runtime_manifest_sha256=args.expected_runtime_manifest_sha256, + expected_model_file_manifest_sha256=args.expected_model_file_manifest_sha256, + expected_parquet_materialization_manifest_sha256=( + args.expected_parquet_materialization_manifest_sha256 + ), + base_runtime_root=base_runtime_root, + package_roots=MappingProxyType(dict(package_roots)), + package_import_paths=MappingProxyType(dict(package_import_paths)), + interpreter_path=interpreter_path, + git_executable_path=git_executable_path, + pycache_prefix=pycache_prefix, + ) + + +def sealed_main( + argv: Sequence[str], + *, + base_runtime_root: Path, + package_roots: Mapping[str, Path], + package_import_paths: Mapping[str, str], + interpreter_path: Path, + git_executable_path: Path, + pycache_prefix: Path, +) -> int: + args = _parser().parse_args(list(argv)) + config = _config_from_args( + args, + base_runtime_root=base_runtime_root, + package_roots=package_roots, + package_import_paths=package_import_paths, + interpreter_path=interpreter_path, + git_executable_path=git_executable_path, + pycache_prefix=pycache_prefix, + ) + if args.mode == "prepare-inputs": + report = prepare_inputs_production(config) + print(json.dumps(dict(report), indent=2, sort_keys=True)) + return 0 + if args.mode == "recover": + report = recover_interrupted(config) + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + services = default_services() + if args.mode == "preflight": + authenticated = services.authenticate(config) + print( + json.dumps( + { + "status": "stage_a_preflight_pass", + "identity_file_sha256": authenticated.bootstrap_identity.file_sha256, + "calibration_binding_file_sha256": authenticated.binding.file_sha256, + "source_commit": authenticated.source_commit, + "stage_a_input_bundle_manifest_file_sha256": ( + authenticated.input_bundle_manifest_file_sha256 + ), + "expected_forward_count": ( + authenticated.bootstrap_identity.expected_forward_count + ), + "content_materialized": False, + "model_weight_files_authenticated": True, + "model_instantiated": False, + "one_run_reserved": False, + "method_order": list(METHOD_ORDER), + "claim_boundary": CLAIM_BOUNDARY, + }, + indent=2, + sort_keys=True, + ) + ) + return 0 + report = run_stage_a(config, services) + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 if report["stage_a_passed"] is True else 2 + + +def main(argv: Sequence[str] | None = None) -> int: + del argv + raise StageAError( + "Stage A must run through launch_static_q468_stage_a.py in the authenticated sealed runtime" + ) + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main(sys.argv[1:])) diff --git a/scripts/verify_statelease_stage0.py b/scripts/verify_statelease_stage0.py index fdbe380..7cbc9f0 100644 --- a/scripts/verify_statelease_stage0.py +++ b/scripts/verify_statelease_stage0.py @@ -161,6 +161,7 @@ "src/recurquant/experiment013_parquet.py", "src/recurquant/experiment013_qwen35_adapter.py", "src/recurquant/experiment013_source.py", + "src/recurquant/experiment013_stage_a.py", "src/recurquant/finite_difference.py", "src/recurquant/fisher_sensitivity.py", "src/recurquant/horizon.py", diff --git a/src/recurquant/__init__.py b/src/recurquant/__init__.py index 5776189..7bba59c 100644 --- a/src/recurquant/__init__.py +++ b/src/recurquant/__init__.py @@ -173,12 +173,20 @@ reference_aligned_trajectory_nmse, ) from .statelease_observer import Qwen35StateLeaseObserver +from .static_q468 import ( + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + STATIC_Q468_MSE_METHOD, + STATIC_Q468_UNIFORM_Q4_METHOD, + STATIC_Q468_UNIFORM_Q8_METHOD, +) from .static_q468_cache import ( + DYNAMIC_Q468_BASELINE_METHOD, DYNAMIC_Q468_ORACLE_METHOD, FROZEN_STATIC_RUNTIME_METHODS, StaticRhtCacheUpdateEvidence, StaticRhtQwen35Cache, StaticRhtRuntimeCheckpoint, + create_qwen35_dynamic_q468_baseline_cache, create_qwen35_dynamic_q468_oracle_cache, create_qwen35_static_rht_cache, ) @@ -229,6 +237,7 @@ "EqualByteLinearAttentionLayer", "EqualByteNoReplayCache", "EqualByteQwen35Cache", + "DYNAMIC_Q468_BASELINE_METHOD", "DYNAMIC_Q468_ORACLE_METHOD", "FROZEN_STATIC_RUNTIME_METHODS", "QueryEmaMixedPackedLinearAttentionLayer", @@ -251,6 +260,10 @@ "STATELEASE_REPLAY_CAPACITY", "STATELEASE_SELECTION_METHOD", "STATELEASE_METHOD", + "STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD", + "STATIC_Q468_MSE_METHOD", + "STATIC_Q468_UNIFORM_Q4_METHOD", + "STATIC_Q468_UNIFORM_Q8_METHOD", "StateLeaseBoundaryDecision", "StateLeaseLinearAttentionLayer", "StateLeaseRecurrentStateCache", @@ -282,6 +295,7 @@ "create_qwen35_cora_exact_budget_cache", "create_qwen35_exact_budget_cache", "create_qwen35_equal_byte_cache", + "create_qwen35_dynamic_q468_baseline_cache", "create_qwen35_dynamic_q468_oracle_cache", "create_qwen35_experiment010_fixed_replay_cache", "create_qwen35_experiment010_statelease_cache", diff --git a/src/recurquant/experiment013_calibration_api.py b/src/recurquant/experiment013_calibration_api.py index b18eb2d..cfe1234 100644 --- a/src/recurquant/experiment013_calibration_api.py +++ b/src/recurquant/experiment013_calibration_api.py @@ -162,6 +162,28 @@ class StepObservation: successful_kernel_calls_per_layer: tuple[int, ...] +@dataclass(frozen=True, slots=True) +class FisherStepObservation: + """One causal H=1 loss-gradient observation from a warm recurrent cache. + + ``boundary_position`` identifies the stored state after token ``x_b``. + The differentiable step consumes ``x_(b+1)`` at ``input_position`` and its + logits are scored against ``x_(b+2)`` at ``target_position``. Tensor + values remain typed as ``object`` so this authenticated API stays stdlib + only. + """ + + boundary_position: int + input_position: int + target_position: int + input_token_id: int + target_token_id: int + step_observation: StepObservation + source_recurrent_state: object + source_state_gradient: object + target_nll: float + + @dataclass(frozen=True, slots=True) class ModelFileIdentity: """One immutable file in the authenticated local model snapshot.""" @@ -206,6 +228,16 @@ def step_token( capture_state: bool, ) -> StepObservation: ... + def step_token_with_fisher( + self, + model: object, + *, + token_id: int, + position: int, + target_token_id: int, + capture_state: bool, + ) -> FisherStepObservation: ... + def end_sequence(self, model: object, record: Mapping[str, object]) -> None: ... def close_model(self, model: object) -> None: ... @@ -227,6 +259,7 @@ def __call__(self, context: AdapterConstructionContext, /) -> CalibrationAdapter "CalibrationAdapter", "CalibrationAdapterFactory", "EXECUTION_BINDING_ARTIFACT_KEYS", + "FisherStepObservation", "ModelFileIdentity", "RUNTIME_AUTHENTICATION_CONTEXT_KEYS", "StepObservation", diff --git a/src/recurquant/experiment013_parquet.py b/src/recurquant/experiment013_parquet.py index fe82955..999e615 100644 --- a/src/recurquant/experiment013_parquet.py +++ b/src/recurquant/experiment013_parquet.py @@ -19,17 +19,13 @@ from pathlib import Path, PurePosixPath from typing import Protocol -EXPERIMENT013_PARQUET_MANIFEST_SCHEMA = ( - "recurquant.experiment013.parquet-materializations.v1" -) +EXPERIMENT013_PARQUET_MANIFEST_SCHEMA = "recurquant.experiment013.parquet-materializations.v1" EXPERIMENT013_PARQUET_MANIFEST_SHA256 = ( "ee5628e50e5d3516fd79077542d355fd915455ac0e53128d372f4177ad63d39c" ) EXPERIMENT013_PARQUET_MANIFEST_SIZE_BYTES = 3918 EXPERIMENT013_PARQUET_MANIFEST_PATH = ( - Path(__file__).resolve().parents[2] - / "research" - / "experiment013-parquet-materializations.json" + Path(__file__).resolve().parents[2] / "research" / "experiment013-parquet-materializations.json" ) _TOP_LEVEL_FIELDS = frozenset({"schema", "datasets"}) @@ -244,9 +240,7 @@ class Experiment013ParquetProjection: immutable_path="default/test/0000.parquet", size_bytes=2_902_210, git_blob_oid="9877db06683d4245bc39aed18ee7cbad013ba5fa", - lfs_sha256=( - "4436f5c03d77c17e0cbc57543b90665b5c1266f55a43992a5ed7922cd34a7558" - ), + lfs_sha256=("4436f5c03d77c17e0cbc57543b90665b5c1266f55a43992a5ed7922cd34a7558"), lfs_size_bytes=2_902_210, ), ), @@ -265,9 +259,7 @@ class Experiment013ParquetProjection: immutable_path="default/partial-train/0000.parquet", size_bytes=603_127_902, git_blob_oid="00245b214ff9806a04f32debff0fd2e7b0737997", - lfs_sha256=( - "ea701af2e8a11bb8601150a47affff658452d687494dbed52a82d3b1fcf48811" - ), + lfs_sha256=("ea701af2e8a11bb8601150a47affff658452d687494dbed52a82d3b1fcf48811"), lfs_size_bytes=603_127_902, ), Experiment013ParquetFile( @@ -276,9 +268,7 @@ class Experiment013ParquetProjection: immutable_path="default/partial-train/0001.parquet", size_bytes=526_793_959, git_blob_oid="1169b6deb8c1cd46a46f0ae752b68806c9b5cca9", - lfs_sha256=( - "5c1c025f46b4a6b52b56167efeb89a2b9378f9ea8a50cdf5ddcbca8c4e17db1f" - ), + lfs_sha256=("5c1c025f46b4a6b52b56167efeb89a2b9378f9ea8a50cdf5ddcbca8c4e17db1f"), lfs_size_bytes=526_793_959, ), Experiment013ParquetFile( @@ -287,9 +277,7 @@ class Experiment013ParquetProjection: immutable_path="default/partial-train/0002.parquet", size_bytes=576_668_259, git_blob_oid="b9413777553240574488fa36b74f4bd286c06719", - lfs_sha256=( - "80cc198a2ef5239bf22a496eb10e6afd6fba075c4f6dd3d26dae7ed82c3bb1ad" - ), + lfs_sha256=("80cc198a2ef5239bf22a496eb10e6afd6fba075c4f6dd3d26dae7ed82c3bb1ad"), lfs_size_bytes=576_668_259, ), Experiment013ParquetFile( @@ -298,9 +286,7 @@ class Experiment013ParquetProjection: immutable_path="default/partial-train/0003.parquet", size_bytes=583_939_098, git_blob_oid="08f92b1ad15eb9f90268fa7cf9823523f1fb056a", - lfs_sha256=( - "326718129b7d13a9f45ae8e6e68ae90d95c15bf40fa457a053716832e4d07c1c" - ), + lfs_sha256=("326718129b7d13a9f45ae8e6e68ae90d95c15bf40fa457a053716832e4d07c1c"), lfs_size_bytes=583_939_098, ), Experiment013ParquetFile( @@ -309,9 +295,7 @@ class Experiment013ParquetProjection: immutable_path="default/partial-train/0004.parquet", size_bytes=588_756_614, git_blob_oid="9cb48d05cf6568879582eb5bd894a7d1b34aee7b", - lfs_sha256=( - "c4dff8b2cd993d1bb6bded41eb0eef56dff5449753ea13c79e730b7a9e1f6907" - ), + lfs_sha256=("c4dff8b2cd993d1bb6bded41eb0eef56dff5449753ea13c79e730b7a9e1f6907"), lfs_size_bytes=588_756_614, ), Experiment013ParquetFile( @@ -320,9 +304,7 @@ class Experiment013ParquetProjection: immutable_path="default/partial-train/0005.parquet", size_bytes=321_273_724, git_blob_oid="6a1b9b38d31ca025cbf06192a3dfd66067eb7571", - lfs_sha256=( - "9ab4d07d379720a9b18e7e3a060a948e2338b7aa338e9534a12d52fbc4fd8e2e" - ), + lfs_sha256=("9ab4d07d379720a9b18e7e3a060a948e2338b7aa338e9534a12d52fbc4fd8e2e"), lfs_size_bytes=321_273_724, ), Experiment013ParquetFile( @@ -331,9 +313,7 @@ class Experiment013ParquetProjection: immutable_path="default/partial-validation/0000.parquet", size_bytes=10_803_864, git_blob_oid="3e86263f595fae38387a938ec882417649c2bbd4", - lfs_sha256=( - "81680529564d4ead1c0e3859509a62d86c7126c32afc95dce6bd98e729e491ef" - ), + lfs_sha256=("81680529564d4ead1c0e3859509a62d86c7126c32afc95dce6bd98e729e491ef"), lfs_size_bytes=10_803_864, ), ), @@ -358,9 +338,7 @@ def _exact_fields(value: Mapping[str, object], expected: frozenset[str], *, name if actual != expected: missing = sorted(expected - actual) extra = sorted(actual - expected) - raise Experiment013ParquetError( - f"{name} fields drifted (missing={missing}, extra={extra})" - ) + raise Experiment013ParquetError(f"{name} fields drifted (missing={missing}, extra={extra})") def _expect_string(value: object, expected: str, *, name: str) -> str: @@ -449,9 +427,13 @@ def validate_experiment013_parquet_manifest( expected_dataset.conversion_revision, name=f"{name}.conversion_revision", ) - if type(raw_dataset["selected_splits"]) is not list or tuple( - raw_dataset["selected_splits"] # type: ignore[arg-type] - ) != expected_dataset.selected_splits: + if ( + type(raw_dataset["selected_splits"]) is not list + or tuple( + raw_dataset["selected_splits"] # type: ignore[arg-type] + ) + != expected_dataset.selected_splits + ): raise Experiment013ParquetError(f"{name}.selected_splits or order drifted") if ( type(raw_dataset["partial"]) is not bool @@ -604,17 +586,22 @@ def _object_field(value: object, name: str) -> object: class HuggingFaceHubMetadataBackend: """Metadata-only implementation using official Hub APIs and exact revisions.""" - def __init__(self, *, token: str | bool | None = None) -> None: - self._token = token + def __init__(self, *, token: bool = False) -> None: + if token is not False: + raise Experiment013ParquetError( + "Experiment 013 Hub metadata access must be unauthenticated" + ) + self._token = False def resolve_dataset_revision(self, *, repo_id: str, revision: str) -> str: try: from huggingface_hub import HfApi - info = HfApi(token=self._token).dataset_info( + info = HfApi(token=False, endpoint="https://huggingface.co").dataset_info( repo_id=repo_id, revision=revision, files_metadata=False, + token=False, ) resolved = info.sha except Exception as error: @@ -635,10 +622,11 @@ def snapshot_parquet_files( try: from huggingface_hub import HfApi, get_hf_file_metadata, hf_hub_url - info = HfApi(token=self._token).dataset_info( + info = HfApi(token=False, endpoint="https://huggingface.co").dataset_info( repo_id=repo_id, revision=revision, files_metadata=True, + token=False, ) siblings = { sibling.rfilename: sibling @@ -660,8 +648,9 @@ def snapshot_parquet_files( filename=path, repo_type="dataset", revision=revision, + endpoint="https://huggingface.co", ) - head = get_hf_file_metadata(url, token=self._token) + head = get_hf_file_metadata(url, token=False) files.append( HubFileMetadata( path=path, @@ -706,8 +695,7 @@ def inspect(self, uri: str) -> ParquetFileLayout: parquet_file = parquet.ParquetFile(stream) # type: ignore[attr-defined] metadata = parquet_file.metadata row_group_rows = tuple( - metadata.row_group(index).num_rows - for index in range(metadata.num_row_groups) + metadata.row_group(index).num_rows for index in range(metadata.num_row_groups) ) columns = tuple(parquet_file.schema_arrow.names) except Exception as error: @@ -768,10 +756,7 @@ def _immutable_uri( dataset: Experiment013ParquetDataset, file: Experiment013ParquetFile, ) -> str: - return ( - f"hf://datasets/{dataset.dataset_id}@{dataset.conversion_revision}/" - f"{file.immutable_path}" - ) + return f"hf://datasets/{dataset.dataset_id}@{dataset.conversion_revision}/{file.immutable_path}" def _selected_files( @@ -897,9 +882,7 @@ def _locate_offset( ) -> Experiment013ParquetRowLocation: if type(global_offset) is not int or global_offset < 0: raise Experiment013ParquetOffsetError("global row offset must be a non-negative integer") - split_row_count = sum( - sum(layout.row_group_rows) for _, _, _, layout in inspected - ) + split_row_count = sum(sum(layout.row_group_rows) for _, _, _, layout in inspected) if global_offset >= split_row_count: raise Experiment013ParquetOffsetError( f"global row offset {global_offset} is outside split row count {split_row_count}" @@ -966,9 +949,7 @@ def _projection_sha256( "columns": list(columns), "dataset_key": dataset_key, "logical_split": logical_split, - "rows": [ - {"global_offset": row.global_offset, "values": list(row.values)} for row in rows - ], + "rows": [{"global_offset": row.global_offset, "values": list(row.values)} for row in rows], } encoded = ( json.dumps(payload, sort_keys=True, separators=(",", ":"), allow_nan=False) + "\n" @@ -1006,9 +987,7 @@ def project_experiment013_parquet_columns( raise Experiment013ParquetError( "bulk Parquet projection is restricted to the frozen canonical-ID column" ) - if expected_count is not None and ( - type(expected_count) is not int or expected_count < 0 - ): + if expected_count is not None and (type(expected_count) is not int or expected_count < 0): raise Experiment013ParquetError("expected projection count must be a non-negative integer") hub = hub_backend if hub_backend is not None else HuggingFaceHubMetadataBackend() parquet = parquet_backend if parquet_backend is not None else PyArrowParquetBackend() diff --git a/src/recurquant/experiment013_qwen35_adapter.py b/src/recurquant/experiment013_qwen35_adapter.py index ee5dc5b..7ad7872 100644 --- a/src/recurquant/experiment013_qwen35_adapter.py +++ b/src/recurquant/experiment013_qwen35_adapter.py @@ -20,7 +20,7 @@ from contextvars import ContextVar, Token from dataclasses import dataclass from pathlib import Path -from types import ModuleType +from types import MethodType, ModuleType from typing import Any, Final import torch @@ -29,14 +29,15 @@ AdapterConstructionContext, AuthenticatedModelFiles, AuthenticatedSequence, + FisherStepObservation, StepObservation, ) -ADAPTER_REVISION: Final = "experiment-013-qwen35-live-adapter-v1" +ADAPTER_REVISION: Final = "experiment-013-qwen35-live-adapter-v2" ADAPTER_SOURCE_PATH: Final = "src/recurquant/experiment013_qwen35_adapter.py" CAPTURE_SOURCE_PATH: Final = "scripts/capture_static_q468_identity_input.py" CAPTURE_MODULE_NAME: Final = "_recurquant_experiment013_capture_for_live_adapter" -SOURCE_MANIFEST_SCHEMA: Final = "recurquant.experiment013.source-manifest.v1" +SOURCE_MANIFEST_SCHEMA: Final = "recurquant.experiment013.source-manifest.v2" MODEL_ID: Final = "Qwen/Qwen3.5-0.8B-Base" MODEL_REVISION: Final = "dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68" @@ -601,6 +602,114 @@ class _MissingType: _Missing = _MissingType() +_CACHE_LAYER_SNAPSHOT_ATTRIBUTES: Final = ( + "keys", + "values", + "conv_states", + "recurrent_states", + "is_conv_states_initialized", + "is_recurrent_states_initialized", + "has_previous_state", + "conv_kernel_size", + "is_initialized", + "device", + "dtype", + "record_past", +) +_CACHE_OBJECT_SNAPSHOT_ATTRIBUTES: Final = ("sequence_length", "_seen_tokens") + + +@dataclass(slots=True) +class _CacheSnapshot: + layers: list[dict[str, tuple[bool, object]]] + cache_attributes: dict[str, tuple[bool, object]] + + +def _snapshot_cache(cache: object) -> _CacheSnapshot: + """Retain enough warm-cache state to roll back one failed H=1 step.""" + + layers = getattr(cache, "layers", None) + if layers is None or not hasattr(layers, "__iter__"): + raise Experiment013AdapterError("DynamicCache does not expose iterable layers") + layer_snapshots: list[dict[str, tuple[bool, object]]] = [] + for layer in layers: + snapshot: dict[str, tuple[bool, object]] = {} + for name in _CACHE_LAYER_SNAPSHOT_ATTRIBUTES: + if not hasattr(layer, name): + snapshot[name] = (False, None) + continue + value = getattr(layer, name) + if name in ("conv_states", "recurrent_states") and isinstance(value, dict): + copied = { + state_index: item.detach().clone() if isinstance(item, torch.Tensor) else item + for state_index, item in value.items() + } + elif name in ("keys", "values") and isinstance(value, torch.Tensor): + # Attention cache updates replace these tensors rather than + # mutating the previous objects, so a detached reference is a + # sufficient rollback point without copying the full history. + copied = value.detach() + elif isinstance(value, dict): + copied = dict(value) + else: + copied = value + snapshot[name] = (True, copied) + layer_snapshots.append(snapshot) + cache_attributes = { + name: (hasattr(cache, name), getattr(cache, name, None)) + for name in _CACHE_OBJECT_SNAPSHOT_ATTRIBUTES + } + return _CacheSnapshot(layers=layer_snapshots, cache_attributes=cache_attributes) + + +def _restore_cache(cache: object, snapshot: _CacheSnapshot) -> None: + layers = getattr(cache, "layers", None) + if layers is None or len(layers) != len(snapshot.layers): + raise RuntimeError("cannot roll back a DynamicCache whose layer count changed") + for layer, attributes in zip(layers, snapshot.layers, strict=True): + for name, (existed, value) in attributes.items(): + if existed: + setattr(layer, name, value) + elif hasattr(layer, name): + delattr(layer, name) + for name, (existed, value) in snapshot.cache_attributes.items(): + if existed: + setattr(cache, name, value) + elif hasattr(cache, name): + delattr(cache, name) + + +def _clone_inference_cache_tensors(cache: object) -> None: + """Replace inference tensors before autograd may need to save them.""" + + for layer in getattr(cache, "layers", ()): + for name in ("keys", "values"): + value = getattr(layer, name, None) + if isinstance(value, torch.Tensor) and value.is_inference(): + setattr(layer, name, value.detach().clone()) + for name in ("conv_states", "recurrent_states"): + values = getattr(layer, name, None) + if isinstance(values, dict): + for state_index, value in tuple(values.items()): + if isinstance(value, torch.Tensor) and value.is_inference(): + values[state_index] = value.detach().clone() + + +def _detach_cache_tensors(cache: object) -> None: + """Release the one-step graph while retaining the successful trajectory.""" + + for layer in getattr(cache, "layers", ()): + for name in ("keys", "values"): + value = getattr(layer, name, None) + if isinstance(value, torch.Tensor): + setattr(layer, name, value.detach()) + for name in ("conv_states", "recurrent_states"): + values = getattr(layer, name, None) + if isinstance(values, dict): + for state_index, value in tuple(values.items()): + if isinstance(value, torch.Tensor): + values[state_index] = value.detach() + @dataclass(slots=True) class _SequenceState: @@ -638,6 +747,7 @@ def __init__(self, context: AdapterConstructionContext) -> None: self._model_loading_diagnostic_counts: dict[str, int] | None = None self._sequence: _SequenceState | None = None self._sequence_failed = False + self._fisher_step_count = 0 def _prepare_materialization(self) -> None: if self._materialization_attempted: @@ -1022,6 +1132,296 @@ def step_token( finally: observer.deactivate(context_token) + def step_token_with_fisher( + self, + model: object, + *, + token_id: int, + position: int, + target_token_id: int, + capture_state: bool, + ) -> FisherStepObservation: + """Advance one warm token and differentiate its next-token NLL to ``S_b``. + + If ``position == b + 1``, the warm cache contains ``S_b``. This method + consumes ``x_(b+1)`` and scores the resulting logits against + ``x_(b+2)``. Successful calls retain the numerically advanced cache but + detach its tensors; failed calls restore the pre-step cache snapshot. + """ + + self._require_loaded_model(model) + sequence = self._sequence + observer = self._observer + device = self._model_device + if sequence is None or observer is None or device is None or self._sequence_failed: + raise Experiment013AdapterError("no healthy calibration sequence is active") + token = _require_non_negative_int(token_id, context="token_id") + target = _require_non_negative_int(target_token_id, context="target_token_id") + current = _require_non_negative_int(position, context="position") + if not isinstance(capture_state, bool): + raise TypeError("capture_state must be bool") + if current != sequence.next_position or current >= sequence.token_count: + raise Experiment013AdapterError( + "adapter token position is not the next causal position" + ) + if current == 0: + raise Experiment013AdapterError("H=1 Fisher calibration requires a warm S_b cache") + if current + 1 >= sequence.token_count: + raise Experiment013AdapterError( + "H=1 Fisher calibration requires an authenticated x_(b+2) target" + ) + if getattr(model, "training", True): + raise Experiment013AdapterError( + "Qwen3.5 model entered training mode during calibration" + ) + if self._cache_length(sequence.cache) != current: + raise Experiment013AdapterError("DynamicCache length drifted before H=1 forward") + + parameters_method = getattr(model, "parameters", None) + if not callable(parameters_method): + raise Experiment013AdapterError("Qwen3.5 model does not expose parameters") + parameters = tuple(parameters_method()) + if not parameters or any(parameter.requires_grad for parameter in parameters): + raise Experiment013AdapterError("H=1 Fisher calibration requires frozen parameters") + if any(parameter.grad is not None for parameter in parameters): + raise Experiment013AdapterError("model parameters must not have pre-existing gradients") + + cache_dictionary = getattr(sequence.cache, "__dict__", None) + if not isinstance(cache_dictionary, dict): + raise Experiment013AdapterError( + "DynamicCache does not support an instance-scoped functional update" + ) + original_update = getattr(sequence.cache, "update_recurrent_state", None) + if not callable(original_update): + raise Experiment013AdapterError("DynamicCache lacks update_recurrent_state") + had_update_attribute = "update_recurrent_state" in cache_dictionary + original_update_attribute = cache_dictionary.get("update_recurrent_state") + snapshot = _snapshot_cache(sequence.cache) + leaves: dict[int, torch.Tensor] = {} + source_snapshots: dict[int, torch.Tensor] = {} + capture = _StepCapture(cache=sequence.cache, position=current, receipts={}) + context_token: Token[_StepCapture | None] | None = None + update_installed = False + step_succeeded = False + + def differentiable_update( + cache: object, + recurrent_states: torch.Tensor, + layer_idx: int, + state_idx: int = 0, + **update_kwargs: Any, + ) -> torch.Tensor: + if layer_idx not in leaves: + return original_update( + recurrent_states, + layer_idx, + state_idx=state_idx, + **update_kwargs, + ) + if state_idx != 0: + raise Experiment013AdapterError( + "Qwen3.5 H=1 Fisher calibration supports recurrent state_idx=0 only" + ) + layers = getattr(cache, "layers", None) + if layers is None or layer_idx >= len(layers): + raise Experiment013AdapterError("DynamicCache recurrent layer disappeared") + layer = layers[layer_idx] + recurrent = getattr(layer, "recurrent_states", None) + initialized = getattr(layer, "is_recurrent_states_initialized", None) + if not isinstance(recurrent, dict) or not isinstance(initialized, dict): + raise Experiment013AdapterError( + "DynamicCache recurrent-state storage differs from Transformers 5.14.1" + ) + recurrent[state_idx] = recurrent_states + initialized[state_idx] = True + return recurrent_states + + try: + with torch.inference_mode(False), torch.enable_grad(): + _clone_inference_cache_tensors(sequence.cache) + for layer_index in RECURRENT_LAYER_INDICES: + state = observer._cache_state(sequence.cache, layer_index) + if ( + state is None + or tuple(state.shape) != STATE_SHAPE + or state.dtype != torch.float32 + or state.device != device + or not torch.isfinite(state).all().item() + ): + raise Experiment013AdapterError( + f"warm DynamicCache state differs at layer {layer_index}" + ) + source_snapshot = state.detach().clone() + leaf = source_snapshot.clone().requires_grad_(True) + sequence.cache.layers[layer_index].recurrent_states[0] = leaf + leaves[layer_index] = leaf + source_snapshots[layer_index] = source_snapshot + + sequence.cache.update_recurrent_state = MethodType( # type: ignore[method-assign] + differentiable_update, + sequence.cache, + ) + update_installed = True + context_token = observer.activate(capture) + input_ids = torch.tensor([[token]], dtype=torch.long, device=device) + position_ids = torch.tensor([[current]], dtype=torch.long, device=device) + output = model( + input_ids=input_ids, + position_ids=position_ids, + past_key_values=sequence.cache, + use_cache=True, + logits_to_keep=1, + ) + if getattr(output, "past_key_values", None) is not sequence.cache: + raise Experiment013AdapterError( + "Qwen3.5 returned a different DynamicCache object" + ) + if self._cache_length(sequence.cache) != current + 1: + raise Experiment013AdapterError( + "DynamicCache did not advance by exactly one token" + ) + logits = getattr(output, "logits", None) + if ( + not isinstance(logits, torch.Tensor) + or logits.ndim != 3 + or tuple(logits.shape[:2]) != (1, 1) + or logits.shape[-1] <= target + or not torch.isfinite(logits).all().item() + ): + raise Experiment013AdapterError( + "Qwen3.5 output does not expose finite one-token target logits" + ) + target_tensor = torch.tensor([target], dtype=torch.long, device=device) + target_nll = torch.nn.functional.cross_entropy( + logits[:, -1, :].to(torch.float32), + target_tensor, + ) + if not torch.isfinite(target_nll).item(): + raise Experiment013AdapterError("H=1 target-token NLL became non-finite") + gradients = torch.autograd.grad( + target_nll, + tuple(leaves[index] for index in RECURRENT_LAYER_INDICES), + retain_graph=False, + create_graph=False, + allow_unused=False, + ) + + if tuple(capture.receipts) != RECURRENT_LAYER_INDICES: + raise Experiment013AdapterError( + "H=1 forward did not produce one ordered receipt for every recurrent layer" + ) + queries: list[torch.Tensor] = [] + states: list[torch.Tensor] = [] + cache_matches: list[torch.Tensor] = [] + for layer_index in RECURRENT_LAYER_INDICES: + receipt = capture.receipts[layer_index] + cached = observer._cache_state(sequence.cache, layer_index) + if ( + cached is None + or tuple(cached.shape) != STATE_SHAPE + or cached.dtype != torch.float32 + or cached.device != receipt.final_state.device + ): + raise Experiment013AdapterError( + f"DynamicCache state geometry differs at layer {layer_index}" + ) + cache_matches.append(torch.eq(cached, receipt.final_state).all()) + queries.append(receipt.query[0, 0].detach()) + if capture_state: + states.append(cached[0].detach()) + match_values = torch.stack(cache_matches).detach().to(device="cpu").tolist() + mismatched = [ + layer_index + for layer_index, matches in zip( + RECURRENT_LAYER_INDICES, + match_values, + strict=True, + ) + if not matches + ] + if mismatched: + raise Experiment013AdapterError( + f"DynamicCache state differs from kernel output at layers {mismatched}" + ) + for layer_index, gradient in zip( + RECURRENT_LAYER_INDICES, + gradients, + strict=True, + ): + if ( + tuple(gradient.shape) != STATE_SHAPE + or gradient.dtype != torch.float32 + or gradient.device != device + or not torch.isfinite(gradient).all().item() + ): + raise Experiment013AdapterError( + f"H=1 state gradient differs at layer {layer_index}" + ) + if not torch.equal( + leaves[layer_index].detach(), + source_snapshots[layer_index], + ): + raise Experiment013AdapterError( + f"H=1 recurrent kernel mutated source state at layer {layer_index}" + ) + if any( + parameter.requires_grad or parameter.grad is not None for parameter in parameters + ): + raise Experiment013AdapterError( + "H=1 calibration changed or populated model parameter gradients" + ) + + step_observation = StepObservation( + position=current, + token_id=token, + layer_indices=RECURRENT_LAYER_INDICES, + recurrence_query=torch.stack(queries, dim=0).contiguous(), + recurrent_state=( + torch.stack(states, dim=0).contiguous() if capture_state else None + ), + successful_kernel_calls_per_layer=(1,) * len(RECURRENT_LAYER_INDICES), + ) + result = FisherStepObservation( + boundary_position=current - 1, + input_position=current, + target_position=current + 1, + input_token_id=token, + target_token_id=target, + step_observation=step_observation, + source_recurrent_state=torch.stack( + [source_snapshots[index][0] for index in RECURRENT_LAYER_INDICES], + dim=0, + ).contiguous(), + source_state_gradient=torch.stack( + [gradient[0].detach() for gradient in gradients], + dim=0, + ).contiguous(), + target_nll=float(target_nll.detach().item()), + ) + # Detachment is part of the successful state transition. If it + # fails, the outer exception path invalidates the sequence and the + # finally block restores the complete pre-step snapshot. + _detach_cache_tensors(sequence.cache) + sequence.next_position += 1 + self._fisher_step_count += 1 + step_succeeded = True + return result + except BaseException: + self._sequence_failed = True + raise + finally: + try: + if not step_succeeded: + _restore_cache(sequence.cache, snapshot) + finally: + if update_installed: + if had_update_attribute: + cache_dictionary["update_recurrent_state"] = original_update_attribute + else: + cache_dictionary.pop("update_recurrent_state", None) + if context_token is not None: + observer.deactivate(context_token) + def end_sequence(self, model: object, record: Mapping[str, object]) -> None: self._require_loaded_model(model) del record @@ -1048,6 +1448,7 @@ def runtime_metadata(self) -> Mapping[str, object]: "adapter_revision": ADAPTER_REVISION, "capture_input_sha256": self._capture_input_sha256, "device": None if self._model_device is None else str(self._model_device), + "fisher_step_count": self._fisher_step_count, "kernel_backend": "transformers_pure_torch_gated_delta_rule", "materialization_attempted": self._materialization_attempted, "materialized_sequence_count": ( diff --git a/src/recurquant/experiment013_source.py b/src/recurquant/experiment013_source.py index 17f5867..1e43a70 100644 --- a/src/recurquant/experiment013_source.py +++ b/src/recurquant/experiment013_source.py @@ -1,8 +1,9 @@ """Fail-closed local-source identity for Experiment 013. The manifest produced here is portable: it contains only repository-relative -paths and Git/content identities. Absolute worktree, Git-directory, index, -and object-store paths are authenticated locally but never serialized. +paths and Git/content identities, including the canonical Git executable's +digest and size. Absolute worktree, Git-directory, index, object-store, and +executable paths are authenticated locally but never serialized. The verifier and its tests are part of the frozen inventory. This does not create a hash cycle: their committed bytes do not embed the resulting manifest @@ -15,15 +16,17 @@ import json import os import re +import shutil +import stat import subprocess import sys from collections.abc import Mapping, Sequence from dataclasses import dataclass from pathlib import Path, PurePosixPath -EXPERIMENT013_SOURCE_MANIFEST_SCHEMA = "recurquant.experiment013.source-manifest.v1" -EXPERIMENT013_SOURCE_MANIFEST_PROFILE = "experiment-013-static-q468-frozen-source-v1" -EXPERIMENT013_REPOSITORY_BINDING_SCHEMA = "recurquant.experiment013.repository-binding.v1" +EXPERIMENT013_SOURCE_MANIFEST_SCHEMA = "recurquant.experiment013.source-manifest.v2" +EXPERIMENT013_SOURCE_MANIFEST_PROFILE = "experiment-013-static-q468-frozen-source-v2" +EXPERIMENT013_REPOSITORY_BINDING_SCHEMA = "recurquant.experiment013.repository-binding.v2" # Keep this explicit. Discovery by glob would silently change the experiment # when an unrelated file was added or removed. @@ -38,13 +41,17 @@ "scripts/capture_static_q468_identity_input.py", "scripts/generate_static_q468_ruler_receipts.py", "scripts/launch_static_q468_calibration.py", + "scripts/launch_static_q468_stage_a.py", "scripts/resolve_static_q468_identity.py", "scripts/run_static_q468_calibration.py", + "scripts/screen_static_q468_stage_a.py", + "src/recurquant/cache.py", "src/recurquant/evidence.py", "src/recurquant/experiment013_calibration_api.py", "src/recurquant/experiment013_parquet.py", "src/recurquant/experiment013_qwen35_adapter.py", "src/recurquant/experiment013_source.py", + "src/recurquant/experiment013_stage_a.py", "src/recurquant/metrics.py", "src/recurquant/mixed_quantization.py", "src/recurquant/multibit_policy.py", @@ -68,10 +75,13 @@ "tests/test_experiment013_parquet.py", "tests/test_experiment013_qwen35_adapter.py", "tests/test_experiment013_source.py", + "tests/test_experiment013_stage_a.py", "tests/test_generate_static_q468_ruler_receipts.py", "tests/test_launch_static_q468_calibration.py", + "tests/test_launch_static_q468_stage_a.py", "tests/test_resolve_static_q468_identity.py", "tests/test_run_static_q468_calibration.py", + "tests/test_screen_static_q468_stage_a.py", "tests/test_static_q468.py", "tests/test_static_q468_cache.py", "tests/test_static_q468_calibration.py", @@ -85,11 +95,13 @@ "profile", "object_format", "source_commit", + "git_executable", "repository_binding", "paths", "canonical_manifest_sha256", } ) +_GIT_EXECUTABLE_FIELDS = frozenset({"sha256", "size_bytes"}) _PATH_FIELDS = frozenset( { "path", @@ -121,12 +133,16 @@ "history_grafts_absent", "inherited_git_environment_scrubbed", "system_and_global_git_config_disabled", + "authenticated_git_executable_bound", + "git_executable_regular_non_reparse", + "path_lookup_not_used_after_git_resolution", } ) _TRUE_BINDING_FIELDS = _BINDING_FIELDS - {"schema", "worktree_layout"} _SHA1_RE = re.compile(r"[0-9a-f]{40}") _SHA256_RE = re.compile(r"[0-9a-f]{64}") _SAFE_GIT_MODES = frozenset({"100644", "100755"}) +_WINDOWS_REPARSE_POINT = 0x400 _FORBIDDEN_LOCAL_CONFIG_KEYS = frozenset( { "core.alternaterefscommand", @@ -164,6 +180,13 @@ class _RepositoryIdentity: public_binding: dict[str, object] +@dataclass(frozen=True) +class GitExecutableIdentity: + path: Path + size_bytes: int + sha256: str + + def _canonical_json_bytes(value: object) -> bytes: try: return (json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n").encode("utf-8") @@ -205,6 +228,89 @@ def _sha256(value: object, *, name: str) -> str: return value +def _positive_int(value: object, *, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise Experiment013SourceError(f"{name} must be a positive integer") + return value + + +def _is_link_or_reparse(path: Path) -> bool: + try: + status = path.lstat() + except OSError as error: + raise Experiment013SourceError(f"required path is unavailable: {path}") from error + return path.is_symlink() or bool( + getattr(status, "st_file_attributes", 0) & _WINDOWS_REPARSE_POINT + ) + + +def _assert_no_link_or_reparse_components(path: Path) -> None: + current = Path(path.anchor) + for part in path.parts[1:]: + current /= part + if _is_link_or_reparse(current): + raise Experiment013SourceError( + "authenticated Git executable traverses a link or reparse point" + ) + + +def authenticate_git_executable( + executable: str | os.PathLike[str] | None = None, +) -> GitExecutableIdentity: + """Resolve and hash one canonical Git binary before any Git subprocess runs.""" + + selected: str | os.PathLike[str] + if executable is None: + discovered = shutil.which("git") + if discovered is None: + raise Experiment013SourceError("Git executable is unavailable") + selected = discovered + else: + selected = executable + try: + resolved = Path(selected).resolve(strict=True) + except (OSError, TypeError, ValueError) as error: + raise Experiment013SourceError("Git executable path is unavailable") from error + if not resolved.is_absolute(): + raise Experiment013SourceError("Git executable path is not absolute") + if resolved.name.casefold() == "git.exe" and resolved.parent.name.casefold() == "cmd": + implementation = resolved.parent.parent / "mingw64" / "bin" / "git.exe" + try: + resolved = implementation.resolve(strict=True) + except OSError as error: + raise Experiment013SourceError( + "Git-for-Windows cmd shim has no canonical mingw64 executable" + ) from error + _assert_no_link_or_reparse_components(resolved) + try: + before = resolved.stat() + data = resolved.read_bytes() + after = resolved.stat() + except OSError as error: + raise Experiment013SourceError("Git executable cannot be authenticated") from error + if ( + not stat.S_ISREG(before.st_mode) + or not stat.S_ISREG(after.st_mode) + or before.st_size <= 0 + or before.st_size != after.st_size + or before.st_mtime_ns != after.st_mtime_ns + or before.st_dev != after.st_dev + or before.st_ino != after.st_ino + or len(data) != after.st_size + or _is_link_or_reparse(resolved) + ): + raise Experiment013SourceError("Git executable changed or is not a regular file") + return GitExecutableIdentity( + path=resolved, + size_bytes=after.st_size, + sha256=hashlib.sha256(data).hexdigest(), + ) + + +def _git_executable_record(identity: GitExecutableIdentity) -> dict[str, object]: + return {"sha256": identity.sha256, "size_bytes": identity.size_bytes} + + def _canonical_relative_path(value: object, *, name: str) -> str: if not isinstance(value, str) or not value or value != value.strip(): raise Experiment013SourceError(f"{name} must be a non-empty canonical path") @@ -238,6 +344,25 @@ def validate_experiment013_source_manifest( raise Experiment013SourceError("source manifest requires the frozen SHA-1 format") source_commit = _sha1(manifest["source_commit"], name="source_commit") + raw_git_executable = manifest["git_executable"] + if not isinstance(raw_git_executable, Mapping): + raise Experiment013SourceError("git_executable must be a mapping") + _exact_fields( + raw_git_executable, + _GIT_EXECUTABLE_FIELDS, + name="git_executable", + ) + git_executable = { + "sha256": _sha256( + raw_git_executable["sha256"], + name="git_executable.sha256", + ), + "size_bytes": _positive_int( + raw_git_executable["size_bytes"], + name="git_executable.size_bytes", + ), + } + raw_binding = manifest["repository_binding"] if not isinstance(raw_binding, Mapping): raise Experiment013SourceError("repository_binding must be a mapping") @@ -294,6 +419,7 @@ def validate_experiment013_source_manifest( "profile": EXPERIMENT013_SOURCE_MANIFEST_PROFILE, "object_format": "sha1", "source_commit": source_commit, + "git_executable": git_executable, "repository_binding": binding, "paths": entries, } @@ -305,8 +431,11 @@ def validate_experiment013_source_manifest( def _sanitized_git_environment() -> dict[str, str]: + inherited = {key.upper(): (key, value) for key, value in os.environ.items()} environment = { - key: value for key, value in os.environ.items() if not key.upper().startswith("GIT_") + inherited[name][0]: inherited[name][1] + for name in ("SYSTEMROOT", "WINDIR", "COMSPEC") + if name in inherited } environment.update( { @@ -321,6 +450,10 @@ def _sanitized_git_environment() -> dict[str, str]: "GIT_CONFIG_VALUE_1": "false", "GIT_CONFIG_KEY_2": "core.hooksPath", "GIT_CONFIG_VALUE_2": os.devnull, + "GIT_AUTHOR_NAME": "RecurQuant Experiment 013", + "GIT_AUTHOR_EMAIL": "experiment013@invalid", + "GIT_COMMITTER_NAME": "RecurQuant Experiment 013", + "GIT_COMMITTER_EMAIL": "experiment013@invalid", "LC_ALL": "C", "LANG": "C", } @@ -329,13 +462,14 @@ def _sanitized_git_environment() -> dict[str, str]: def _run_git( + git: GitExecutableIdentity, root: Path, *arguments: str, input_bytes: bytes | None = None, ) -> subprocess.CompletedProcess[bytes]: try: return subprocess.run( - ["git", *arguments], + [str(git.path), *arguments], cwd=root, check=False, capture_output=True, @@ -348,11 +482,12 @@ def _run_git( def _git_bytes( + git: GitExecutableIdentity, root: Path, *arguments: str, input_bytes: bytes | None = None, ) -> bytes: - process = _run_git(root, *arguments, input_bytes=input_bytes) + process = _run_git(git, root, *arguments, input_bytes=input_bytes) if process.returncode != 0: detail = process.stderr.decode("utf-8", errors="replace").strip() raise Experiment013SourceError( @@ -361,9 +496,9 @@ def _git_bytes( return process.stdout -def _git_text(root: Path, *arguments: str) -> str: +def _git_text(git: GitExecutableIdentity, root: Path, *arguments: str) -> str: try: - return _git_bytes(root, *arguments).decode("utf-8").strip() + return _git_bytes(git, root, *arguments).decode("utf-8").strip() except UnicodeDecodeError as error: raise Experiment013SourceError("Git returned non-UTF-8 identity data") from error @@ -387,8 +522,8 @@ def _path_has_symlink_component(root: Path, relative: str) -> bool: return False -def _assert_safe_local_config(root: Path) -> None: - raw = _git_bytes(root, "config", "--local", "--no-includes", "--null", "--list") +def _assert_safe_local_config(git: GitExecutableIdentity, root: Path) -> None: + raw = _git_bytes(git, root, "config", "--local", "--no-includes", "--null", "--list") for record in (item for item in raw.split(b"\0") if item): key_bytes, separator, value_bytes = record.partition(b"\n") if not separator or not key_bytes: @@ -409,8 +544,8 @@ def _assert_safe_local_config(root: Path) -> None: raise Experiment013SourceError("local Git config enables replacement objects") -def _assert_no_hidden_index_flags(root: Path) -> None: - raw = _git_bytes(root, "ls-files", "--cached", "-v", "-z") +def _assert_no_hidden_index_flags(git: GitExecutableIdentity, root: Path) -> None: + raw = _git_bytes(git, root, "ls-files", "--cached", "-v", "-z") records = [record for record in raw.split(b"\0") if record] malformed = [record for record in records if len(record) < 3 or record[1:2] != b" "] if malformed: @@ -423,25 +558,28 @@ def _assert_no_hidden_index_flags(root: Path) -> None: ) -def _assert_no_tracked_or_untracked_changes(root: Path) -> None: +def _assert_no_tracked_or_untracked_changes(git: GitExecutableIdentity, root: Path) -> None: checks = ( ("tracked staged", ("diff", "--cached", "--no-ext-diff", "--quiet", "HEAD", "--")), ("tracked unstaged", ("diff", "--no-ext-diff", "--quiet", "--")), ) for label, arguments in checks: - process = _run_git(root, *arguments) + process = _run_git(git, root, *arguments) if process.returncode == 1: raise Experiment013SourceError(f"repository has {label} changes") if process.returncode != 0: raise Experiment013SourceError(f"cannot authenticate {label} changes") - status = _git_bytes(root, "status", "--porcelain=v1", "-z", "--untracked-files=all") + status = _git_bytes(git, root, "status", "--porcelain=v1", "-z", "--untracked-files=all") if status: raise Experiment013SourceError( "repository contains non-ignored untracked paths or tracked status drift" ) -def _authenticate_repository(repo_root: str | Path) -> _RepositoryIdentity: +def _authenticate_repository( + repo_root: str | Path, + git: GitExecutableIdentity, +) -> _RepositoryIdentity: try: root = Path(repo_root).resolve(strict=True) except OSError as error: @@ -449,20 +587,22 @@ def _authenticate_repository(repo_root: str | Path) -> _RepositoryIdentity: if not root.is_dir(): raise Experiment013SourceError("repository root must be a directory") - top_level = _resolved_git_path(root, _git_text(root, "rev-parse", "--show-toplevel")) - git_dir = _resolved_git_path(root, _git_text(root, "rev-parse", "--absolute-git-dir")) - common_dir = _resolved_git_path(root, _git_text(root, "rev-parse", "--git-common-dir")) - object_dir = _resolved_git_path(root, _git_text(root, "rev-parse", "--git-path", "objects")) - index_path = _resolved_git_path(root, _git_text(root, "rev-parse", "--git-path", "index")) + top_level = _resolved_git_path(root, _git_text(git, root, "rev-parse", "--show-toplevel")) + git_dir = _resolved_git_path(root, _git_text(git, root, "rev-parse", "--absolute-git-dir")) + common_dir = _resolved_git_path(root, _git_text(git, root, "rev-parse", "--git-common-dir")) + object_dir = _resolved_git_path( + root, _git_text(git, root, "rev-parse", "--git-path", "objects") + ) + index_path = _resolved_git_path(root, _git_text(git, root, "rev-parse", "--git-path", "index")) if top_level != root: raise Experiment013SourceError("Git top-level does not equal the requested repository root") - if _git_text(root, "rev-parse", "--is-inside-work-tree") != "true": + if _git_text(git, root, "rev-parse", "--is-inside-work-tree") != "true": raise Experiment013SourceError("repository root is not inside a Git worktree") - if _git_text(root, "rev-parse", "--is-bare-repository") != "false": + if _git_text(git, root, "rev-parse", "--is-bare-repository") != "false": raise Experiment013SourceError("bare Git repositories are forbidden") - if _git_text(root, "rev-parse", "--show-object-format") != "sha1": + if _git_text(git, root, "rev-parse", "--show-object-format") != "sha1": raise Experiment013SourceError("Experiment 013 requires the SHA-1 Git object format") - if _git_text(root, "rev-parse", "--is-shallow-repository") != "false": + if _git_text(git, root, "rev-parse", "--is-shallow-repository") != "false": raise Experiment013SourceError("shallow Git history is forbidden") if not git_dir.is_dir() or not common_dir.is_dir() or not index_path.is_file(): raise Experiment013SourceError("Git directory or index identity is malformed") @@ -527,11 +667,11 @@ def _authenticate_repository(repo_root: str | Path) -> _RepositoryIdentity: ): if unsafe_path.exists(): raise Experiment013SourceError("Git alternates, grafts, or shallow metadata exists") - if _git_text(root, "for-each-ref", "--format=%(refname)", "refs/replace"): + if _git_text(git, root, "for-each-ref", "--format=%(refname)", "refs/replace"): raise Experiment013SourceError("Git replacement refs are forbidden") - _assert_safe_local_config(root) - _assert_no_hidden_index_flags(root) - _assert_no_tracked_or_untracked_changes(root) + _assert_safe_local_config(git, root) + _assert_no_hidden_index_flags(git, root) + _assert_no_tracked_or_untracked_changes(git, root) binding: dict[str, object] = { "schema": EXPERIMENT013_REPOSITORY_BINDING_SCHEMA, @@ -553,12 +693,20 @@ def _authenticate_repository(repo_root: str | Path) -> _RepositoryIdentity: "history_grafts_absent": True, "inherited_git_environment_scrubbed": True, "system_and_global_git_config_disabled": True, + "authenticated_git_executable_bound": True, + "git_executable_regular_non_reparse": True, + "path_lookup_not_used_after_git_resolution": True, } return _RepositoryIdentity(root, git_dir, common_dir, index_path, object_dir, binding) -def _tree_entries(root: Path, commit: str) -> dict[str, tuple[str, str]]: +def _tree_entries( + git: GitExecutableIdentity, + root: Path, + commit: str, +) -> dict[str, tuple[str, str]]: raw = _git_bytes( + git, root, "ls-tree", "-r", @@ -590,8 +738,16 @@ def _tree_entries(root: Path, commit: str) -> dict[str, tuple[str, str]]: return entries -def _index_entries(root: Path) -> dict[str, tuple[str, str]]: - raw = _git_bytes(root, "ls-files", "--stage", "-z", "--", *EXPERIMENT013_SOURCE_PATHS) +def _index_entries(git: GitExecutableIdentity, root: Path) -> dict[str, tuple[str, str]]: + raw = _git_bytes( + git, + root, + "ls-files", + "--stage", + "-z", + "--", + *EXPERIMENT013_SOURCE_PATHS, + ) entries: dict[str, tuple[str, str]] = {} for record in (item for item in raw.split(b"\0") if item): metadata, separator, path_bytes = record.partition(b"\t") @@ -619,9 +775,16 @@ def _index_entries(root: Path) -> dict[str, tuple[str, str]]: return entries -def _worktree_oids(root: Path) -> dict[str, str]: +def _worktree_oids(git: GitExecutableIdentity, root: Path) -> dict[str, str]: payload = "".join(f"{path}\n" for path in EXPERIMENT013_SOURCE_PATHS).encode("utf-8") - raw = _git_bytes(root, "hash-object", "--no-filters", "--stdin-paths", input_bytes=payload) + raw = _git_bytes( + git, + root, + "hash-object", + "--no-filters", + "--stdin-paths", + input_bytes=payload, + ) lines = raw.decode("ascii", errors="strict").splitlines() if len(lines) != len(EXPERIMENT013_SOURCE_PATHS): raise Experiment013SourceError("Git did not hash every source worktree path") @@ -649,10 +812,14 @@ def _raw_file_identities(root: Path) -> dict[str, tuple[str, str]]: return identities -def _source_entries(root: Path, commit: str) -> list[dict[str, str]]: - tree = _tree_entries(root, commit) - index = _index_entries(root) - worktree = _worktree_oids(root) +def _source_entries( + git: GitExecutableIdentity, + root: Path, + commit: str, +) -> list[dict[str, str]]: + tree = _tree_entries(git, root, commit) + index = _index_entries(git, root) + worktree = _worktree_oids(git, root) raw = _raw_file_identities(root) entries: list[dict[str, str]] = [] for relative in EXPERIMENT013_SOURCE_PATHS: @@ -677,27 +844,32 @@ def _source_entries(root: Path, commit: str) -> list[dict[str, str]]: return entries -def _head(root: Path) -> str: - head = _sha1(_git_text(root, "rev-parse", "HEAD"), name="repository HEAD") - if _git_text(root, "cat-file", "-t", head) != "commit": +def _head(git: GitExecutableIdentity, root: Path) -> str: + head = _sha1(_git_text(git, root, "rev-parse", "HEAD"), name="repository HEAD") + if _git_text(git, root, "cat-file", "-t", head) != "commit": raise Experiment013SourceError("repository HEAD is not a commit") return head -def capture_experiment013_source_manifest(repo_root: str | Path) -> dict[str, object]: +def capture_experiment013_source_manifest( + repo_root: str | Path, + *, + git_executable: str | os.PathLike[str] | None = None, +) -> dict[str, object]: """Capture the exact committed Experiment 013 source identity. Non-ignored untracked files and all tracked staged/unstaged changes are rejected. Ignored artifacts do not affect capture. """ - first_repository = _authenticate_repository(repo_root) - first_head = _head(first_repository.root) - first_entries = _source_entries(first_repository.root, first_head) + git = authenticate_git_executable(git_executable) + first_repository = _authenticate_repository(repo_root, git) + first_head = _head(git, first_repository.root) + first_entries = _source_entries(git, first_repository.root, first_head) - second_repository = _authenticate_repository(first_repository.root) - second_head = _head(second_repository.root) - second_entries = _source_entries(second_repository.root, second_head) + second_repository = _authenticate_repository(first_repository.root, git) + second_head = _head(git, second_repository.root) + second_entries = _source_entries(git, second_repository.root, second_head) if ( first_head != second_head or first_entries != second_entries @@ -710,6 +882,7 @@ def capture_experiment013_source_manifest(repo_root: str | Path) -> dict[str, ob "profile": EXPERIMENT013_SOURCE_MANIFEST_PROFILE, "object_format": "sha1", "source_commit": first_head, + "git_executable": _git_executable_record(git), "repository_binding": first_repository.public_binding, "paths": first_entries, } @@ -717,11 +890,16 @@ def capture_experiment013_source_manifest(repo_root: str | Path) -> dict[str, ob return validate_experiment013_source_manifest(payload) -def _assert_ancestor(root: Path, ancestor: str, descendant: str) -> None: - object_type = _run_git(root, "cat-file", "-e", f"{ancestor}^{{commit}}") +def _assert_ancestor( + git: GitExecutableIdentity, + root: Path, + ancestor: str, + descendant: str, +) -> None: + object_type = _run_git(git, root, "cat-file", "-e", f"{ancestor}^{{commit}}") if object_type.returncode != 0: raise Experiment013SourceError("source commit is unavailable from the local object store") - process = _run_git(root, "merge-base", "--is-ancestor", ancestor, descendant) + process = _run_git(git, root, "merge-base", "--is-ancestor", ancestor, descendant) if process.returncode == 1: raise Experiment013SourceError("source commit is not an ancestor of current HEAD") if process.returncode != 0: @@ -731,6 +909,8 @@ def _assert_ancestor(root: Path, ancestor: str, descendant: str) -> None: def verify_experiment013_source_manifest( manifest: Mapping[str, object], repo_root: str | Path, + *, + git_executable: str | os.PathLike[str] | None = None, ) -> dict[str, object]: """Verify a frozen manifest against the clean source used at point-of-use. @@ -741,18 +921,23 @@ def verify_experiment013_source_manifest( normalized = validate_experiment013_source_manifest(manifest) source_commit = str(normalized["source_commit"]) + git = authenticate_git_executable(git_executable) + if _git_executable_record(git) != normalized["git_executable"]: + raise Experiment013SourceError( + "authenticated Git executable differs from the frozen source manifest" + ) - first_repository = _authenticate_repository(repo_root) - first_head = _head(first_repository.root) - _assert_ancestor(first_repository.root, source_commit, first_head) - first_entries = _source_entries(first_repository.root, source_commit) + first_repository = _authenticate_repository(repo_root, git) + first_head = _head(git, first_repository.root) + _assert_ancestor(git, first_repository.root, source_commit, first_head) + first_entries = _source_entries(git, first_repository.root, source_commit) if first_entries != normalized["paths"]: raise Experiment013SourceError("live Experiment 013 source differs from its manifest") - second_repository = _authenticate_repository(first_repository.root) - second_head = _head(second_repository.root) - _assert_ancestor(second_repository.root, source_commit, second_head) - second_entries = _source_entries(second_repository.root, source_commit) + second_repository = _authenticate_repository(first_repository.root, git) + second_head = _head(git, second_repository.root) + _assert_ancestor(git, second_repository.root, source_commit, second_head) + second_entries = _source_entries(git, second_repository.root, source_commit) if first_head != second_head or first_entries != second_entries: raise Experiment013SourceError("repository source identity changed during verification") if second_entries != normalized["paths"]: @@ -839,6 +1024,8 @@ def verify_loaded_experiment013_recurquant_modules( "EXPERIMENT013_SOURCE_MANIFEST_SCHEMA", "EXPERIMENT013_SOURCE_PATHS", "Experiment013SourceError", + "GitExecutableIdentity", + "authenticate_git_executable", "canonical_experiment013_source_manifest_bytes", "canonical_experiment013_source_manifest_sha256", "capture_experiment013_source_manifest", diff --git a/src/recurquant/experiment013_stage_a.py b/src/recurquant/experiment013_stage_a.py new file mode 100644 index 0000000..3f42ca5 --- /dev/null +++ b/src/recurquant/experiment013_stage_a.py @@ -0,0 +1,1020 @@ +"""Deterministic Stage-A evidence reduction and gates for Experiment 013. + +The module consumes already-authenticated, per-transition measurements. It is +deliberately unaware of models, datasets, caches, and devices: those belong to +the sealed evaluator. Its job is to reject incomplete or reordered inputs, +reduce the fixed twelve-example/nine-method screen, run the prespecified paired +bootstrap, and produce a canonical self-verifying JSON artifact. + +Stage A is a falsification screen. Passing these gates is not confirmation, +selector-superiority evidence, a deployment result, or a breakthrough claim. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +import unicodedata +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from fractions import Fraction +from types import MappingProxyType +from typing import Any, Final, Literal, TypeAlias, cast + +import numpy as np +import torch + +from .evidence import canonical_json_bytes +from .metrics import tail_mean +from .static_q468 import ( + STATIC_Q48_COMPARATOR_METHOD, + STATIC_Q468_ABLATION_METHOD, + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + STATIC_Q468_MSE_METHOD, + STATIC_Q468_PRIMARY_METHOD, + STATIC_Q468_UNIFORM_Q4_METHOD, + STATIC_Q468_UNIFORM_Q8_METHOD, +) +from .static_q468_cache import DYNAMIC_Q468_BASELINE_METHOD + +StageAFamily: TypeAlias = Literal["pg19", "ruler", "humaneval_plus"] + +STAGE_A_ARTIFACT_KIND: Final = "recurquant_experiment013_stage_a_falsification" +STAGE_A_SCHEMA_VERSION: Final = 1 +STAGE_A_ARTIFACT_REVISION: Final = "experiment-013-stage-a-falsification-v1" +STAGE_A_PROFILE: Final = "experiment-013-qwen35-0.8b-stage-a-frozen-v1" + +FP32_METHOD: Final = "fp32_reference" +UNIFORM_RHT_Q4_METHOD: Final = STATIC_Q468_UNIFORM_Q4_METHOD +UNIFORM_RHT_Q8_METHOD: Final = STATIC_Q468_UNIFORM_Q8_METHOD + +STAGE_A_FAMILY_ORDER: Final[tuple[StageAFamily, ...]] = ( + "pg19", + "ruler", + "humaneval_plus", +) +STAGE_A_METHOD_ORDER: Final = ( + FP32_METHOD, + UNIFORM_RHT_Q4_METHOD, + UNIFORM_RHT_Q8_METHOD, + STATIC_Q48_COMPARATOR_METHOD, + STATIC_Q468_ABLATION_METHOD, + DYNAMIC_Q468_BASELINE_METHOD, + STATIC_Q468_MSE_METHOD, + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + STATIC_Q468_PRIMARY_METHOD, +) +DECISION_COMPARATOR_ORDER: Final = ( + DYNAMIC_Q468_BASELINE_METHOD, + STATIC_Q468_MSE_METHOD, + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, +) + +STAGE_A_EXAMPLES_PER_FAMILY: Final = 4 +STAGE_A_EXAMPLE_COUNT: Final = 12 +BOOTSTRAP_SEED: Final = 2_339 +BOOTSTRAP_SAMPLES: Final = 10_000 +BOOTSTRAP_UPPER_QUANTILE: Final = 0.95 +BOOTSTRAP_RNG_ALGORITHM: Final = "numpy-pcg64-integers-v1" +BOOTSTRAP_QUANTILE_ALGORITHM: Final = ( + "nearest-rank: ascending stable sort; zero-based index ceil(q*N)-1; no interpolation" +) +MAX_ONE_SIDED_95_UPPER_EXCESS_NLL: Final = 0.010 +MAX_FAMILY_POINT_EXCESS_NLL: Final = 0.015 +MAX_FAMILY_MACRO_TOP1_TRAIL: Final = 0.005 +TAIL_KL_FRACTION: Final = 0.05 + +CLAIM_BOUNDARY: Final = ( + "Stage A is a falsification screen only. Passage is not confirmation, selector " + "superiority evidence, deployment evidence, state of the art, or a breakthrough claim." +) + +_SHA256_RE = re.compile(r"[0-9a-f]{64}") +_OUTER_FIELDS = frozenset( + {"artifact_kind", "canonical_evidence_sha256", "evidence", "schema_version"} +) +_EVIDENCE_FIELDS = frozenset( + { + "artifact_kind", + "artifact_revision", + "claim_boundary", + "decision", + "dependencies", + "example_summaries", + "examples", + "method_roles", + "method_summaries", + "metric_contract", + "profile", + "schema_version", + "token_rows", + } +) +_DEPENDENCY_FIELDS = frozenset( + {"stage_a_calibration_binding_file_sha256", "stage_a_identity_file_sha256"} +) +_EXAMPLE_FIELDS = frozenset( + { + "canonical_id", + "continuation_token_count", + "family", + "identity_record_sha256", + "selection_rank", + } +) +_TOKEN_ROW_FIELDS = frozenset( + { + "canonical_id", + "family", + "identity_record_sha256", + "kl", + "method_id", + "method_nll", + "reference_nll", + "selection_rank", + "top1_agreement", + "transition_index", + } +) +_METRIC_FIELDS = ( + "excess_nll", + "reference_nll", + "method_nll", + "mean_kl", + "cvar95_kl", + "max_kl", + "top1_agreement", +) + + +@dataclass(frozen=True, slots=True) +class StageAExample: + """One frozen Stage-A example and its cache-exposed metric span.""" + + family: StageAFamily + canonical_id: str + selection_rank: int + continuation_token_count: int + identity_record_sha256: str + + @property + def transition_count(self) -> int: + return self.continuation_token_count - 1 + + +@dataclass(frozen=True, slots=True) +class StageATokenRow: + """One method's metrics for one identity-bound cache-exposed transition.""" + + family: StageAFamily + canonical_id: str + selection_rank: int + identity_record_sha256: str + method_id: str + transition_index: int + reference_nll: float + method_nll: float + kl: float + top1_agreement: bool + + +@dataclass(frozen=True, slots=True) +class StageAEvidenceArtifact: + """Strictly decoded Stage-A evidence with both content commitments.""" + + evidence: Mapping[str, Any] + canonical_evidence_sha256: str + file_sha256: str + serialized_bytes: bytes + _verified_passed: bool + + @property + def passed(self) -> bool: + return self._verified_passed + + +def _deep_freeze_json(value: Any) -> Any: + """Return an immutable view of a JSON value tree. + + The verifier must not hand callers a mutable object whose later mutation + can change the meaning of an already authenticated artifact. + """ + + if isinstance(value, dict): + return MappingProxyType({key: _deep_freeze_json(nested) for key, nested in value.items()}) + if isinstance(value, list): + return tuple(_deep_freeze_json(nested) for nested in value) + return value + + +def _sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _require_sha256(value: object, *, name: str) -> str: + if not isinstance(value, str) or _SHA256_RE.fullmatch(value) is None: + raise ValueError(f"{name} must be a lowercase 64-character SHA-256 digest") + return value + + +def _require_exact_fields( + value: Mapping[str, object], expected: frozenset[str], *, name: str +) -> None: + actual = frozenset(value) + if actual != expected: + raise ValueError( + f"{name} fields drifted (missing={sorted(expected - actual)}, " + f"extra={sorted(actual - expected)})" + ) + + +def _require_canonical_id(value: object, *, name: str) -> str: + if not isinstance(value, str) or not value or value != value.strip(): + raise ValueError(f"{name} must be a non-empty stripped string") + if unicodedata.normalize("NFC", value) != value: + raise ValueError(f"{name} must use NFC Unicode normalization") + if any(unicodedata.category(character) == "Cc" for character in value): + raise ValueError(f"{name} must not contain control characters") + return value + + +def _require_int(value: object, *, name: str, minimum: int = 0) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise TypeError(f"{name} must be an integer") + if value < minimum: + raise ValueError(f"{name} must be at least {minimum}") + return value + + +def _require_finite(value: object, *, name: str, nonnegative: bool = False) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise TypeError(f"{name} must be a finite real number") + result = float(value) + if not math.isfinite(result): + raise ValueError(f"{name} must be finite") + if nonnegative and result < 0.0: + raise ValueError(f"{name} must be nonnegative") + return 0.0 if result == 0.0 else result + + +def _normalized_examples(examples: Sequence[StageAExample]) -> tuple[StageAExample, ...]: + if isinstance(examples, (str, bytes, bytearray)) or not isinstance(examples, Sequence): + raise TypeError("examples must be a sequence") + if len(examples) != STAGE_A_EXAMPLE_COUNT: + raise ValueError("Stage A requires exactly twelve examples") + + normalized: list[StageAExample] = [] + identities: set[tuple[str, str]] = set() + for index, raw in enumerate(examples): + if not isinstance(raw, StageAExample): + raise TypeError(f"examples[{index}] must be a StageAExample") + expected_family = STAGE_A_FAMILY_ORDER[index // STAGE_A_EXAMPLES_PER_FAMILY] + expected_rank = index % STAGE_A_EXAMPLES_PER_FAMILY + if raw.family != expected_family or raw.selection_rank != expected_rank: + raise ValueError( + "Stage-A examples must be ordered by frozen family order and selection rank" + ) + canonical_id = _require_canonical_id(raw.canonical_id, name=f"examples[{index}].id") + continuation = _require_int( + raw.continuation_token_count, + name=f"examples[{index}].continuation_token_count", + minimum=2, + ) + identity_hash = _require_sha256( + raw.identity_record_sha256, + name=f"examples[{index}].identity_record_sha256", + ) + identity = (raw.family, canonical_id) + if identity in identities: + raise ValueError(f"duplicate Stage-A example identity: {identity}") + identities.add(identity) + normalized.append( + StageAExample( + family=raw.family, + canonical_id=canonical_id, + selection_rank=expected_rank, + continuation_token_count=continuation, + identity_record_sha256=identity_hash, + ) + ) + return tuple(normalized) + + +def _normalize_row(row: StageATokenRow, *, index: int) -> StageATokenRow: + if not isinstance(row, StageATokenRow): + raise TypeError(f"token_rows[{index}] must be a StageATokenRow") + family = cast(StageAFamily, row.family) + if family not in STAGE_A_FAMILY_ORDER: + raise ValueError(f"token_rows[{index}].family is unknown") + canonical_id = _require_canonical_id(row.canonical_id, name=f"token_rows[{index}].id") + rank = _require_int(row.selection_rank, name=f"token_rows[{index}].selection_rank") + identity_hash = _require_sha256( + row.identity_record_sha256, + name=f"token_rows[{index}].identity_record_sha256", + ) + if row.method_id not in STAGE_A_METHOD_ORDER: + raise ValueError(f"token_rows[{index}].method_id is not in the frozen method order") + transition = _require_int( + row.transition_index, + name=f"token_rows[{index}].transition_index", + ) + reference_nll = _require_finite( + row.reference_nll, + name=f"token_rows[{index}].reference_nll", + nonnegative=True, + ) + method_nll = _require_finite( + row.method_nll, + name=f"token_rows[{index}].method_nll", + nonnegative=True, + ) + kl = _require_finite( + row.kl, + name=f"token_rows[{index}].kl", + nonnegative=True, + ) + if not isinstance(row.top1_agreement, bool): + raise TypeError(f"token_rows[{index}].top1_agreement must be a bool") + return StageATokenRow( + family=family, + canonical_id=canonical_id, + selection_rank=rank, + identity_record_sha256=identity_hash, + method_id=row.method_id, + transition_index=transition, + reference_nll=reference_nll, + method_nll=method_nll, + kl=kl, + top1_agreement=row.top1_agreement, + ) + + +def _normalized_rows( + examples: tuple[StageAExample, ...], token_rows: Sequence[StageATokenRow] +) -> tuple[StageATokenRow, ...]: + if isinstance(token_rows, (str, bytes, bytearray)) or not isinstance(token_rows, Sequence): + raise TypeError("token_rows must be a sequence") + expected_count = sum(example.transition_count for example in examples) * len( + STAGE_A_METHOD_ORDER + ) + if len(token_rows) != expected_count: + raise ValueError( + f"Stage-A token-row count differs from the complete grid: " + f"expected {expected_count}, observed {len(token_rows)}" + ) + + normalized = tuple(_normalize_row(row, index=index) for index, row in enumerate(token_rows)) + cursor = 0 + references: dict[tuple[str, str, int], float] = {} + for example in examples: + for method_id in STAGE_A_METHOD_ORDER: + for transition_index in range(example.transition_count): + row = normalized[cursor] + expected_identity = ( + example.family, + example.canonical_id, + example.selection_rank, + example.identity_record_sha256, + method_id, + transition_index, + ) + observed_identity = ( + row.family, + row.canonical_id, + row.selection_rank, + row.identity_record_sha256, + row.method_id, + row.transition_index, + ) + if observed_identity != expected_identity: + raise ValueError( + f"missing, duplicate, or reordered Stage-A token row at index {cursor}" + ) + reference_key = (example.family, example.canonical_id, transition_index) + recorded_reference = references.setdefault(reference_key, row.reference_nll) + if row.reference_nll != recorded_reference: + raise ValueError( + "reference NLL differs across methods for one cache-exposed transition" + ) + if method_id == FP32_METHOD: + if row.method_nll != row.reference_nll: + raise ValueError("FP32 method NLL must exactly equal reference NLL") + if not row.top1_agreement: + raise ValueError("FP32 top-1 agreement must be true") + if row.kl != 0.0: + raise ValueError("FP32 self-KL must be exactly zero") + cursor += 1 + return normalized + + +def _mean(values: Sequence[float]) -> float: + if not values: + raise ValueError("cannot reduce an empty metric sequence") + result = math.fsum(values) / len(values) + return 0.0 if result == 0.0 else result + + +def _cvar95(values: Sequence[float]) -> float: + if not values: + raise ValueError("cannot reduce an empty KL sequence") + # Match the reviewed fidelity_summary contract exactly: values are first + # rounded to FP32, top-k is selected there, and the tail mean is accumulated + # in FP32 by Torch. + tensor = torch.tensor(values, dtype=torch.float32) + if not torch.isfinite(tensor).all().item(): + raise ValueError("KL values must remain finite after FP32 conversion") + return float(tail_mean(tensor, fraction=TAIL_KL_FRACTION).item()) + + +def _metric_summary(rows: Sequence[StageATokenRow]) -> dict[str, float | int]: + if not rows: + raise ValueError("cannot summarize an empty token-row sequence") + reference_nll = [row.reference_nll for row in rows] + method_nll = [row.method_nll for row in rows] + excess = [row.method_nll - row.reference_nll for row in rows] + kl = [row.kl for row in rows] + top1 = [1.0 if row.top1_agreement else 0.0 for row in rows] + return { + "token_count": len(rows), + "top1_agreement_count": sum(row.top1_agreement for row in rows), + "excess_nll": _mean(excess), + "reference_nll": _mean(reference_nll), + "method_nll": _mean(method_nll), + "mean_kl": _mean(kl), + "cvar95_kl": _cvar95(kl), + "max_kl": max(kl), + "top1_agreement": _mean(top1), + } + + +def _mean_summaries(summaries: Sequence[Mapping[str, float | int]]) -> dict[str, float]: + if not summaries: + raise ValueError("cannot macro-average empty summaries") + return { + field: _mean([float(summary[field]) for summary in summaries]) for field in _METRIC_FIELDS + } + + +def _build_summaries( + examples: tuple[StageAExample, ...], rows: tuple[StageATokenRow, ...] +) -> tuple[list[dict[str, object]], list[dict[str, object]]]: + by_key: dict[tuple[str, str, str], list[StageATokenRow]] = {} + for row in rows: + by_key.setdefault((row.family, row.canonical_id, row.method_id), []).append(row) + + per_example: list[dict[str, object]] = [] + for example in examples: + for method_id in STAGE_A_METHOD_ORDER: + selected = by_key[(example.family, example.canonical_id, method_id)] + per_example.append( + { + "family": example.family, + "canonical_id": example.canonical_id, + "selection_rank": example.selection_rank, + "identity_record_sha256": example.identity_record_sha256, + "method_id": method_id, + "metrics": _metric_summary(selected), + } + ) + + per_example_lookup = { + (str(item["family"]), str(item["canonical_id"]), str(item["method_id"])): cast( + Mapping[str, float | int], item["metrics"] + ) + for item in per_example + } + method_summaries: list[dict[str, object]] = [] + for method_id in STAGE_A_METHOD_ORDER: + family_summaries: list[dict[str, object]] = [] + for family in STAGE_A_FAMILY_ORDER: + family_examples = [example for example in examples if example.family == family] + example_metrics = [ + per_example_lookup[(family, example.canonical_id, method_id)] + for example in family_examples + ] + token_subset = [ + row for row in rows if row.family == family and row.method_id == method_id + ] + family_summaries.append( + { + "family": family, + "example_count": len(family_examples), + "token_count": len(token_subset), + "example_macro": _mean_summaries(example_metrics), + "token_micro": _metric_summary(token_subset), + } + ) + family_macro = _mean_summaries( + [cast(Mapping[str, float | int], item["example_macro"]) for item in family_summaries] + ) + method_summaries.append( + { + "method_id": method_id, + "by_family": family_summaries, + "family_macro": family_macro, + "token_micro_diagnostic": _metric_summary( + [row for row in rows if row.method_id == method_id] + ), + } + ) + return per_example, method_summaries + + +def _nearest_rank_quantile(values: np.ndarray, probability: float) -> float: + if values.ndim != 1 or values.size == 0 or not np.isfinite(values).all(): + raise ValueError("quantile values must be a non-empty finite vector") + if not 0.0 <= probability <= 1.0: + raise ValueError("quantile probability must be in [0, 1]") + ordered = np.sort(values, kind="stable") + index = max(0, math.ceil(probability * ordered.size) - 1) + return float(ordered[index]) + + +def stratified_bootstrap_upper_bound( + differences_by_family: Mapping[str, Sequence[float]], +) -> dict[str, float | int | str]: + """Return the frozen family-stratified one-sided 95% upper bound.""" + + if tuple(differences_by_family) != STAGE_A_FAMILY_ORDER: + raise ValueError("bootstrap families must use the exact frozen order") + arrays: list[np.ndarray] = [] + for family in STAGE_A_FAMILY_ORDER: + raw_values = differences_by_family[family] + if len(raw_values) != STAGE_A_EXAMPLES_PER_FAMILY: + raise ValueError("each Stage-A bootstrap family must contain exactly four examples") + values = np.asarray(raw_values, dtype=np.float64) + if values.shape != (STAGE_A_EXAMPLES_PER_FAMILY,) or not np.isfinite(values).all(): + raise ValueError("Stage-A bootstrap differences must be finite scalar values") + arrays.append(values) + + generator = np.random.Generator(np.random.PCG64(BOOTSTRAP_SEED)) + sampled_family_means: list[np.ndarray] = [] + for values in arrays: + indices = generator.integers( + 0, + STAGE_A_EXAMPLES_PER_FAMILY, + size=(BOOTSTRAP_SAMPLES, STAGE_A_EXAMPLES_PER_FAMILY), + ) + sampled_family_means.append(values[indices].mean(axis=1, dtype=np.float64)) + sampled_macro = np.stack(sampled_family_means, axis=0).mean(axis=0, dtype=np.float64) + family_point_means = [_mean(list(values)) for values in arrays] + return { + "bootstrap_samples": BOOTSTRAP_SAMPLES, + "seed": BOOTSTRAP_SEED, + "rng_algorithm": BOOTSTRAP_RNG_ALGORITHM, + "quantile_probability": BOOTSTRAP_UPPER_QUANTILE, + "quantile_algorithm": BOOTSTRAP_QUANTILE_ALGORITHM, + "family_equal_point_estimate": _mean(family_point_means), + "one_sided_95_upper_bound": _nearest_rank_quantile(sampled_macro, BOOTSTRAP_UPPER_QUANTILE), + } + + +def _summary_lookup( + method_summaries: Sequence[Mapping[str, object]], +) -> dict[str, Mapping[str, object]]: + return {str(summary["method_id"]): summary for summary in method_summaries} + + +def _family_lookup(summary: Mapping[str, object]) -> dict[str, Mapping[str, object]]: + raw = cast(Sequence[Mapping[str, object]], summary["by_family"]) + return {str(item["family"]): item for item in raw} + + +def _build_decision( + per_example: Sequence[Mapping[str, object]], + method_summaries: Sequence[Mapping[str, object]], +) -> dict[str, object]: + example_metric_lookup = { + (str(item["family"]), str(item["canonical_id"]), str(item["method_id"])): cast( + Mapping[str, float | int], item["metrics"] + ) + for item in per_example + } + summaries = _summary_lookup(method_summaries) + candidate_summary = summaries[STATIC_Q468_PRIMARY_METHOD] + candidate_families = _family_lookup(candidate_summary) + + comparator_decisions: list[dict[str, object]] = [] + + def family_macro_top1(method_id: str) -> Fraction: + family_values: list[Fraction] = [] + for family in STAGE_A_FAMILY_ORDER: + example_values: list[Fraction] = [] + for item in per_example: + if item["family"] != family or item["method_id"] != method_id: + continue + metrics = cast(Mapping[str, float | int], item["metrics"]) + example_values.append( + Fraction( + int(metrics["top1_agreement_count"]), + int(metrics["token_count"]), + ) + ) + family_values.append(sum(example_values, start=Fraction()) / len(example_values)) + return sum(family_values, start=Fraction()) / len(family_values) + + candidate_top1_fraction = family_macro_top1(STATIC_Q468_PRIMARY_METHOD) + for comparator_method in DECISION_COMPARATOR_ORDER: + differences_by_family: dict[str, list[float]] = {} + family_points: list[dict[str, object]] = [] + for family in STAGE_A_FAMILY_ORDER: + candidate_rows = [ + item + for item in per_example + if item["family"] == family and item["method_id"] == STATIC_Q468_PRIMARY_METHOD + ] + differences = [ + float( + example_metric_lookup[ + (family, str(candidate_row["canonical_id"]), STATIC_Q468_PRIMARY_METHOD) + ]["excess_nll"] + ) + - float( + example_metric_lookup[ + (family, str(candidate_row["canonical_id"]), comparator_method) + ]["excess_nll"] + ) + for candidate_row in candidate_rows + ] + differences_by_family[family] = differences + family_points.append( + { + "family": family, + "candidate_minus_comparator_excess_nll": _mean(differences), + } + ) + bootstrap = stratified_bootstrap_upper_bound(differences_by_family) + candidate_top1 = float(candidate_top1_fraction) + comparator_top1_fraction = family_macro_top1(comparator_method) + comparator_top1 = float(comparator_top1_fraction) + top1_trail_fraction = comparator_top1_fraction - candidate_top1_fraction + top1_trail = float(top1_trail_fraction) + checks = { + "one_sided_95_upper_bound_at_most_0_010": ( + float(bootstrap["one_sided_95_upper_bound"]) <= MAX_ONE_SIDED_95_UPPER_EXCESS_NLL + ), + "every_family_point_at_most_0_015": all( + float(item["candidate_minus_comparator_excess_nll"]) <= MAX_FAMILY_POINT_EXCESS_NLL + for item in family_points + ), + "family_macro_top1_trail_at_most_0_005": (top1_trail_fraction <= Fraction(5, 1_000)), + } + comparator_decisions.append( + { + "comparator_method": comparator_method, + "difference_direction": "candidate_excess_nll_minus_comparator_excess_nll", + "family_points": family_points, + "paired_family_stratified_bootstrap": bootstrap, + "candidate_family_macro_top1_agreement": candidate_top1, + "comparator_family_macro_top1_agreement": comparator_top1, + "candidate_top1_trail": top1_trail, + "candidate_top1_trail_exact_fraction": ( + f"{top1_trail_fraction.numerator}/{top1_trail_fraction.denominator}" + ), + "checks": checks, + "passed": all(checks.values()), + } + ) + + q48_families = _family_lookup(summaries[STATIC_Q48_COMPARATOR_METHOD]) + q48_points: list[dict[str, object]] = [] + for family in STAGE_A_FAMILY_ORDER: + candidate_excess = float( + cast(Mapping[str, float], candidate_families[family]["example_macro"])["excess_nll"] + ) + q48_excess = float( + cast(Mapping[str, float], q48_families[family]["example_macro"])["excess_nll"] + ) + q48_points.append( + { + "family": family, + "candidate_excess_nll": candidate_excess, + "q48_excess_nll": q48_excess, + "q48_minus_candidate_excess_nll": q48_excess - candidate_excess, + "candidate_strictly_lower": candidate_excess < q48_excess, + } + ) + q48_passed = all(item["candidate_strictly_lower"] is True for item in q48_points) + return { + "candidate_method": STATIC_Q468_PRIMARY_METHOD, + "noninferiority_comparators": comparator_decisions, + "q48_every_family_strict_superiority": { + "comparator_method": STATIC_Q48_COMPARATOR_METHOD, + "family_points": q48_points, + "equality_fails": True, + "passed": q48_passed, + }, + "excluded_from_decision": { + STATIC_Q468_ABLATION_METHOD: "selection-step-matched diagnostic only", + UNIFORM_RHT_Q4_METHOD: "descriptive anchor only", + UNIFORM_RHT_Q8_METHOD: "descriptive anchor only", + FP32_METHOD: "matched reference trajectory", + }, + "stage_a_passed": all(item["passed"] is True for item in comparator_decisions) + and q48_passed, + } + + +def _example_payload(example: StageAExample) -> dict[str, object]: + return { + "family": example.family, + "canonical_id": example.canonical_id, + "selection_rank": example.selection_rank, + "continuation_token_count": example.continuation_token_count, + "identity_record_sha256": example.identity_record_sha256, + } + + +def _row_payload(row: StageATokenRow) -> dict[str, object]: + return { + "family": row.family, + "canonical_id": row.canonical_id, + "selection_rank": row.selection_rank, + "identity_record_sha256": row.identity_record_sha256, + "method_id": row.method_id, + "transition_index": row.transition_index, + "reference_nll": row.reference_nll, + "method_nll": row.method_nll, + "kl": row.kl, + "top1_agreement": row.top1_agreement, + } + + +def build_stage_a_evidence_artifact( + examples: Sequence[StageAExample], + token_rows: Sequence[StageATokenRow], + *, + stage_a_identity_file_sha256: str, + stage_a_calibration_binding_file_sha256: str, +) -> bytes: + """Build the canonical Stage-A evidence artifact from the complete row grid.""" + + identity_hash = _require_sha256( + stage_a_identity_file_sha256, + name="stage_a_identity_file_sha256", + ) + binding_hash = _require_sha256( + stage_a_calibration_binding_file_sha256, + name="stage_a_calibration_binding_file_sha256", + ) + normalized_examples = _normalized_examples(examples) + normalized_rows = _normalized_rows(normalized_examples, token_rows) + per_example, method_summaries = _build_summaries(normalized_examples, normalized_rows) + evidence: dict[str, object] = { + "artifact_kind": STAGE_A_ARTIFACT_KIND, + "artifact_revision": STAGE_A_ARTIFACT_REVISION, + "schema_version": STAGE_A_SCHEMA_VERSION, + "profile": STAGE_A_PROFILE, + "claim_boundary": CLAIM_BOUNDARY, + "dependencies": { + "stage_a_identity_file_sha256": identity_hash, + "stage_a_calibration_binding_file_sha256": binding_hash, + }, + "metric_contract": { + "family_order": list(STAGE_A_FAMILY_ORDER), + "examples_per_family": STAGE_A_EXAMPLES_PER_FAMILY, + "method_order": list(STAGE_A_METHOD_ORDER), + "cache_exposed_transitions": "m-1 for an identity-bound continuation of m>=2 tokens", + "excess_nll": "mean(method_nll-reference_nll) over cache-exposed transitions", + "example_macro": "equal-weight arithmetic mean of examples within one family", + "family_macro": ( + "equal-weight arithmetic mean of PG19, RULER, and HumanEval+ family means" + ), + "token_micro": "all cache-exposed transitions equally weighted; diagnostic only", + "cvar95_kl": ( + "FP32 mean of largest ceil(0.05*N) finite token KL values, " + "matching fidelity_summary" + ), + "bootstrap_seed": BOOTSTRAP_SEED, + "bootstrap_samples": BOOTSTRAP_SAMPLES, + "bootstrap_rng_algorithm": BOOTSTRAP_RNG_ALGORITHM, + "bootstrap_quantile_algorithm": BOOTSTRAP_QUANTILE_ALGORITHM, + }, + "method_roles": { + "fixed_order": list(STAGE_A_METHOD_ORDER), + "candidate": STATIC_Q468_PRIMARY_METHOD, + "decision_comparators": list(DECISION_COMPARATOR_ORDER), + "strict_family_superiority_comparator": STATIC_Q48_COMPARATOR_METHOD, + "diagnostic_only": [STATIC_Q468_ABLATION_METHOD], + "descriptive_only": [UNIFORM_RHT_Q4_METHOD, UNIFORM_RHT_Q8_METHOD], + "reference": FP32_METHOD, + }, + "examples": [_example_payload(example) for example in normalized_examples], + "token_rows": [_row_payload(row) for row in normalized_rows], + "example_summaries": per_example, + "method_summaries": method_summaries, + "decision": _build_decision(per_example, method_summaries), + } + canonical_hash = _sha256_bytes(canonical_json_bytes(evidence)) + return canonical_json_bytes( + { + "artifact_kind": STAGE_A_ARTIFACT_KIND, + "schema_version": STAGE_A_SCHEMA_VERSION, + "evidence": evidence, + "canonical_evidence_sha256": canonical_hash, + } + ) + + +def _strict_json(raw: bytes) -> dict[str, Any]: + def pairs_hook(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"Stage-A artifact contains duplicate key {key!r}") + result[key] = value + return result + + def reject_constant(value: str) -> None: + raise ValueError(f"non-finite JSON constant is forbidden: {value}") + + try: + value = json.loads( + raw.decode("utf-8"), + object_pairs_hook=pairs_hook, + parse_constant=reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError("Stage-A artifact must be strict UTF-8 JSON") from error + if not isinstance(value, dict): + raise ValueError("Stage-A artifact root must be a JSON object") + return value + + +def _example_from_payload(value: object, *, index: int) -> StageAExample: + if not isinstance(value, Mapping): + raise ValueError(f"examples[{index}] must be an object") + _require_exact_fields(value, _EXAMPLE_FIELDS, name=f"examples[{index}]") + return StageAExample( + family=cast(StageAFamily, value["family"]), + canonical_id=cast(str, value["canonical_id"]), + selection_rank=cast(int, value["selection_rank"]), + continuation_token_count=cast(int, value["continuation_token_count"]), + identity_record_sha256=cast(str, value["identity_record_sha256"]), + ) + + +def _row_from_payload(value: object, *, index: int) -> StageATokenRow: + if not isinstance(value, Mapping): + raise ValueError(f"token_rows[{index}] must be an object") + _require_exact_fields(value, _TOKEN_ROW_FIELDS, name=f"token_rows[{index}]") + return StageATokenRow( + family=cast(StageAFamily, value["family"]), + canonical_id=cast(str, value["canonical_id"]), + selection_rank=cast(int, value["selection_rank"]), + identity_record_sha256=cast(str, value["identity_record_sha256"]), + method_id=cast(str, value["method_id"]), + transition_index=cast(int, value["transition_index"]), + reference_nll=cast(float, value["reference_nll"]), + method_nll=cast(float, value["method_nll"]), + kl=cast(float, value["kl"]), + top1_agreement=cast(bool, value["top1_agreement"]), + ) + + +def deserialize_stage_a_evidence_artifact( + data: bytes, + *, + expected_file_sha256: str | None = None, + expected_canonical_evidence_sha256: str | None = None, + expected_stage_a_identity_file_sha256: str | None = None, + expected_stage_a_calibration_binding_file_sha256: str | None = None, +) -> StageAEvidenceArtifact: + """Strictly verify by reconstructing every summary, bootstrap, and gate.""" + + if not isinstance(data, bytes): + raise TypeError("Stage-A artifact data must be bytes") + file_hash = _sha256_bytes(data) + if expected_file_sha256 is not None and file_hash != _require_sha256( + expected_file_sha256, name="expected_file_sha256" + ): + raise ValueError("Stage-A artifact file SHA-256 differs from the expected digest") + root = _strict_json(data) + _require_exact_fields(root, _OUTER_FIELDS, name="Stage-A artifact") + if ( + root["artifact_kind"] != STAGE_A_ARTIFACT_KIND + or root["schema_version"] != STAGE_A_SCHEMA_VERSION + ): + raise ValueError("Stage-A artifact kind or schema version drifted") + evidence = root["evidence"] + if not isinstance(evidence, Mapping): + raise ValueError("Stage-A evidence must be an object") + _require_exact_fields(evidence, _EVIDENCE_FIELDS, name="Stage-A evidence") + if ( + evidence["artifact_kind"] != STAGE_A_ARTIFACT_KIND + or evidence["schema_version"] != STAGE_A_SCHEMA_VERSION + or evidence["artifact_revision"] != STAGE_A_ARTIFACT_REVISION + or evidence["profile"] != STAGE_A_PROFILE + or evidence["claim_boundary"] != CLAIM_BOUNDARY + ): + raise ValueError("Stage-A evidence identity drifted") + recorded_canonical = _require_sha256( + root["canonical_evidence_sha256"], name="canonical_evidence_sha256" + ) + computed_canonical = _sha256_bytes(canonical_json_bytes(dict(evidence))) + if recorded_canonical != computed_canonical: + raise ValueError("Stage-A canonical evidence SHA-256 differs from its contents") + if expected_canonical_evidence_sha256 is not None and computed_canonical != _require_sha256( + expected_canonical_evidence_sha256, + name="expected_canonical_evidence_sha256", + ): + raise ValueError("Stage-A canonical evidence SHA-256 differs from the expected digest") + + dependencies = evidence["dependencies"] + if not isinstance(dependencies, Mapping): + raise ValueError("Stage-A dependencies must be an object") + _require_exact_fields(dependencies, _DEPENDENCY_FIELDS, name="Stage-A dependencies") + identity_hash = _require_sha256( + dependencies["stage_a_identity_file_sha256"], + name="stage_a_identity_file_sha256", + ) + binding_hash = _require_sha256( + dependencies["stage_a_calibration_binding_file_sha256"], + name="stage_a_calibration_binding_file_sha256", + ) + if expected_stage_a_identity_file_sha256 is not None and identity_hash != _require_sha256( + expected_stage_a_identity_file_sha256, + name="expected_stage_a_identity_file_sha256", + ): + raise ValueError("Stage-A identity dependency differs from the expected digest") + if ( + expected_stage_a_calibration_binding_file_sha256 is not None + and binding_hash + != _require_sha256( + expected_stage_a_calibration_binding_file_sha256, + name="expected_stage_a_calibration_binding_file_sha256", + ) + ): + raise ValueError("Stage-A calibration binding differs from the expected digest") + + raw_examples = evidence["examples"] + raw_rows = evidence["token_rows"] + if not isinstance(raw_examples, list) or not isinstance(raw_rows, list): + raise ValueError("Stage-A examples and token_rows must be JSON arrays") + examples = tuple( + _example_from_payload(value, index=index) for index, value in enumerate(raw_examples) + ) + rows = tuple(_row_from_payload(value, index=index) for index, value in enumerate(raw_rows)) + rebuilt = build_stage_a_evidence_artifact( + examples, + rows, + stage_a_identity_file_sha256=identity_hash, + stage_a_calibration_binding_file_sha256=binding_hash, + ) + if rebuilt != data: + raise ValueError("Stage-A artifact differs from its deterministic reconstruction") + decision = evidence["decision"] + if not isinstance(decision, Mapping) or not isinstance(decision.get("stage_a_passed"), bool): + raise ValueError("Stage-A decision must contain a boolean stage_a_passed result") + return StageAEvidenceArtifact( + evidence=cast(Mapping[str, Any], _deep_freeze_json(dict(evidence))), + canonical_evidence_sha256=computed_canonical, + file_sha256=file_hash, + serialized_bytes=bytes(data), + _verified_passed=cast(bool, decision["stage_a_passed"]), + ) + + +def verify_stage_a_evidence_artifact( + data: bytes, + **expected: str | None, +) -> StageAEvidenceArtifact: + """Alias for the strict reconstructing deserializer.""" + + return deserialize_stage_a_evidence_artifact(data, **expected) + + +__all__ = [ + "BOOTSTRAP_QUANTILE_ALGORITHM", + "BOOTSTRAP_RNG_ALGORITHM", + "BOOTSTRAP_SAMPLES", + "BOOTSTRAP_SEED", + "BOOTSTRAP_UPPER_QUANTILE", + "CLAIM_BOUNDARY", + "DECISION_COMPARATOR_ORDER", + "FP32_METHOD", + "MAX_FAMILY_MACRO_TOP1_TRAIL", + "MAX_FAMILY_POINT_EXCESS_NLL", + "MAX_ONE_SIDED_95_UPPER_EXCESS_NLL", + "STAGE_A_ARTIFACT_KIND", + "STAGE_A_ARTIFACT_REVISION", + "STAGE_A_EXAMPLE_COUNT", + "STAGE_A_EXAMPLES_PER_FAMILY", + "STAGE_A_FAMILY_ORDER", + "STAGE_A_METHOD_ORDER", + "STAGE_A_PROFILE", + "STAGE_A_SCHEMA_VERSION", + "TAIL_KL_FRACTION", + "UNIFORM_RHT_Q4_METHOD", + "UNIFORM_RHT_Q8_METHOD", + "StageAEvidenceArtifact", + "StageAExample", + "StageAFamily", + "StageATokenRow", + "build_stage_a_evidence_artifact", + "deserialize_stage_a_evidence_artifact", + "stratified_bootstrap_upper_bound", + "verify_stage_a_evidence_artifact", +] diff --git a/src/recurquant/static_q468.py b/src/recurquant/static_q468.py index a8dcba8..908250d 100644 --- a/src/recurquant/static_q468.py +++ b/src/recurquant/static_q468.py @@ -59,6 +59,10 @@ STATIC_Q468_PRIMARY_METHOD = "rht_q468_static_k29334" STATIC_Q468_ABLATION_METHOD = "rht_q468_static_k27030" +STATIC_Q468_MSE_METHOD = "rht_q468_static_mse_k29334" +STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD = "rht_q468_static_diag_empirical_fisher_h1_k29334" +STATIC_Q468_UNIFORM_Q4_METHOD = "rht_q468_uniform_q4" +STATIC_Q468_UNIFORM_Q8_METHOD = "rht_q468_uniform_q8" STATIC_Q48_COMPARATOR_METHOD = "rht_q48_static_p14739" PRIMARY_MODEL_ID = "Qwen/Qwen3.5-0.8B-Base" @@ -69,6 +73,8 @@ FROZEN_STATIC_Q468_PRIMARY_STEPS = 29_334 FROZEN_STATIC_Q468_ABLATION_STEPS = 27_030 +FROZEN_STATIC_Q468_UNIFORM_Q4_STEPS = 0 +FROZEN_STATIC_Q468_UNIFORM_Q8_STEPS = 73_728 FROZEN_STATIC_Q48_PROMOTIONS = 14_739 FROZEN_STATELEASE_RESIDENT_BYTES = 3_454_664 FROZEN_RECURRENT_LAYER_INDICES = ( @@ -386,7 +392,7 @@ def static_q48_byte_ledger( def frozen_static_byte_accounting() -> dict[str, dict[str, object]]: - """Return the three frozen Experiment 013 byte ledgers.""" + """Return the seven frozen Experiment 013 static-policy byte ledgers.""" geometry = FROZEN_QWEN35_STATIC_Q468_GEOMETRY return { @@ -400,6 +406,26 @@ def frozen_static_byte_accounting() -> dict[str, dict[str, object]]: FROZEN_STATIC_Q468_ABLATION_STEPS, method_id=STATIC_Q468_ABLATION_METHOD, ).evidence_dict(), + STATIC_Q468_MSE_METHOD: static_q468_byte_ledger( + geometry, + FROZEN_STATIC_Q468_PRIMARY_STEPS, + method_id=STATIC_Q468_MSE_METHOD, + ).evidence_dict(), + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD: static_q468_byte_ledger( + geometry, + FROZEN_STATIC_Q468_PRIMARY_STEPS, + method_id=STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + ).evidence_dict(), + STATIC_Q468_UNIFORM_Q4_METHOD: static_q468_byte_ledger( + geometry, + FROZEN_STATIC_Q468_UNIFORM_Q4_STEPS, + method_id=STATIC_Q468_UNIFORM_Q4_METHOD, + ).evidence_dict(), + STATIC_Q468_UNIFORM_Q8_METHOD: static_q468_byte_ledger( + geometry, + FROZEN_STATIC_Q468_UNIFORM_Q8_STEPS, + method_id=STATIC_Q468_UNIFORM_Q8_METHOD, + ).evidence_dict(), STATIC_Q48_COMPARATOR_METHOD: static_q48_byte_ledger( geometry, FROZEN_STATIC_Q48_PROMOTIONS, @@ -528,6 +554,10 @@ def __post_init__(self) -> None: frozen_steps = { STATIC_Q468_PRIMARY_METHOD: FROZEN_STATIC_Q468_PRIMARY_STEPS, STATIC_Q468_ABLATION_METHOD: FROZEN_STATIC_Q468_ABLATION_STEPS, + STATIC_Q468_MSE_METHOD: FROZEN_STATIC_Q468_PRIMARY_STEPS, + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD: (FROZEN_STATIC_Q468_PRIMARY_STEPS), + STATIC_Q468_UNIFORM_Q4_METHOD: FROZEN_STATIC_Q468_UNIFORM_Q4_STEPS, + STATIC_Q468_UNIFORM_Q8_METHOD: FROZEN_STATIC_Q468_UNIFORM_Q8_STEPS, }.get(self.method_id) if frozen_steps is not None: if self.geometry != FROZEN_QWEN35_STATIC_Q468_GEOMETRY: @@ -542,6 +572,24 @@ def __post_init__(self) -> None: or self.transformers_version != FROZEN_TRANSFORMERS_VERSION ): raise ValueError("reserved static Q468 method requires the frozen model identity") + if self.method_id in { + STATIC_Q468_PRIMARY_METHOD, + STATIC_Q468_MSE_METHOD, + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + }: + ledger = static_q468_byte_ledger( + self.geometry, + steps, + method_id=self.method_id, + ) + if ( + ledger.resident_bytes != FROZEN_STATELEASE_RESIDENT_BYTES + or ledger.target_resident_bytes != FROZEN_STATELEASE_RESIDENT_BYTES + or not ledger.exact_budget_eligible + ): + raise ValueError( + "reserved exact-budget static Q468 method has the wrong resident ledger" + ) if self.rht_seed != RHT_SEED: raise ValueError(f"rht_seed must equal the codec seed {RHT_SEED}") @@ -650,19 +698,13 @@ def code_map_sha256(self) -> str: digest.update(b"recurquant.static-q468-code-map.v1\0") digest.update(bytes.fromhex(self.geometry.geometry_sha256)) digest.update(self.marginal_steps.to_bytes(8, "little", signed=False)) - digest.update( - self.packed_precision_codes.detach().to("cpu").contiguous().numpy().tobytes() - ) + digest.update(self.packed_precision_codes.detach().to("cpu").contiguous().numpy().tobytes()) return digest.hexdigest() @property def pool_offsets_sha256(self) -> str: offsets = ( - self.pool_offsets.detach() - .to("cpu") - .contiguous() - .numpy() - .astype(" None: if self.method_id in { STATIC_Q468_PRIMARY_METHOD, STATIC_Q468_ABLATION_METHOD, + STATIC_Q468_MSE_METHOD, + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + STATIC_Q468_UNIFORM_Q4_METHOD, + STATIC_Q468_UNIFORM_Q8_METHOD, }: raise ValueError("reserved static Q468 method cannot identify a Q48 policy") if self.method_id == STATIC_Q48_COMPARATOR_METHOD: @@ -1255,19 +1301,13 @@ def mask_sha256(self) -> str: digest.update(b"recurquant.static-q48-mask.v1\0") digest.update(bytes.fromhex(self.geometry.geometry_sha256)) digest.update(self.promoted_rows.to_bytes(8, "little", signed=False)) - digest.update( - self.packed_precision_mask.detach().to("cpu").contiguous().numpy().tobytes() - ) + digest.update(self.packed_precision_mask.detach().to("cpu").contiguous().numpy().tobytes()) return digest.hexdigest() @property def pool_offsets_sha256(self) -> str: offsets = ( - self.pool_offsets.detach() - .to("cpu") - .contiguous() - .numpy() - .astype(" None: raise ValueError("low_payload contains the reserved symmetric INT4 code -8") if self.high_payload.numel() and (self.high_payload == -128).any().item(): raise ValueError("high_payload contains the reserved symmetric INT8 code -128") - if self.scales.dtype != torch.float16 or tuple(self.scales.shape) != ( - geometry.total_rows, - ): + if self.scales.dtype != torch.float16 or tuple(self.scales.shape) != (geometry.total_rows,): raise TypeError( f"scales must have shape {(geometry.total_rows,)} and dtype torch.float16" ) diff --git a/src/recurquant/static_q468_cache.py b/src/recurquant/static_q468_cache.py index 068204c..e47a254 100644 --- a/src/recurquant/static_q468_cache.py +++ b/src/recurquant/static_q468_cache.py @@ -30,7 +30,11 @@ FROZEN_QWEN35_STATIC_Q468_GEOMETRY, STATIC_Q48_COMPARATOR_METHOD, STATIC_Q468_ABLATION_METHOD, + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + STATIC_Q468_MSE_METHOD, STATIC_Q468_PRIMARY_METHOD, + STATIC_Q468_UNIFORM_Q4_METHOD, + STATIC_Q468_UNIFORM_Q8_METHOD, StaticPackedRhtQ48State, StaticPackedRhtQ468State, StaticRhtByteLedger, @@ -55,11 +59,18 @@ StaticPackedRhtState: TypeAlias = StaticPackedRhtQ468State | StaticPackedRhtQ48State StaticPolicyKind: TypeAlias = Literal["q468", "q48"] -DYNAMIC_Q468_ORACLE_METHOD = "rht_q468_dynamic_k27030" +DYNAMIC_Q468_BASELINE_METHOD = "rht_q468_dynamic_k27030" +# Compatibility alias for callers using the pre-correction public name. The +# method is a dynamic comparison baseline, not an oracle for held-out quality. +DYNAMIC_Q468_ORACLE_METHOD = DYNAMIC_Q468_BASELINE_METHOD FROZEN_STATIC_RUNTIME_METHODS = frozenset( ( STATIC_Q468_PRIMARY_METHOD, STATIC_Q468_ABLATION_METHOD, + STATIC_Q468_MSE_METHOD, + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + STATIC_Q468_UNIFORM_Q4_METHOD, + STATIC_Q468_UNIFORM_Q8_METHOD, STATIC_Q48_COMPARATOR_METHOD, ) ) @@ -484,7 +495,7 @@ def create_qwen35_static_rht_cache( if not isinstance(policy, (StaticRhtQ468Policy, StaticRhtQ48Policy)): raise TypeError("policy must be a StaticRhtQ468Policy or StaticRhtQ48Policy") if policy.method_id not in FROZEN_STATIC_RUNTIME_METHODS: - raise ValueError("public Experiment 013 runtime accepts only the three frozen methods") + raise ValueError("public Experiment 013 runtime accepts only the seven frozen methods") if policy.geometry != FROZEN_QWEN35_STATIC_Q468_GEOMETRY: raise ValueError("frozen Experiment 013 method requires the frozen Qwen3.5 geometry") return StaticRhtQwen35Cache( @@ -495,27 +506,44 @@ def create_qwen35_static_rht_cache( ) -def create_qwen35_dynamic_q468_oracle_cache( +def create_qwen35_dynamic_q468_baseline_cache( model_or_config: object, *, record_evidence: bool = False, ) -> EqualByteQwen35Cache: - """Create the named exact dynamic K27030 quality oracle. + """Create the named dynamic K27030 comparison baseline. This is the existing global RHT Q4/Q6/Q8 allocator with its frozen 27,030 - marginal steps and declared persistent query-energy EMA. The wrapper adds - an experiment method identity; it does not change the codec. + marginal steps and declared persistent query-energy EMA. The factory adds + an experiment method identity; it does not change the codec or claim an + oracle for held-out model quality. """ _validate_transformers_compatibility() config = _validated_text_config(model_or_config) if FROZEN_QWEN35_EQUAL_BYTE_LAYOUT.multibit_marginal_steps != 27_030: - raise RuntimeError("dynamic Q468 oracle layout no longer has K27030") + raise RuntimeError("dynamic Q468 baseline layout no longer has K27030") cache = create_qwen35_equal_byte_cache( config, codec=RHT_Q4_Q6_Q8, layout=FROZEN_QWEN35_EQUAL_BYTE_LAYOUT, record_evidence=record_evidence, ) - cache.method_id = DYNAMIC_Q468_ORACLE_METHOD # type: ignore[attr-defined] + cache.method_id = DYNAMIC_Q468_BASELINE_METHOD # type: ignore[attr-defined] return cache + + +def create_qwen35_dynamic_q468_oracle_cache( + model_or_config: object, + *, + record_evidence: bool = False, +) -> EqualByteQwen35Cache: + """Compatibility wrapper for the former dynamic-Q468 ``oracle`` name. + + New code should use :func:`create_qwen35_dynamic_q468_baseline_cache`. + """ + + return create_qwen35_dynamic_q468_baseline_cache( + model_or_config, + record_evidence=record_evidence, + ) diff --git a/src/recurquant/static_q468_calibration.py b/src/recurquant/static_q468_calibration.py index b5ccfcb..c22b248 100644 --- a/src/recurquant/static_q468_calibration.py +++ b/src/recurquant/static_q468_calibration.py @@ -27,6 +27,7 @@ import unicodedata from collections.abc import Mapping, Sequence from dataclasses import dataclass +from types import MappingProxyType from typing import Any, Literal, TypeAlias, cast import numpy as np @@ -36,6 +37,8 @@ from .metrics import spearman_correlation from .multibit_policy import allocate_exact_multibit_codes_fast from .multibit_quantization import _pack_precision_codes +from .quantization import QuantizationSpec, quantize_dequantize +from .rht import RHT_SEED, right_rht_encode from .static_q468 import ( FROZEN_QWEN35_STATIC_Q468_GEOMETRY, FROZEN_STATIC_Q468_ABLATION_STEPS, @@ -53,6 +56,10 @@ "question_answering", ] SplitHalf: TypeAlias = Literal["a", "b"] +UnweightedSelectorProfile: TypeAlias = Literal[ + "rht_q468_static_mse_k29334", + "rht_q468_static_diag_empirical_fisher_h1_k29334", +] FROZEN_ANCHOR_COUNT = 16 CALIBRATION_FAMILY_ORDER = ("mbpp", "pg19", "ruler") @@ -86,14 +93,64 @@ GENERIC_REDUCTION_PROFILE = "generic-anchor-row-reduction-v1" FROZEN_REDUCTION_PROFILE = "experiment-013-qwen35-0.8b-anchor-reduction-v1" +FROZEN_UNWEIGHTED_MSE_PROFILE = "rht_q468_static_mse_k29334" +FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE = "rht_q468_static_diag_empirical_fisher_h1_k29334" +FROZEN_FISHER_HORIZON = 1 FROZEN_SOURCE_AXIS_ORDER = ("anchor", "layer", "head", "key_row") _SOURCE_TENSOR_NAMES = ("query_energy", "q4_mse", "q6_mse", "q8_mse") +COMPARATOR_SCORE_ARTIFACT_KIND = "recurquant_experiment013_static_q468_comparator_scores" +COMPARATOR_SCORE_ARTIFACT_SCHEMA_VERSION = 1 +COMPARATOR_SCORE_ARTIFACT_REVISION = "experiment-013-static-q468-comparator-scores-v1" +COMPARATOR_SCORE_ARTIFACT_PROFILE = "experiment-013-qwen35-0.8b-static-q468-comparators-frozen-v1" +FROZEN_COMPARATOR_PROFILE_ORDER: tuple[UnweightedSelectorProfile, ...] = ( + FROZEN_UNWEIGHTED_MSE_PROFILE, + FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE, +) +FROZEN_COMPARATOR_ENDPOINT_AXIS_ORDER = ( + "endpoint_position", + "layer", + "head", + "key_row", +) +FROZEN_COMPARATOR_POSITION_CONTRACTS = { + FROZEN_UNWEIGHTED_MSE_PROFILE: "A(T)=frozen_anchor_positions(T)", + FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE: ("B(T)=frozen_anchor_positions(T-2)"), +} + _SHA256_RE = re.compile(r"[0-9a-f]{64}") _SCORE_HASH_DOMAIN = b"recurquant.experiment013.sequence-scores.v1\0" _ANCHOR_INPUT_HASH_DOMAIN = b"recurquant.experiment013.anchor-inputs.v1\0" _CODE_MAP_HASH_DOMAIN = b"recurquant.static-q468-code-map.v1\0" _IDENTITY_RECORD_HASH_DOMAIN = b"recurquant.experiment013.identity-record.v1\0" +_FISHER_BOUNDARY_HASH_DOMAIN = b"recurquant.experiment013.fisher-boundary.v1\0" +_FISHER_BOUNDARY_TOKEN_HASH_DOMAIN = b"recurquant.experiment013.fisher-boundary-token-sequence.v1\0" +_COMPARATOR_POSITION_HASH_DOMAIN = b"recurquant.experiment013.comparator-position-manifest.v1\0" +_COMPARATOR_ENDPOINT_INPUT_HASH_DOMAIN = b"recurquant.experiment013.comparator-endpoint-input.v1\0" +_COMPARATOR_TARGET_NLL_HASH_DOMAIN = b"recurquant.experiment013.comparator-target-nll.v1\0" +_COMPARATOR_SEQUENCE_SCORE_HASH_DOMAIN = b"recurquant.experiment013.comparator-sequence-score.v1\0" +_COMPARATOR_AGGREGATE_SCORE_HASH_DOMAIN = ( + b"recurquant.experiment013.comparator-aggregate-score.v1\0" +) +_COMPARATOR_SEQUENCE_MANIFEST_HASH_DOMAIN = ( + b"recurquant.experiment013.comparator-sequence-manifest.v1\0" +) +_COMPARATOR_POSITION_MANIFEST_HASH_DOMAIN = ( + b"recurquant.experiment013.comparator-ordered-position-manifest.v1\0" +) +_FISHER_BOUNDARY_SCHEMA = "recurquant.experiment013.fisher-boundary.v1" +_FISHER_BOUNDARY_FIELDS = frozenset( + { + "schema", + "horizon", + "boundary_positions", + "input_positions", + "target_positions", + "input_token_ids_sha256", + "target_token_ids_sha256", + "fisher_boundary_sha256", + } +) _TOKEN_SPAN_ORDER = ( "prefill_start", "prefill_stop", @@ -122,6 +179,7 @@ "tokenizer_manifest_sha256", "token_span", "anchor_manifest_sha256", + "fisher_boundary", } ) _AGGREGATION_CONTRACT = { @@ -135,6 +193,11 @@ ), "sequence_reduction": "equal-weight arithmetic mean within MBPP and PG19", } +_COMPARATOR_AGGREGATION_CONTRACT = { + **_AGGREGATION_CONTRACT, + "endpoint_reduction": "equal-weight CPU-float64 mean over frozen positions", + "profiles": list(FROZEN_COMPARATOR_PROFILE_ORDER), +} class CalibrationArtifactError(ValueError): @@ -258,6 +321,255 @@ def frozen_anchor_positions(token_count: int) -> tuple[int, ...]: return positions +def fisher_h1_boundary_positions(token_count: int) -> tuple[int, ...]: + """Return boundary positions with both causal input and target available. + + A boundary ``b`` stores ``S_b``; H=1 consumes ``x_(b+1)`` and scores the + resulting logits against ``x_(b+2)``. Therefore a length-``T`` sequence + has exactly ``T - 2`` eligible boundaries before the frozen anchor equation + is applied. + """ + + tokens = _strict_positive_int(token_count, name="token_count") + if tokens < 3: + raise ValueError("H=1 Fisher calibration requires at least three tokens") + return frozen_anchor_positions(tokens - 2) + + +def _q468_endpoint_specs(geometry: StaticRhtQ468Geometry) -> tuple[QuantizationSpec, ...]: + common = { + "group_size": geometry.value_width, + "scale_bits": 16, + "flatten_last_dims": 1, + "rounding": "nearest", + "seed": RHT_SEED, + } + return tuple(QuantizationSpec(bits=bits, **common) for bits in (4, 6, 8)) + + +def _endpoint_state_tensor( + value: object, + *, + name: str, + geometry: StaticRhtQ468Geometry, + device: torch.device | None = None, +) -> torch.Tensor: + if not isinstance(value, torch.Tensor): + raise TypeError(f"{name} must be a torch.Tensor") + expected = ( + len(geometry.layer_indices), + geometry.heads, + geometry.key_rows, + geometry.value_width, + ) + if tuple(value.shape) != expected: + raise ValueError(f"{name} must have shape {expected}") + if value.device.type == "meta" or not value.is_floating_point(): + raise TypeError(f"{name} must be a materialized floating-point tensor") + if device is not None and value.device != device: + raise ValueError(f"{name} must be on {device}") + normalized = value.detach().to(torch.float32) + if not torch.isfinite(normalized).all().item(): + raise ValueError(f"{name} must contain only finite values") + return normalized + + +def compute_rht_unweighted_mse_endpoints( + recurrent_state: torch.Tensor, + *, + geometry: StaticRhtQ468Geometry = FROZEN_QWEN35_STATIC_Q468_GEOMETRY, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Compute equal-weight per-row RHT Q4/Q6/Q8 MSE without query proxies.""" + + if not isinstance(geometry, StaticRhtQ468Geometry): + raise TypeError("geometry must be a StaticRhtQ468Geometry") + state = _endpoint_state_tensor( + recurrent_state, + name="recurrent_state", + geometry=geometry, + ) + per_bit: list[list[torch.Tensor]] = [[], [], []] + specifications = _q468_endpoint_specs(geometry) + with torch.no_grad(): + for local_index, layer_index in enumerate(geometry.layer_indices): + encoded = right_rht_encode( + state[local_index].unsqueeze(0), + layer_index=layer_index, + expected_heads=geometry.heads, + output_dtype=torch.float32, + ) + for destination, specification in zip(per_bit, specifications, strict=True): + restored = quantize_dequantize(encoded, specification).tensor + error_fp64 = ( + (restored - encoded) + .detach() + .to( + device="cpu", + dtype=torch.float64, + ) + ) + destination.append(error_fp64.square().mean(dim=-1).squeeze(0).contiguous()) + return cast( + tuple[torch.Tensor, torch.Tensor, torch.Tensor], + tuple(torch.stack(rows, dim=0).contiguous() for rows in per_bit), + ) + + +def compute_rht_diagonal_empirical_fisher_h1_endpoints( + recurrent_state: torch.Tensor, + state_gradient: torch.Tensor, + *, + geometry: StaticRhtQ468Geometry = FROZEN_QWEN35_STATIC_Q468_GEOMETRY, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Compute ``0.5 * sum_v((RHT grad)^2 * quantization_error^2)``. + + Both state and loss gradient are transformed by the same orthonormal right + RHT. Scores are returned as CPU-FP64 ``[layer, head, key_row]`` tensors and + intentionally use neither query-energy weighting nor proxy normalization. + """ + + if not isinstance(geometry, StaticRhtQ468Geometry): + raise TypeError("geometry must be a StaticRhtQ468Geometry") + state = _endpoint_state_tensor( + recurrent_state, + name="recurrent_state", + geometry=geometry, + ) + gradient = _endpoint_state_tensor( + state_gradient, + name="state_gradient", + geometry=geometry, + device=state.device, + ) + per_bit: list[list[torch.Tensor]] = [[], [], []] + specifications = _q468_endpoint_specs(geometry) + with torch.no_grad(): + for local_index, layer_index in enumerate(geometry.layer_indices): + encoded_state = right_rht_encode( + state[local_index].unsqueeze(0), + layer_index=layer_index, + expected_heads=geometry.heads, + output_dtype=torch.float32, + ) + encoded_gradient = right_rht_encode( + gradient[local_index].unsqueeze(0), + layer_index=layer_index, + expected_heads=geometry.heads, + output_dtype=torch.float32, + ) + gradient_fp64 = encoded_gradient.detach().to( + device="cpu", + dtype=torch.float64, + ) + squared_gradient = gradient_fp64.square() + for destination, specification in zip(per_bit, specifications, strict=True): + restored = quantize_dequantize(encoded_state, specification).tensor + error_fp64 = ( + (restored - encoded_state) + .detach() + .to( + device="cpu", + dtype=torch.float64, + ) + ) + risk = 0.5 * (squared_gradient * error_fp64.square()).sum(dim=-1) + destination.append(risk.squeeze(0).contiguous()) + return cast( + tuple[torch.Tensor, torch.Tensor, torch.Tensor], + tuple(torch.stack(rows, dim=0).contiguous() for rows in per_bit), + ) + + +@dataclass(frozen=True, slots=True) +class UnweightedEndpointBatch: + """Per-anchor Q4/Q6/Q8 endpoint scores for one static selector.""" + + selector_profile: UnweightedSelectorProfile + token_count: int + anchor_positions: tuple[int, ...] + q4_scores: torch.Tensor + q6_scores: torch.Tensor + q8_scores: torch.Tensor + + +@dataclass(frozen=True, slots=True) +class FrozenComparatorEndpointBatch: + """Identity-v5-bound endpoint scores for one frozen comparator sequence.""" + + selector_profile: UnweightedSelectorProfile + family: CalibrationFamily + config: str + ruler_category: RulerCategory | None + canonical_id: str + seed: int | None + configured_length: int | None + token_count: int + endpoint_positions: tuple[int, ...] + q4_scores: torch.Tensor + q6_scores: torch.Tensor + q8_scores: torch.Tensor + sequence_token_ids: tuple[int, ...] + identity_record: Mapping[str, object] + target_nlls: torch.Tensor | None = None + + +def reduce_unweighted_endpoint_anchors( + batch: UnweightedEndpointBatch, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Take an equal CPU-FP64 mean over anchors, with no proxy weighting.""" + + if not isinstance(batch, UnweightedEndpointBatch): + raise TypeError("batch must be an UnweightedEndpointBatch") + if batch.selector_profile == FROZEN_UNWEIGHTED_MSE_PROFILE: + expected_positions = frozen_anchor_positions(batch.token_count) + elif batch.selector_profile == FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE: + expected_positions = fisher_h1_boundary_positions(batch.token_count) + else: + raise ValueError("selector_profile is not a frozen unweighted selector") + if not isinstance(batch.anchor_positions, tuple): + raise TypeError("anchor_positions must be a tuple") + if batch.anchor_positions != expected_positions: + raise ValueError("anchor_positions differ from the selector's frozen equation") + anchors = len(expected_positions) + values = tuple( + _cpu_fp64_matrix(value, name=name, anchors=anchors) + for name, value in ( + ("q4_scores", batch.q4_scores), + ("q6_scores", batch.q6_scores), + ("q8_scores", batch.q8_scores), + ) + ) + if values[1].shape != values[0].shape or values[2].shape != values[0].shape: + raise ValueError("Q4/Q6/Q8 endpoint score shapes must match exactly") + return cast( + tuple[torch.Tensor, torch.Tensor, torch.Tensor], + tuple(value.mean(dim=0).contiguous() for value in values), + ) + + +def allocate_unweighted_endpoint_policy( + batch: UnweightedEndpointBatch, + *, + marginal_steps: int, +) -> torch.Tensor: + """Allocate an exact Q4/Q6/Q8 code map from unweighted endpoint scores.""" + + scores = reduce_unweighted_endpoint_anchors(batch) + rows = scores[0].numel() + steps = _strict_nonnegative_int(marginal_steps, name="marginal_steps") + if steps > 2 * rows: + raise ValueError("marginal_steps exceeds two steps per row") + code_map = allocate_exact_multibit_codes_fast( + *(value.reshape(1, rows) for value in scores), + marginal_steps=steps, + ).reshape(-1) + if code_map.dtype != torch.uint8 or code_map.device.type != "cpu": + raise RuntimeError("exact allocator returned a non-canonical code map") + if int(code_map.to(torch.int64).sum().item()) != steps: + raise RuntimeError("exact allocator did not satisfy the requested marginal budget") + return code_map.contiguous() + + def _cpu_fp64_matrix( value: object, *, @@ -307,6 +619,13 @@ def _tensor_bytes(value: torch.Tensor) -> bytes: return array.astype(" str: + digest = hashlib.sha256() + digest.update(domain) + digest.update(canonical_json_bytes(value)) + return digest.hexdigest() + + @dataclass(frozen=True, slots=True) class CalibrationSourceTensorContract: """Shape, axis, dtype, and reduction profile bound into score evidence.""" @@ -407,6 +726,64 @@ def sequence_token_ids_sha256(token_ids: Sequence[int]) -> str: return hashlib.sha256(_resolver_canonical_json_bytes(normalized)).hexdigest() +def _fisher_token_ids_sha256(token_ids: Sequence[int], *, role: str) -> str: + payload = {"role": role, "token_ids": list(token_ids)} + return hashlib.sha256( + _FISHER_BOUNDARY_TOKEN_HASH_DOMAIN + _resolver_canonical_json_bytes(payload) + ).hexdigest() + + +def _validate_fisher_boundary( + value: object, + *, + sequence_token_ids: tuple[int, ...], +) -> None: + if not isinstance(value, Mapping) or set(value) != _FISHER_BOUNDARY_FIELDS: + raise ValueError("fisher_boundary must contain the exact frozen H=1 fields") + if value["schema"] != _FISHER_BOUNDARY_SCHEMA: + raise ValueError("fisher_boundary schema drifted") + horizon = value["horizon"] + if isinstance(horizon, bool) or not isinstance(horizon, int) or horizon != 1: + raise ValueError("fisher_boundary horizon must equal H=1") + boundary_positions = list(fisher_h1_boundary_positions(len(sequence_token_ids))) + input_positions = [position + 1 for position in boundary_positions] + target_positions = [position + 1 for position in input_positions] + expected_positions = { + "boundary_positions": boundary_positions, + "input_positions": input_positions, + "target_positions": target_positions, + } + for name, expected in expected_positions.items(): + if not isinstance(value[name], list) or value[name] != expected: + raise ValueError(f"fisher_boundary.{name} differs from the causal H=1 contract") + for name in ( + "input_token_ids_sha256", + "target_token_ids_sha256", + "fisher_boundary_sha256", + ): + _sha256(value[name], name=f"fisher_boundary.{name}") + expected_input_hash = _fisher_token_ids_sha256( + [sequence_token_ids[position] for position in input_positions], + role="input", + ) + expected_target_hash = _fisher_token_ids_sha256( + [sequence_token_ids[position] for position in target_positions], + role="target", + ) + if value["input_token_ids_sha256"] != expected_input_hash: + raise ValueError("fisher_boundary input token-ID hash differs from exact sequence tokens") + if value["target_token_ids_sha256"] != expected_target_hash: + raise ValueError("fisher_boundary target token-ID hash differs from exact sequence tokens") + payload = { + name: value[name] for name in sorted(_FISHER_BOUNDARY_FIELDS - {"fisher_boundary_sha256"}) + } + expected_self_hash = hashlib.sha256( + _FISHER_BOUNDARY_HASH_DOMAIN + _resolver_canonical_json_bytes(payload) + ).hexdigest() + if value["fisher_boundary_sha256"] != expected_self_hash: + raise ValueError("fisher_boundary self-hash drifted") + + def _normalize_token_span(value: object, *, sequence_length: int) -> tuple[tuple[str, int], ...]: if not isinstance(value, Mapping) or set(value) != set(_TOKEN_SPAN_ORDER): raise ValueError("token_span must contain the exact six capture span fields") @@ -633,7 +1010,7 @@ class _ValidatedIdentityLineage: def _validate_frozen_identity_lineage( - batch: AnchorDistortionBatch, + batch: AnchorDistortionBatch | FrozenComparatorEndpointBatch, ) -> _ValidatedIdentityLineage: token_ids = batch.sequence_token_ids record = batch.identity_record @@ -665,6 +1042,10 @@ def _validate_frozen_identity_lineage( raise ValueError("identity sequence length differs from the processed token count") if record["sequence_token_ids_sha256"] != token_hash: raise ValueError("sequence token-ID SHA-256 differs from the capture identity") + _validate_fisher_boundary( + record["fisher_boundary"], + sequence_token_ids=token_ids, + ) for field, actual in ( ("family", batch.family), ("config", batch.config), @@ -711,6 +1092,298 @@ def _validate_frozen_identity_lineage( ) +def _comparator_expected_positions( + selector_profile: object, + token_count: int, +) -> tuple[int, ...]: + if selector_profile == FROZEN_UNWEIGHTED_MSE_PROFILE: + return frozen_anchor_positions(token_count) + if selector_profile == FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE: + return fisher_h1_boundary_positions(token_count) + raise ValueError("selector_profile must be one of the two frozen comparator methods") + + +def _comparator_position_payload( + *, + selector_profile: UnweightedSelectorProfile, + token_count: int, + endpoint_positions: tuple[int, ...], + sequence_token_ids_sha256_value: str, + identity_anchor_manifest_sha256_value: str, + identity_record_sha256_value: str, + fisher_boundary_sha256: str, +) -> dict[str, object]: + return { + "endpoint_positions": list(endpoint_positions), + "fisher_boundary_sha256": fisher_boundary_sha256, + "identity_anchor_manifest_sha256": identity_anchor_manifest_sha256_value, + "identity_record_sha256": identity_record_sha256_value, + "position_contract": FROZEN_COMPARATOR_POSITION_CONTRACTS[selector_profile], + "selector_profile": selector_profile, + "sequence_token_ids_sha256": sequence_token_ids_sha256_value, + "token_count": token_count, + } + + +def _comparator_sequence_score_sha256( + *, + selector_profile: UnweightedSelectorProfile, + position_manifest_sha256: str, + endpoint_inputs_sha256: str, + identity_record_sha256_value: str, + d4: torch.Tensor, + d6: torch.Tensor, + d8: torch.Tensor, +) -> str: + metadata = { + "endpoint_inputs_sha256": endpoint_inputs_sha256, + "identity_record_sha256": identity_record_sha256_value, + "position_manifest_sha256": position_manifest_sha256, + "row_count": d4.numel(), + "selector_profile": selector_profile, + } + return _hash_score_triplet( + d4, + d6, + d8, + domain=_COMPARATOR_SEQUENCE_SCORE_HASH_DOMAIN, + metadata=metadata, + ) + + +@dataclass(frozen=True, slots=True) +class ComparatorSequenceScores: + """One identity-bound sequence reduced to comparator row scores.""" + + selector_profile: UnweightedSelectorProfile + family: CalibrationFamily + config: str + ruler_category: RulerCategory | None + canonical_id: str + seed: int | None + configured_length: int | None + token_count: int + endpoint_positions: tuple[int, ...] + position_manifest_sha256: str + endpoint_inputs_sha256: str + sequence_scores_sha256: str + d4: torch.Tensor + d6: torch.Tensor + d8: torch.Tensor + source_shape: tuple[int, ...] + sequence_token_ids_sha256: str + token_span: tuple[tuple[str, int], ...] + identity_anchor_manifest_sha256: str + identity_record_sha256: str + fisher_boundary_sha256: str + target_nlls_sha256: str | None + + @property + def row_count(self) -> int: + return self.d4.numel() + + def identity_tuple(self) -> tuple[object, ...]: + return ( + self.family, + self.ruler_category, + self.config, + self.canonical_id, + self.seed, + self.configured_length, + self.token_count, + ) + + def position_manifest_record(self) -> dict[str, object]: + return { + "canonical_id": self.canonical_id, + "config": self.config, + "configured_length": self.configured_length, + "family": self.family, + "identity_record_sha256": self.identity_record_sha256, + "position_manifest_sha256": self.position_manifest_sha256, + "ruler_category": self.ruler_category, + "seed": self.seed, + "token_count": self.token_count, + } + + def manifest_record(self) -> dict[str, object]: + return { + **self.position_manifest_record(), + "endpoint_inputs_sha256": self.endpoint_inputs_sha256, + "fisher_boundary_sha256": self.fisher_boundary_sha256, + "identity_anchor_manifest_sha256": self.identity_anchor_manifest_sha256, + "selector_profile": self.selector_profile, + "sequence_scores_sha256": self.sequence_scores_sha256, + "sequence_token_ids_sha256": self.sequence_token_ids_sha256, + "source_shape": list(self.source_shape), + "target_nlls_sha256": self.target_nlls_sha256, + "token_span": dict(self.token_span), + } + + +def reduce_frozen_comparator_endpoints( + batch: FrozenComparatorEndpointBatch, +) -> ComparatorSequenceScores: + """Reduce one strict Identity-v5 comparator endpoint batch on CPU-FP64.""" + + if not isinstance(batch, FrozenComparatorEndpointBatch): + raise TypeError("batch must be a FrozenComparatorEndpointBatch") + ( + family, + config, + ruler_category, + canonical_id, + seed, + configured_length, + token_count, + ) = _metadata( + family=batch.family, + config=batch.config, + ruler_category=batch.ruler_category, + canonical_id=batch.canonical_id, + seed=batch.seed, + configured_length=batch.configured_length, + token_count=batch.token_count, + ) + expected_positions = _comparator_expected_positions(batch.selector_profile, token_count) + if not isinstance(batch.endpoint_positions, tuple): + raise TypeError("endpoint_positions must be a tuple") + if batch.endpoint_positions != expected_positions: + raise ValueError( + "endpoint_positions differ from the selector-specific frozen A(T)/B(T) equation" + ) + if not isinstance(batch.sequence_token_ids, tuple): + raise TypeError("sequence_token_ids must be a tuple") + if not isinstance(batch.identity_record, Mapping): + raise TypeError("identity_record must be a mapping") + lineage = _validate_frozen_identity_lineage(batch) + + source_shape = ( + len(expected_positions), + *FROZEN_SOURCE_TENSOR_CONTRACT.trailing_shape, + ) + endpoint_values: list[torch.Tensor] = [] + for name, value in ( + ("q4_scores", batch.q4_scores), + ("q6_scores", batch.q6_scores), + ("q8_scores", batch.q8_scores), + ): + if not isinstance(value, torch.Tensor): + raise TypeError(f"{name} must be a torch.Tensor") + if tuple(value.shape) != source_shape: + raise ValueError(f"{name} must have frozen source shape {source_shape}") + if value.device.type != "cpu" or value.dtype != torch.float64: + raise TypeError(f"{name} must be a CPU torch.float64 tensor") + endpoint_values.append( + _cpu_fp64_matrix( + value, + name=name, + anchors=len(expected_positions), + expected_shape=source_shape, + ) + ) + + target_nlls_sha256: str | None = None + if batch.selector_profile == FROZEN_UNWEIGHTED_MSE_PROFILE: + if batch.target_nlls is not None: + raise ValueError("unweighted MSE comparator cannot carry target_nlls") + else: + if batch.target_nlls is None: + raise ValueError("diagonal empirical-Fisher H1 comparator requires target_nlls") + target_nlls = batch.target_nlls + if not isinstance(target_nlls, torch.Tensor): + raise TypeError("target_nlls must be a torch.Tensor or None") + if ( + target_nlls.device.type != "cpu" + or target_nlls.dtype != torch.float64 + or tuple(target_nlls.shape) != (len(expected_positions),) + ): + raise TypeError( + "target_nlls must be a CPU torch.float64 vector matching endpoint positions" + ) + if not torch.isfinite(target_nlls).all().item() or (target_nlls < 0).any().item(): + raise ValueError("target_nlls must contain only finite non-negative values") + target_nlls_sha256 = hashlib.sha256( + _COMPARATOR_TARGET_NLL_HASH_DOMAIN + _tensor_bytes(target_nlls) + ).hexdigest() + + raw_boundary = batch.identity_record["fisher_boundary"] + assert isinstance(raw_boundary, Mapping) + fisher_boundary_hash = _sha256( + raw_boundary["fisher_boundary_sha256"], + name="identity_record.fisher_boundary.fisher_boundary_sha256", + ) + position_payload = _comparator_position_payload( + selector_profile=batch.selector_profile, + token_count=token_count, + endpoint_positions=expected_positions, + sequence_token_ids_sha256_value=lineage.sequence_token_ids_sha256, + identity_anchor_manifest_sha256_value=lineage.identity_anchor_manifest_sha256, + identity_record_sha256_value=lineage.identity_record_sha256, + fisher_boundary_sha256=fisher_boundary_hash, + ) + position_manifest_sha256 = _domain_json_sha256( + _COMPARATOR_POSITION_HASH_DOMAIN, + position_payload, + ) + endpoint_metadata = { + "axis_order": list(FROZEN_COMPARATOR_ENDPOINT_AXIS_ORDER), + "dtype": CALIBRATION_SCORE_DTYPE, + "position_manifest_sha256": position_manifest_sha256, + "selector_profile": batch.selector_profile, + "shape": list(source_shape), + "target_nlls_sha256": target_nlls_sha256, + } + endpoint_digest = hashlib.sha256() + endpoint_digest.update(_COMPARATOR_ENDPOINT_INPUT_HASH_DOMAIN) + endpoint_digest.update(canonical_json_bytes(endpoint_metadata)) + for label, value in zip( + (b"Q4\0", b"Q6\0", b"Q8\0"), + endpoint_values, + strict=True, + ): + endpoint_digest.update(label) + endpoint_digest.update(_tensor_bytes(value)) + endpoint_inputs_sha256 = endpoint_digest.hexdigest() + reduced = cast( + tuple[torch.Tensor, torch.Tensor, torch.Tensor], + tuple(value.mean(dim=0).contiguous() for value in endpoint_values), + ) + sequence_scores_sha256 = _comparator_sequence_score_sha256( + selector_profile=batch.selector_profile, + position_manifest_sha256=position_manifest_sha256, + endpoint_inputs_sha256=endpoint_inputs_sha256, + identity_record_sha256_value=lineage.identity_record_sha256, + d4=reduced[0], + d6=reduced[1], + d8=reduced[2], + ) + return ComparatorSequenceScores( + selector_profile=batch.selector_profile, + family=family, + config=config, + ruler_category=ruler_category, + canonical_id=canonical_id, + seed=seed, + configured_length=configured_length, + token_count=token_count, + endpoint_positions=expected_positions, + position_manifest_sha256=position_manifest_sha256, + endpoint_inputs_sha256=endpoint_inputs_sha256, + sequence_scores_sha256=sequence_scores_sha256, + d4=reduced[0], + d6=reduced[1], + d8=reduced[2], + source_shape=source_shape, + sequence_token_ids_sha256=lineage.sequence_token_ids_sha256, + token_span=lineage.token_span, + identity_anchor_manifest_sha256=lineage.identity_anchor_manifest_sha256, + identity_record_sha256=lineage.identity_record_sha256, + fisher_boundary_sha256=fisher_boundary_hash, + target_nlls_sha256=target_nlls_sha256, + ) + + def _reduce_anchor_distortions( batch: AnchorDistortionBatch, *, @@ -1050,6 +1723,72 @@ def _mean_score_rows(rows: list[torch.Tensor], *, context: str) -> torch.Tensor: return result.contiguous() +def _family_balanced_score_aggregate( + ordered: Sequence[CalibrationSequenceScores | ComparatorSequenceScores], +) -> tuple[ + tuple[torch.Tensor, torch.Tensor, torch.Tensor], + tuple[tuple[str, int], ...], + tuple[tuple[str, int], ...], +]: + """Apply one shared MBPP/PG19/four-category-RULER macro equation.""" + + grouped = { + family: [sequence for sequence in ordered if sequence.family == family] + for family in CALIBRATION_FAMILY_ORDER + } + if any(not grouped[family] for family in CALIBRATION_FAMILY_ORDER): + raise ValueError("MBPP, PG19, and RULER must each contain at least one sequence") + ruler_categories = tuple( + category + for category in RULER_CATEGORY_ORDER + if any(sequence.ruler_category == category for sequence in grouped["ruler"]) + ) + if ruler_categories != RULER_CATEGORY_ORDER: + raise ValueError("RULER calibration must contain all four frozen categories") + + aggregate_by_bit: list[torch.Tensor] = [] + for attribute in ("d4", "d6", "d8"): + mbpp = _mean_score_rows( + [getattr(sequence, attribute) for sequence in grouped["mbpp"]], + context=f"MBPP {attribute}", + ) + pg19 = _mean_score_rows( + [getattr(sequence, attribute) for sequence in grouped["pg19"]], + context=f"PG19 {attribute}", + ) + ruler_category_means = [ + _mean_score_rows( + [ + getattr(sequence, attribute) + for sequence in grouped["ruler"] + if sequence.ruler_category == category + ], + context=f"RULER {category} {attribute}", + ) + for category in ruler_categories + ] + ruler = _mean_score_rows(ruler_category_means, context=f"RULER macro {attribute}") + aggregate_by_bit.append( + _mean_score_rows([mbpp, pg19, ruler], context=f"broad-family macro {attribute}") + ) + family_counts = tuple((family, len(grouped[family])) for family in CALIBRATION_FAMILY_ORDER) + ruler_counts = tuple( + ( + category, + sum(sequence.ruler_category == category for sequence in grouped["ruler"]), + ) + for category in ruler_categories + ) + return ( + cast( + tuple[torch.Tensor, torch.Tensor, torch.Tensor], + tuple(aggregate_by_bit), + ), + family_counts, + ruler_counts, + ) + + def aggregate_calibration_scores( sequences: list[CalibrationSequenceScores] | tuple[CalibrationSequenceScores, ...], ) -> CalibrationAggregate: @@ -1090,65 +1829,273 @@ def aggregate_calibration_scores( else: identity_manifest_sha256 = None - grouped = { - family: [sequence for sequence in ordered if sequence.family == family] - for family in CALIBRATION_FAMILY_ORDER - } - if any(not grouped[family] for family in CALIBRATION_FAMILY_ORDER): - raise ValueError("MBPP, PG19, and RULER must each contain at least one sequence") - ruler_categories = tuple( - category - for category in RULER_CATEGORY_ORDER - if any(sequence.ruler_category == category for sequence in grouped["ruler"]) + aggregate_by_bit, family_counts, ruler_counts = _family_balanced_score_aggregate(ordered) + + manifest = [sequence.manifest_record() for sequence in ordered] + manifest_sha256 = hashlib.sha256(canonical_json_bytes(manifest)).hexdigest() + return CalibrationAggregate( + d4=aggregate_by_bit[0], + d6=aggregate_by_bit[1], + d8=aggregate_by_bit[2], + family_sequence_counts=family_counts, + ruler_category_sequence_counts=ruler_counts, + sequence_score_manifest_sha256=manifest_sha256, + source_contract=source_contract, + identity_record_manifest_sha256=identity_manifest_sha256, ) - if ruler_categories != RULER_CATEGORY_ORDER: - raise ValueError("RULER calibration must contain all four frozen categories") - aggregate_by_bit: list[torch.Tensor] = [] - for attribute in ("d4", "d6", "d8"): - mbpp = _mean_score_rows( - [getattr(sequence, attribute) for sequence in grouped["mbpp"]], - context=f"MBPP {attribute}", - ) - pg19 = _mean_score_rows( - [getattr(sequence, attribute) for sequence in grouped["pg19"]], - context=f"PG19 {attribute}", - ) - ruler_category_means = [ - _mean_score_rows( - [ - getattr(sequence, attribute) - for sequence in grouped["ruler"] - if sequence.ruler_category == category - ], - context=f"RULER {category} {attribute}", + +@dataclass(frozen=True, slots=True) +class ComparatorAggregate: + """Family-balanced score arrays and complete hashed comparator lineage.""" + + selector_profile: UnweightedSelectorProfile + d4: torch.Tensor + d6: torch.Tensor + d8: torch.Tensor + family_sequence_counts: tuple[tuple[str, int], ...] + ruler_category_sequence_counts: tuple[tuple[str, int], ...] + position_manifest_sha256: str + sequence_score_manifest_sha256: str + identity_record_manifest_sha256: str + aggregate_scores_sha256: str + + @property + def row_count(self) -> int: + return self.d4.numel() + + def scores(self) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + return self.d4, self.d6, self.d8 + + +def _comparator_sequence_sort_key(sequence: ComparatorSequenceScores) -> tuple[object, ...]: + return ( + CALIBRATION_FAMILY_ORDER.index(sequence.family), + "" if sequence.ruler_category is None else sequence.ruler_category, + sequence.config, + sequence.canonical_id, + -1 if sequence.seed is None else sequence.seed, + -1 if sequence.configured_length is None else sequence.configured_length, + sequence.token_count, + ) + + +def _validate_comparator_sequence_score( + sequence: object, + *, + expected_rows: int | None, +) -> ComparatorSequenceScores: + if not isinstance(sequence, ComparatorSequenceScores): + raise TypeError("sequences must contain ComparatorSequenceScores") + _metadata( + family=sequence.family, + config=sequence.config, + ruler_category=sequence.ruler_category, + canonical_id=sequence.canonical_id, + seed=sequence.seed, + configured_length=sequence.configured_length, + token_count=sequence.token_count, + ) + expected_positions = _comparator_expected_positions( + sequence.selector_profile, + sequence.token_count, + ) + if sequence.endpoint_positions != expected_positions: + raise ValueError("comparator sequence positions drifted from frozen A(T)/B(T)") + expected_shape = ( + len(expected_positions), + *FROZEN_SOURCE_TENSOR_CONTRACT.trailing_shape, + ) + if sequence.source_shape != expected_shape: + raise ValueError("comparator sequence source shape drifted") + rows = FROZEN_QWEN35_STATIC_Q468_GEOMETRY.total_rows + if expected_rows is not None and rows != expected_rows: + raise ValueError("comparator sequence row count differs") + for name, value in (("D4", sequence.d4), ("D6", sequence.d6), ("D8", sequence.d8)): + if not isinstance(value, torch.Tensor): + raise TypeError(f"{name} must be a torch.Tensor") + if ( + value.device.type != "cpu" + or value.dtype != torch.float64 + or tuple(value.shape) != (rows,) + or not value.is_contiguous() + ): + raise TypeError(f"{name} must be a contiguous CPU torch.float64 row vector") + d4, d6, d8 = cast( + tuple[torch.Tensor, torch.Tensor, torch.Tensor], + tuple( + _cpu_fp64_scores(value, name=name, expected_rows=rows) + for name, value in ( + ("D4", sequence.d4), + ("D6", sequence.d6), + ("D8", sequence.d8), ) - for category in ruler_categories - ] - ruler = _mean_score_rows(ruler_category_means, context=f"RULER macro {attribute}") - aggregate_by_bit.append( - _mean_score_rows([mbpp, pg19, ruler], context=f"broad-family macro {attribute}") - ) + ), + ) + token_hash = _sha256( + sequence.sequence_token_ids_sha256, + name="sequence_token_ids_sha256", + ) + span = _normalize_token_span( + dict(sequence.token_span), + sequence_length=sequence.token_count, + ) + identity_anchor_hash = identity_anchor_manifest_sha256( + canonical_id=sequence.canonical_id, + sequence_length=sequence.token_count, + sequence_token_ids_sha256_value=token_hash, + token_span=span, + ) + if identity_anchor_hash != sequence.identity_anchor_manifest_sha256: + raise ValueError("comparator identity anchor manifest SHA-256 drifted") + identity_record_hash = _sha256( + sequence.identity_record_sha256, + name="identity_record_sha256", + ) + fisher_boundary_hash = _sha256( + sequence.fisher_boundary_sha256, + name="fisher_boundary_sha256", + ) + if sequence.target_nlls_sha256 is not None: + _sha256(sequence.target_nlls_sha256, name="target_nlls_sha256") + if ( + sequence.selector_profile == FROZEN_UNWEIGHTED_MSE_PROFILE + and sequence.target_nlls_sha256 is not None + ): + raise ValueError("unweighted MSE sequence cannot claim target NLL input") + if ( + sequence.selector_profile == FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE + and sequence.target_nlls_sha256 is None + ): + raise ValueError("diagonal empirical-Fisher H1 sequence requires a target NLL receipt") + endpoint_hash = _sha256( + sequence.endpoint_inputs_sha256, + name="endpoint_inputs_sha256", + ) + position_payload = _comparator_position_payload( + selector_profile=sequence.selector_profile, + token_count=sequence.token_count, + endpoint_positions=expected_positions, + sequence_token_ids_sha256_value=token_hash, + identity_anchor_manifest_sha256_value=identity_anchor_hash, + identity_record_sha256_value=identity_record_hash, + fisher_boundary_sha256=fisher_boundary_hash, + ) + expected_position_hash = _domain_json_sha256( + _COMPARATOR_POSITION_HASH_DOMAIN, + position_payload, + ) + if sequence.position_manifest_sha256 != expected_position_hash: + raise ValueError("comparator position-manifest SHA-256 drifted") + expected_score_hash = _comparator_sequence_score_sha256( + selector_profile=sequence.selector_profile, + position_manifest_sha256=expected_position_hash, + endpoint_inputs_sha256=endpoint_hash, + identity_record_sha256_value=identity_record_hash, + d4=d4, + d6=d6, + d8=d8, + ) + if sequence.sequence_scores_sha256 != expected_score_hash: + raise ValueError("comparator sequence-score SHA-256 drifted") + return sequence + + +def _comparator_aggregate_score_sha256( + aggregate: ComparatorAggregate, +) -> str: + metadata = { + "family_sequence_counts": [ + {"count": count, "family": family} for family, count in aggregate.family_sequence_counts + ], + "identity_record_manifest_sha256": aggregate.identity_record_manifest_sha256, + "position_manifest_sha256": aggregate.position_manifest_sha256, + "row_count": aggregate.row_count, + "ruler_category_sequence_counts": [ + {"category": category, "count": count} + for category, count in aggregate.ruler_category_sequence_counts + ], + "selector_profile": aggregate.selector_profile, + "sequence_score_manifest_sha256": aggregate.sequence_score_manifest_sha256, + } + return _hash_score_triplet( + *aggregate.scores(), + domain=_COMPARATOR_AGGREGATE_SCORE_HASH_DOMAIN, + metadata=metadata, + ) + + +def aggregate_comparator_scores( + sequences: list[ComparatorSequenceScores] | tuple[ComparatorSequenceScores, ...], +) -> ComparatorAggregate: + """Aggregate one comparator with the candidate's exact family-balanced macro.""" + + if not isinstance(sequences, (list, tuple)) or not sequences: + raise ValueError("sequences must be a non-empty list or tuple") + if any(not isinstance(sequence, ComparatorSequenceScores) for sequence in sequences): + raise TypeError("sequences must contain ComparatorSequenceScores") + selector_profiles = {sequence.selector_profile for sequence in sequences} + if len(selector_profiles) != 1: + raise ValueError("one comparator aggregate cannot mix selector profiles") + selector_profile = next(iter(selector_profiles)) + _comparator_expected_positions(selector_profile, 3) + expected_rows = FROZEN_QWEN35_STATIC_Q468_GEOMETRY.total_rows + for sequence in sequences: + _validate_comparator_sequence_score(sequence, expected_rows=expected_rows) + ordered = sorted(sequences, key=_comparator_sequence_sort_key) + identities: set[tuple[object, ...]] = set() + for sequence in ordered: + identity = sequence.identity_tuple() + if identity in identities: + raise ValueError(f"duplicate comparator sequence identity: {identity!r}") + identities.add(identity) - manifest = [sequence.manifest_record() for sequence in ordered] - manifest_sha256 = hashlib.sha256(canonical_json_bytes(manifest)).hexdigest() - family_counts = tuple((family, len(grouped[family])) for family in CALIBRATION_FAMILY_ORDER) - ruler_counts = tuple( - ( - category, - sum(sequence.ruler_category == category for sequence in grouped["ruler"]), - ) - for category in ruler_categories + aggregate_scores, family_counts, ruler_counts = _family_balanced_score_aggregate(ordered) + identity_records = [ + { + "canonical_id": sequence.canonical_id, + "config": sequence.config, + "configured_length": sequence.configured_length, + "family": sequence.family, + "identity_record_sha256": sequence.identity_record_sha256, + "ruler_category": sequence.ruler_category, + "seed": sequence.seed, + "sequence_length": sequence.token_count, + } + for sequence in ordered + ] + identity_manifest_hash = calibration_identity_record_manifest_sha256(identity_records) + position_manifest_hash = _domain_json_sha256( + _COMPARATOR_POSITION_MANIFEST_HASH_DOMAIN, + [sequence.position_manifest_record() for sequence in ordered], ) - return CalibrationAggregate( - d4=aggregate_by_bit[0], - d6=aggregate_by_bit[1], - d8=aggregate_by_bit[2], + sequence_manifest_hash = _domain_json_sha256( + _COMPARATOR_SEQUENCE_MANIFEST_HASH_DOMAIN, + [sequence.manifest_record() for sequence in ordered], + ) + provisional = ComparatorAggregate( + selector_profile=cast(UnweightedSelectorProfile, selector_profile), + d4=aggregate_scores[0], + d6=aggregate_scores[1], + d8=aggregate_scores[2], family_sequence_counts=family_counts, ruler_category_sequence_counts=ruler_counts, - sequence_score_manifest_sha256=manifest_sha256, - source_contract=source_contract, - identity_record_manifest_sha256=identity_manifest_sha256, + position_manifest_sha256=position_manifest_hash, + sequence_score_manifest_sha256=sequence_manifest_hash, + identity_record_manifest_sha256=identity_manifest_hash, + aggregate_scores_sha256="0" * 64, + ) + return ComparatorAggregate( + selector_profile=provisional.selector_profile, + d4=provisional.d4, + d6=provisional.d6, + d8=provisional.d8, + family_sequence_counts=provisional.family_sequence_counts, + ruler_category_sequence_counts=provisional.ruler_category_sequence_counts, + position_manifest_sha256=provisional.position_manifest_sha256, + sequence_score_manifest_sha256=provisional.sequence_score_manifest_sha256, + identity_record_manifest_sha256=provisional.identity_record_manifest_sha256, + aggregate_scores_sha256=_comparator_aggregate_score_sha256(provisional), ) @@ -2306,6 +3253,523 @@ def verify_calibration_score_artifact( } +def _validate_frozen_comparator_aggregate( + aggregate: object, + *, + expected_profile: UnweightedSelectorProfile, +) -> ComparatorAggregate: + if not isinstance(aggregate, ComparatorAggregate): + raise TypeError("aggregate must be a ComparatorAggregate") + if aggregate.selector_profile != expected_profile: + raise ValueError("comparator aggregate selector profile differs from its slot") + expected_rows = FROZEN_QWEN35_STATIC_Q468_GEOMETRY.total_rows + for name, value in (("D4", aggregate.d4), ("D6", aggregate.d6), ("D8", aggregate.d8)): + if not isinstance(value, torch.Tensor): + raise TypeError(f"{name} must be a torch.Tensor") + if ( + value.device.type != "cpu" + or value.dtype != torch.float64 + or tuple(value.shape) != (expected_rows,) + or not value.is_contiguous() + ): + raise TypeError(f"{name} must be a contiguous CPU torch.float64 row vector") + scores = tuple( + _cpu_fp64_scores(value, name=name, expected_rows=expected_rows) + for name, value in ( + ("D4", aggregate.d4), + ("D6", aggregate.d6), + ("D8", aggregate.d8), + ) + ) + if any( + not torch.equal(left, right) for left, right in zip(scores, aggregate.scores(), strict=True) + ): + raise ValueError("comparator aggregate scores are not canonical CPU-FP64 vectors") + if aggregate.family_sequence_counts != ( + ("mbpp", 128), + ("pg19", 16), + ("ruler", 16), + ): + raise ValueError("frozen comparator counts must be MBPP=128, PG19=16, RULER=16") + if aggregate.ruler_category_sequence_counts != tuple( + (category, 4) for category in RULER_CATEGORY_ORDER + ): + raise ValueError("each frozen comparator RULER category must contain four sequences") + for name, value in ( + ("position_manifest_sha256", aggregate.position_manifest_sha256), + ("sequence_score_manifest_sha256", aggregate.sequence_score_manifest_sha256), + ("identity_record_manifest_sha256", aggregate.identity_record_manifest_sha256), + ("aggregate_scores_sha256", aggregate.aggregate_scores_sha256), + ): + _sha256(value, name=name) + if aggregate.aggregate_scores_sha256 != _comparator_aggregate_score_sha256(aggregate): + raise ValueError("comparator aggregate-score SHA-256 drifted") + return aggregate + + +def _allocate_frozen_comparator_codes(aggregate: ComparatorAggregate) -> torch.Tensor: + rows = FROZEN_QWEN35_STATIC_Q468_GEOMETRY.total_rows + codes = allocate_exact_multibit_codes_fast( + *(value.reshape(1, rows) for value in aggregate.scores()), + marginal_steps=FROZEN_STATIC_Q468_PRIMARY_STEPS, + ).reshape(-1) + if codes.device.type != "cpu" or codes.dtype != torch.uint8 or codes.shape != (rows,): + raise RuntimeError("comparator allocator returned a non-canonical code map") + if int(codes.to(torch.int64).sum().item()) != FROZEN_STATIC_Q468_PRIMARY_STEPS: + raise RuntimeError("comparator allocator missed exact K29334") + return codes.contiguous() + + +def _frozen_comparator_selector_record( + aggregate: ComparatorAggregate, +) -> dict[str, object]: + codes = _allocate_frozen_comparator_codes(aggregate) + geometry = FROZEN_QWEN35_STATIC_Q468_GEOMETRY + return { + "allocation": { + "allocator_revision": STATIC_Q468_ALLOCATOR_REVISION, + "code_counts_q4_q6_q8": [int((codes == code).sum().item()) for code in range(3)], + "code_map_sha256": static_q468_code_map_sha256( + codes, + geometry=geometry, + marginal_steps=FROZEN_STATIC_Q468_PRIMARY_STEPS, + ), + "marginal_steps": FROZEN_STATIC_Q468_PRIMARY_STEPS, + "packed_precision_bytes": math.ceil(geometry.total_rows * 2 / 8), + }, + "calibration_scores_sha256": aggregate.aggregate_scores_sha256, + "family_sequence_counts": [ + {"count": count, "family": family} for family, count in aggregate.family_sequence_counts + ], + "method_id": aggregate.selector_profile, + "position_contract": FROZEN_COMPARATOR_POSITION_CONTRACTS[aggregate.selector_profile], + "position_manifest_sha256": aggregate.position_manifest_sha256, + "ruler_category_sequence_counts": [ + {"category": category, "count": count} + for category, count in aggregate.ruler_category_sequence_counts + ], + "scores": { + "axis_order": ["bitwidth", "flattened_layer_head_key_row"], + "bitwidths": [4, 6, 8], + "data_base64": _score_data_b64(aggregate.scores()), + "dtype": CALIBRATION_SCORE_DTYPE, + "shape": [3, geometry.total_rows], + }, + "sequence_score_manifest_sha256": aggregate.sequence_score_manifest_sha256, + } + + +def build_frozen_comparator_score_artifact( + mse_aggregate: ComparatorAggregate, + fisher_aggregate: ComparatorAggregate, + *, + calibration_identity_sha256: str, +) -> bytes: + """Build canonical ``comparator-scores.json`` with both exact-K29334 methods.""" + + identity_sha256 = _sha256( + calibration_identity_sha256, + name="calibration_identity_sha256", + ) + normalized_mse = _validate_frozen_comparator_aggregate( + mse_aggregate, + expected_profile=FROZEN_UNWEIGHTED_MSE_PROFILE, + ) + normalized_fisher = _validate_frozen_comparator_aggregate( + fisher_aggregate, + expected_profile=FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE, + ) + if ( + normalized_mse.identity_record_manifest_sha256 + != normalized_fisher.identity_record_manifest_sha256 + ): + raise ValueError("both comparator profiles must bind the same Identity-v5 records") + if ( + normalized_mse.family_sequence_counts != normalized_fisher.family_sequence_counts + or normalized_mse.ruler_category_sequence_counts + != normalized_fisher.ruler_category_sequence_counts + ): + raise ValueError("both comparator profiles must cover the same frozen sequence counts") + + geometry = FROZEN_QWEN35_STATIC_Q468_GEOMETRY + evidence = { + "aggregation_contract": _COMPARATOR_AGGREGATION_CONTRACT, + "artifact_profile": COMPARATOR_SCORE_ARTIFACT_PROFILE, + "artifact_revision": COMPARATOR_SCORE_ARTIFACT_REVISION, + "calibration_identity_sha256": identity_sha256, + "endpoint_tensor_contract": { + "axis_order": list(FROZEN_COMPARATOR_ENDPOINT_AXIS_ORDER), + "dtype": CALIBRATION_SCORE_DTYPE, + "trailing_shape": list(FROZEN_SOURCE_TENSOR_CONTRACT.trailing_shape), + }, + "geometry": geometry.canonical_dict(), + "geometry_sha256": geometry.geometry_sha256, + "identity_record_manifest_sha256": (normalized_mse.identity_record_manifest_sha256), + "selectors": [ + _frozen_comparator_selector_record(normalized_mse), + _frozen_comparator_selector_record(normalized_fisher), + ], + } + canonical_evidence_sha256 = hashlib.sha256(canonical_json_bytes(evidence)).hexdigest() + return canonical_json_bytes( + { + "artifact_kind": COMPARATOR_SCORE_ARTIFACT_KIND, + "canonical_evidence_sha256": canonical_evidence_sha256, + "evidence": evidence, + "schema_version": COMPARATOR_SCORE_ARTIFACT_SCHEMA_VERSION, + } + ) + + +@dataclass(frozen=True, slots=True) +class ComparatorSelectorArtifact: + method_id: UnweightedSelectorProfile + aggregate: ComparatorAggregate + position_manifest_sha256: str + calibration_scores_sha256: str + marginal_steps: int + precision_codes: torch.Tensor + code_map_sha256: str + + +@dataclass(frozen=True, slots=True) +class ComparatorScoreArtifact: + selectors: Mapping[str, ComparatorSelectorArtifact] + calibration_identity_sha256: str + canonical_evidence_sha256: str + file_sha256: str + + +def _decode_comparator_scores( + value: object, + *, + context: str, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + record = _mapping(value, context=context) + _exact_keys( + record, + {"axis_order", "bitwidths", "data_base64", "dtype", "shape"}, + context=context, + ) + geometry = FROZEN_QWEN35_STATIC_Q468_GEOMETRY + if record["axis_order"] != ["bitwidth", "flattened_layer_head_key_row"]: + raise CalibrationArtifactError(f"{context}.axis_order drifted") + if record["bitwidths"] != [4, 6, 8]: + raise CalibrationArtifactError(f"{context}.bitwidths must be Q4, Q6, Q8") + if record["dtype"] != CALIBRATION_SCORE_DTYPE: + raise CalibrationArtifactError(f"{context}.dtype must be float64-le") + if record["shape"] != [3, geometry.total_rows]: + raise CalibrationArtifactError(f"{context}.shape differs from frozen geometry") + encoded = record["data_base64"] + if not isinstance(encoded, str): + raise CalibrationArtifactError(f"{context}.data_base64 must be a string") + try: + raw = base64.b64decode(encoded, validate=True) + except (ValueError, binascii.Error) as exc: + raise CalibrationArtifactError(f"{context}.data_base64 is invalid") from exc + if base64.b64encode(raw).decode("ascii") != encoded: + raise CalibrationArtifactError(f"{context}.data_base64 is not canonical") + expected_bytes = 3 * geometry.total_rows * 8 + if len(raw) != expected_bytes: + raise CalibrationArtifactError( + f"{context} byte length differs: expected {expected_bytes}, got {len(raw)}" + ) + array = np.frombuffer(raw, dtype=" ComparatorScoreArtifact: + """Strictly decode and recompute canonical comparator-score evidence.""" + + if not isinstance(data, bytes): + raise TypeError("data must be bytes") + file_sha256 = hashlib.sha256(data).hexdigest() + try: + document = json.loads( + data.decode("utf-8"), + parse_constant=_reject_json_constant, + object_pairs_hook=_unique_json_object, + ) + except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as exc: + if isinstance(exc, CalibrationArtifactError): + raise + raise CalibrationArtifactError( + f"comparator artifact is not strict UTF-8 JSON: {exc}" + ) from exc + root = _mapping(document, context="comparator artifact") + _exact_keys( + root, + {"artifact_kind", "canonical_evidence_sha256", "evidence", "schema_version"}, + context="comparator artifact", + ) + if root["artifact_kind"] != COMPARATOR_SCORE_ARTIFACT_KIND: + raise CalibrationArtifactError("comparator artifact kind drifted") + if ( + _artifact_int( + root["schema_version"], + context="comparator artifact.schema_version", + minimum=1, + ) + != COMPARATOR_SCORE_ARTIFACT_SCHEMA_VERSION + ): + raise CalibrationArtifactError("comparator artifact schema version drifted") + evidence = _mapping(root["evidence"], context="comparator artifact.evidence") + _exact_keys( + evidence, + { + "aggregation_contract", + "artifact_profile", + "artifact_revision", + "calibration_identity_sha256", + "endpoint_tensor_contract", + "geometry", + "geometry_sha256", + "identity_record_manifest_sha256", + "selectors", + }, + context="comparator artifact.evidence", + ) + if evidence["artifact_profile"] != COMPARATOR_SCORE_ARTIFACT_PROFILE: + raise CalibrationArtifactError("comparator artifact profile drifted") + if evidence["artifact_revision"] != COMPARATOR_SCORE_ARTIFACT_REVISION: + raise CalibrationArtifactError("comparator artifact revision drifted") + if evidence["aggregation_contract"] != _COMPARATOR_AGGREGATION_CONTRACT: + raise CalibrationArtifactError("comparator aggregation contract drifted") + geometry = _parse_geometry(evidence["geometry"]) + if geometry != FROZEN_QWEN35_STATIC_Q468_GEOMETRY: + raise CalibrationArtifactError("comparator geometry differs from frozen Qwen3.5 geometry") + geometry_hash = _artifact_sha( + evidence["geometry_sha256"], + context="comparator artifact.evidence.geometry_sha256", + ) + if geometry_hash != geometry.geometry_sha256: + raise CalibrationArtifactError("comparator geometry SHA-256 mismatch") + expected_endpoint_contract = { + "axis_order": list(FROZEN_COMPARATOR_ENDPOINT_AXIS_ORDER), + "dtype": CALIBRATION_SCORE_DTYPE, + "trailing_shape": list(FROZEN_SOURCE_TENSOR_CONTRACT.trailing_shape), + } + if evidence["endpoint_tensor_contract"] != expected_endpoint_contract: + raise CalibrationArtifactError("comparator endpoint tensor contract drifted") + calibration_identity_hash = _artifact_sha( + evidence["calibration_identity_sha256"], + context="comparator artifact.evidence.calibration_identity_sha256", + ) + if expected_calibration_identity_sha256 is not None: + expected_identity = _artifact_sha( + expected_calibration_identity_sha256, + context="expected_calibration_identity_sha256", + ) + if calibration_identity_hash != expected_identity: + raise CalibrationArtifactError( + "comparator calibration identity SHA-256 differs from expected identity" + ) + identity_manifest_hash = _artifact_sha( + evidence["identity_record_manifest_sha256"], + context="comparator artifact.evidence.identity_record_manifest_sha256", + ) + recorded_canonical_hash = _artifact_sha( + root["canonical_evidence_sha256"], + context="comparator artifact.canonical_evidence_sha256", + ) + computed_canonical_hash = hashlib.sha256(canonical_json_bytes(evidence)).hexdigest() + if recorded_canonical_hash != computed_canonical_hash: + raise CalibrationArtifactError("comparator canonical evidence SHA-256 mismatch") + if canonical_json_bytes(root) != data: + raise CalibrationArtifactError("comparator artifact bytes are not canonical JSON") + + selector_values = _sequence( + evidence["selectors"], + context="comparator artifact.evidence.selectors", + ) + if len(selector_values) != len(FROZEN_COMPARATOR_PROFILE_ORDER): + raise CalibrationArtifactError("comparator artifact must contain exactly two selectors") + decoded_selectors: dict[str, ComparatorSelectorArtifact] = {} + for index, (raw_selector, expected_method) in enumerate( + zip(selector_values, FROZEN_COMPARATOR_PROFILE_ORDER, strict=True) + ): + context = f"comparator artifact.evidence.selectors[{index}]" + selector = _mapping(raw_selector, context=context) + _exact_keys( + selector, + { + "allocation", + "calibration_scores_sha256", + "family_sequence_counts", + "method_id", + "position_contract", + "position_manifest_sha256", + "ruler_category_sequence_counts", + "scores", + "sequence_score_manifest_sha256", + }, + context=context, + ) + if selector["method_id"] != expected_method: + raise CalibrationArtifactError( + "comparator selectors must contain exactly MSE then diagonal Fisher H1" + ) + if selector["position_contract"] != FROZEN_COMPARATOR_POSITION_CONTRACTS[expected_method]: + raise CalibrationArtifactError(f"{context}.position_contract drifted") + family_counts = _parse_counts( + selector["family_sequence_counts"], + context=f"{context}.family_sequence_counts", + key_name="family", + expected_names=CALIBRATION_FAMILY_ORDER, + ) + ruler_counts = _parse_counts( + selector["ruler_category_sequence_counts"], + context=f"{context}.ruler_category_sequence_counts", + key_name="category", + expected_names=RULER_CATEGORY_ORDER, + ) + scores = _decode_comparator_scores(selector["scores"], context=f"{context}.scores") + position_hash = _artifact_sha( + selector["position_manifest_sha256"], + context=f"{context}.position_manifest_sha256", + ) + sequence_manifest_hash = _artifact_sha( + selector["sequence_score_manifest_sha256"], + context=f"{context}.sequence_score_manifest_sha256", + ) + aggregate_score_hash = _artifact_sha( + selector["calibration_scores_sha256"], + context=f"{context}.calibration_scores_sha256", + ) + aggregate = ComparatorAggregate( + selector_profile=expected_method, + d4=scores[0], + d6=scores[1], + d8=scores[2], + family_sequence_counts=family_counts, + ruler_category_sequence_counts=ruler_counts, + position_manifest_sha256=position_hash, + sequence_score_manifest_sha256=sequence_manifest_hash, + identity_record_manifest_sha256=identity_manifest_hash, + aggregate_scores_sha256=aggregate_score_hash, + ) + _validate_frozen_comparator_aggregate( + aggregate, + expected_profile=expected_method, + ) + + allocation = _mapping(selector["allocation"], context=f"{context}.allocation") + _exact_keys( + allocation, + { + "allocator_revision", + "code_counts_q4_q6_q8", + "code_map_sha256", + "marginal_steps", + "packed_precision_bytes", + }, + context=f"{context}.allocation", + ) + if allocation["allocator_revision"] != STATIC_Q468_ALLOCATOR_REVISION: + raise CalibrationArtifactError(f"{context}.allocation allocator revision drifted") + marginal_steps = _artifact_int( + allocation["marginal_steps"], + context=f"{context}.allocation.marginal_steps", + ) + if marginal_steps != FROZEN_STATIC_Q468_PRIMARY_STEPS: + raise CalibrationArtifactError(f"{context} allocation must spend exact K29334") + expected_packed_bytes = math.ceil(geometry.total_rows * 2 / 8) + if ( + _artifact_int( + allocation["packed_precision_bytes"], + context=f"{context}.allocation.packed_precision_bytes", + ) + != expected_packed_bytes + ): + raise CalibrationArtifactError(f"{context} packed precision byte count drifted") + raw_counts = _sequence( + allocation["code_counts_q4_q6_q8"], + context=f"{context}.allocation.code_counts_q4_q6_q8", + ) + if len(raw_counts) != 3: + raise CalibrationArtifactError(f"{context} allocation needs Q4/Q6/Q8 counts") + counts = [ + _artifact_int(value, context=f"{context}.allocation.code_counts[{position}]") + for position, value in enumerate(raw_counts) + ] + codes = _allocate_frozen_comparator_codes(aggregate) + computed_counts = [int((codes == code).sum().item()) for code in range(3)] + if counts != computed_counts or sum(counts) != geometry.total_rows: + raise CalibrationArtifactError(f"{context} code counts differ from exact allocation") + code_map_hash = _artifact_sha( + allocation["code_map_sha256"], + context=f"{context}.allocation.code_map_sha256", + ) + computed_code_hash = static_q468_code_map_sha256( + codes, + geometry=geometry, + marginal_steps=marginal_steps, + ) + if code_map_hash != computed_code_hash: + raise CalibrationArtifactError(f"{context} code-map SHA-256 drifted") + decoded_selectors[expected_method] = ComparatorSelectorArtifact( + method_id=expected_method, + aggregate=aggregate, + position_manifest_sha256=position_hash, + calibration_scores_sha256=aggregate_score_hash, + marginal_steps=marginal_steps, + precision_codes=codes, + code_map_sha256=computed_code_hash, + ) + + if set(decoded_selectors) != set(FROZEN_COMPARATOR_PROFILE_ORDER): + raise CalibrationArtifactError("comparator artifact is missing a frozen selector") + return ComparatorScoreArtifact( + selectors=MappingProxyType(decoded_selectors), + calibration_identity_sha256=calibration_identity_hash, + canonical_evidence_sha256=computed_canonical_hash, + file_sha256=file_sha256, + ) + + +def verify_comparator_score_artifact( + data: bytes, + *, + expected_calibration_identity_sha256: str | None = None, +) -> dict[str, Any]: + """Return a JSON-compatible fail-closed comparator verification report.""" + + file_sha256 = hashlib.sha256(data).hexdigest() if isinstance(data, bytes) else None + try: + artifact = deserialize_comparator_score_artifact( + data, + expected_calibration_identity_sha256=expected_calibration_identity_sha256, + ) + except (TypeError, ValueError) as exc: + return { + "errors": [str(exc)], + "file_sha256": file_sha256, + "selectors": [], + "valid": False, + } + return { + "errors": [], + "file_sha256": artifact.file_sha256, + "selectors": [ + { + "code_map_sha256": artifact.selectors[method].code_map_sha256, + "method_id": method, + } + for method in FROZEN_COMPARATOR_PROFILE_ORDER + ], + "valid": True, + } + + def _stability_threshold_record() -> dict[str, str]: return { "maximum_layer_mean_bitwidth_shift": MAX_LAYER_MEAN_BITWIDTH_SHIFT.hex(), diff --git a/tests/test_capture_static_q468_identity_input.py b/tests/test_capture_static_q468_identity_input.py index 837d1ab..ba4f38f 100644 --- a/tests/test_capture_static_q468_identity_input.py +++ b/tests/test_capture_static_q468_identity_input.py @@ -8,6 +8,7 @@ import subprocess import sys import urllib.parse +from dataclasses import replace from pathlib import Path from types import SimpleNamespace from typing import Any @@ -15,22 +16,31 @@ import pytest import torch +import recurquant.static_q468_calibration as calibration +from recurquant import experiment013_source from recurquant.static_q468 import ( FROZEN_QWEN35_STATIC_Q468_GEOMETRY, FROZEN_STATIC_Q468_ABLATION_STEPS, FROZEN_STATIC_Q468_PRIMARY_STEPS, STATIC_Q468_ABLATION_METHOD, + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + STATIC_Q468_MSE_METHOD, STATIC_Q468_PRIMARY_METHOD, build_static_rht_q468_policy, serialize_static_rht_q468_policy, ) from recurquant.static_q468_calibration import ( + FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE, FROZEN_SOURCE_TENSOR_CONTRACT, + FROZEN_UNWEIGHTED_MSE_PROFILE, CalibrationAggregate, + ComparatorAggregate, build_frozen_calibration_score_artifact, + build_frozen_comparator_score_artifact, build_frozen_split_half_stability_artifact, calibration_identity_record_manifest_sha256, deserialize_calibration_score_artifact, + deserialize_comparator_score_artifact, ) REPOSITORY_ROOT = Path(__file__).resolve().parents[1] @@ -41,6 +51,7 @@ sys.modules[SPEC.name] = capture SPEC.loader.exec_module(capture) resolver = capture.resolver +FIXTURE_GIT_EXECUTABLE = experiment013_source.authenticate_git_executable().path FIXTURE_BINDING_ARTIFACT = b"verified-fixture-binding-artifact" FIXTURE_EXECUTION_ARTIFACTS = { "repository_source_manifest_file_sha256": b"fixture-source-manifest", @@ -53,9 +64,8 @@ FIXTURE_RUNTIME_CONTEXT = { "base_runtime_root": REPOSITORY_ROOT / "fixture-base-runtime", "staged_interpreter": REPOSITORY_ROOT / "fixture-base-runtime" / "python.exe", - "package_runtime_roots": { - "fixture-packages": REPOSITORY_ROOT / "fixture-packages" - }, + "git_executable": FIXTURE_GIT_EXECUTABLE, + "package_runtime_roots": {"fixture-packages": REPOSITORY_ROOT / "fixture-packages"}, "package_import_paths": {"fixture-packages": "Lib/site-packages"}, } @@ -244,6 +254,7 @@ def authenticate_execution( else { "base_runtime_root": runtime_context.base_runtime_root, "staged_interpreter": runtime_context.staged_interpreter, + "git_executable": runtime_context.git_executable, "package_runtime_roots": runtime_context.package_runtime_roots, "package_import_paths": runtime_context.package_import_paths, } @@ -332,6 +343,7 @@ def runtime_authentication_context(self) -> dict[str, object]: return { "base_runtime_root": FIXTURE_RUNTIME_CONTEXT["base_runtime_root"], "staged_interpreter": FIXTURE_RUNTIME_CONTEXT["staged_interpreter"], + "git_executable": FIXTURE_RUNTIME_CONTEXT["git_executable"], "package_runtime_roots": dict( FIXTURE_RUNTIME_CONTEXT["package_runtime_roots"] # type: ignore[arg-type] ), @@ -357,8 +369,12 @@ def mbpp_train_rows(self) -> tuple[dict[str, Any], ...]: def pg19_projection(self, split: str) -> tuple[Any, ...]: self.accesses.append(f"pg19_projection:{split}") count = 13_684 if split == "train" else 50 + ebook_base = 100_000 if split == "train" else 200_000 return tuple( - capture.ProjectionRow(f"https://pg19.example/{split}/{offset}", offset) + capture.ProjectionRow( + f"http://www.gutenberg.org/ebooks/{ebook_base + offset}", + offset, + ) for offset in range(count) ) @@ -689,6 +705,26 @@ def _binding() -> bytes: return FIXTURE_BINDING_ARTIFACT +def _frozen_stage_a_identity(source: FakeSource | None = None) -> bytes: + captured = capture.capture_identity_input( + phase="stage_a", + source=FakeSource() if source is None else source, + calibration_binding=_binding(), + ) + candidate = resolver.build_candidate( + captured, + expected_revisions=resolver.FROZEN_DATASET_REVISIONS, + calibration_binding_artifact=_binding(), + ) + candidate_bytes = resolver.canonical_json_bytes(candidate) + frozen = resolver.promote_candidate( + candidate, + candidate_file_sha256=resolver.sha256_bytes(candidate_bytes), + calibration_binding_artifact=_binding(), + ) + return resolver.canonical_json_bytes(frozen) + + def _frozen_aggregate( *, half: bool, @@ -717,6 +753,34 @@ def _frozen_aggregate( ) +def _frozen_comparator_aggregate( + selector_profile: str, + *, + identity_manifest_sha256: str, +) -> ComparatorAggregate: + rows = FROZEN_QWEN35_STATIC_Q468_GEOMETRY.total_rows + row_axis = torch.arange(rows, dtype=torch.float64) / rows + offset = 0.0 if selector_profile == FROZEN_UNWEIGHTED_MSE_PROFILE else 0.125 + provisional = ComparatorAggregate( + selector_profile=selector_profile, # type: ignore[arg-type] + d4=4.0 + offset + row_axis, + d6=2.0 + offset + row_axis / 2, + d8=1.0 + offset + row_axis / 4, + family_sequence_counts=(("mbpp", 128), ("pg19", 16), ("ruler", 16)), + ruler_category_sequence_counts=tuple( + (category, 4) for category in calibration.RULER_CATEGORY_ORDER + ), + position_manifest_sha256=_hash(f"positions:{selector_profile}"), + sequence_score_manifest_sha256=_hash(f"sequences:{selector_profile}"), + identity_record_manifest_sha256=identity_manifest_sha256, + aggregate_scores_sha256="0" * 64, + ) + return replace( + provisional, + aggregate_scores_sha256=calibration._comparator_aggregate_score_sha256(provisional), + ) + + def test_calibration_capture_is_deterministic_and_resolver_compatible() -> None: first = capture.capture_identity_input(phase="calibration", source=FakeSource()) second = capture.capture_identity_input(phase="calibration", source=FakeSource()) @@ -726,6 +790,11 @@ def test_calibration_capture_is_deterministic_and_resolver_compatible() -> None: first, expected_revisions=resolver.FROZEN_DATASET_REVISIONS ) assert candidate["evidence"]["record_count"] == 160 + assert capture.CAPTURE_VERSION == resolver.RESOLVER_VERSION == 5 + assert first["schema"] == "recurquant.experiment013.identity-input.v5" + assert candidate["evidence"]["identity_schema"] == ( + "recurquant.experiment013.identity-candidate.v5" + ) counts = { family: sum(row["family"] == family for row in first["records"]) for family in resolver.DATASET_KEYS @@ -858,6 +927,9 @@ def test_public_calibration_materialization_is_the_exact_capture_with_tokens() - assert record["sequence_token_ids_sha256"] == capture._token_hash( sequence.sequence_token_ids ) + assert record["fisher_boundary"] == resolver.build_fisher_boundary_contract( + sequence.sequence_token_ids + ) assert record["sequence_length"] == len(sequence.sequence_token_ids) assert record["token_span"] == { "prefill_start": 0, @@ -889,6 +961,44 @@ def test_public_calibration_materialization_returns_no_raw_content() -> None: assert b"auxiliary_files" not in record_bytes assert b"source_payload" not in record_bytes assert b"formatted_payload" not in record_bytes + assert b'"input_token_ids":' not in record_bytes + assert b'"target_token_ids":' not in record_bytes + + +def test_materialized_sequence_recomputes_fisher_boundary_from_exact_tokens() -> None: + sequence = capture.materialize_calibration_identity_sequences(source=FakeSource()).sequences[0] + record = sequence.identity_record + record["fisher_boundary"]["input_token_ids_sha256"] = "0" * 64 + record["fisher_boundary"]["fisher_boundary_sha256"] = resolver.fisher_boundary_sha256( + record["fisher_boundary"] + ) + record["identity_record_sha256"] = resolver.identity_record_sha256(record) + + with pytest.raises(ValueError, match="tokens differ from their identity record"): + capture.MaterializedCalibrationSequence( + _identity_record_bytes=capture.canonical_json_bytes(record), + prompt_token_ids=sequence.prompt_token_ids, + target_token_ids=sequence.target_token_ids, + ) + + +def test_capture_rejects_fisher_sequences_shorter_than_three_tokens() -> None: + with pytest.raises(ValueError, match="at least three tokens"): + capture._base_record( + phase="calibration", + family="pg19", + canonical_id="short-sequence", + config="default", + seed=None, + configured_length=None, + ruler_category=None, + generator_receipt_sha256=None, + source_payload={"source": "fixture"}, + formatted_payload={"formatted": "fixture"}, + prompt_ids=(1, 2), + target_ids=(), + tokenizer_manifest_sha256="a" * 64, + ) @pytest.mark.parametrize( @@ -1040,12 +1150,56 @@ def test_stage_a_binding_is_derived_from_identity_scores_split_and_policies() -> method_id=STATIC_Q468_PRIMARY_METHOD, ) ) + comparator_bytes = build_frozen_comparator_score_artifact( + _frozen_comparator_aggregate( + FROZEN_UNWEIGHTED_MSE_PROFILE, + identity_manifest_sha256=full_identity_manifest, + ), + _frozen_comparator_aggregate( + FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE, + identity_manifest_sha256=full_identity_manifest, + ), + calibration_identity_sha256=identity.file_sha256, + ) + comparators = deserialize_comparator_score_artifact( + comparator_bytes, + expected_calibration_identity_sha256=identity.file_sha256, + ) + + def comparator_policy_bytes(profile: str, method_id: str) -> bytes: + selector = comparators.selectors[profile] + return serialize_static_rht_q468_policy( + build_static_rht_q468_policy( + selector.aggregate.d4, + selector.aggregate.d6, + selector.aggregate.d8, + geometry=FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + marginal_steps=FROZEN_STATIC_Q468_PRIMARY_STEPS, + calibration_manifest_sha256=(selector.aggregate.sequence_score_manifest_sha256), + identity_artifact_sha256=identity.file_sha256, + tokenizer_manifest_sha256=identity.tokenizer_manifest_sha256, + source_commit="f" * 40, + method_id=method_id, + ) + ) + + mse_policy_bytes = comparator_policy_bytes( + FROZEN_UNWEIGHTED_MSE_PROFILE, + STATIC_Q468_MSE_METHOD, + ) + fisher_policy_bytes = comparator_policy_bytes( + FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE, + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + ) binding_bytes = resolver.build_stage_a_calibration_binding_artifact( frozen_identity_artifact=identity_bytes, calibration_score_artifact=score_bytes, split_half_stability_artifact=split_bytes, static_k27030_policy_artifact=policy27030_bytes, static_k29334_policy_artifact=policy29334_bytes, + comparator_score_artifact=comparator_bytes, + static_fisher_k29334_policy_artifact=fisher_policy_bytes, + static_mse_k29334_policy_artifact=mse_policy_bytes, ) verified = resolver.deserialize_stage_a_calibration_binding_artifact(binding_bytes) @@ -1057,6 +1211,9 @@ def test_stage_a_binding_is_derived_from_identity_scores_split_and_policies() -> "split_half_stability_artifact_file_sha256": resolver.sha256_bytes(split_bytes), "static_k27030_policy_file_sha256": resolver.sha256_bytes(policy27030_bytes), "static_k29334_policy_file_sha256": resolver.sha256_bytes(policy29334_bytes), + "comparator_score_artifact_file_sha256": resolver.sha256_bytes(comparator_bytes), + "static_fisher_k29334_policy_file_sha256": resolver.sha256_bytes(fisher_policy_bytes), + "static_mse_k29334_policy_file_sha256": resolver.sha256_bytes(mse_policy_bytes), } tampered = json.loads(binding_bytes) @@ -1113,6 +1270,280 @@ def test_stage_a_capture_uses_exact_schedules_and_token_caps() -> None: ) +def test_stage_a_materialization_authenticates_exact_inventory_tokens_and_spans() -> None: + frozen_bytes = _frozen_stage_a_identity() + frozen = resolver.deserialize_frozen_stage_a_identity_artifact( + frozen_bytes, + calibration_binding_artifact=_binding(), + ) + source = FakeSource() + + materialized = capture.materialize_stage_a_identity_sequences( + source=source, + frozen_stage_a_identity_artifact=frozen_bytes, + calibration_binding_artifact=_binding(), + expected_frozen_stage_a_identity_file_sha256=resolver.sha256_bytes(frozen_bytes), + ) + + assert materialized.frozen_identity_file_sha256 == frozen.file_sha256 + assert materialized.frozen_identity_canonical_evidence_sha256 == ( + frozen.canonical_evidence_sha256 + ) + assert materialized.calibration_binding_file_sha256 == resolver.sha256_bytes(_binding()) + assert len(materialized.sequences) == len(materialized.by_identity_record_sha256) == 12 + assert [ + (sequence.identity_record["family"], sequence.identity_record["selection_rank"]) + for sequence in materialized.sequences + ] == [(family, rank) for family in ("pg19", "ruler", "humaneval_plus") for rank in range(4)] + assert capture.canonical_json_bytes(materialized.identity_records) == ( + capture.canonical_json_bytes( + tuple( + {name: record[name] for name in resolver.RECORD_FIELDS} for record in frozen.records + ) + ) + ) + + for sequence in materialized.sequences: + record = sequence.identity_record + span = record["token_span"] + assert record["identity_record_sha256"] == resolver.identity_record_sha256(record) + assert record["prompt_token_ids_sha256"] == capture._token_hash(sequence.prompt_token_ids) + assert record["target_token_ids_sha256"] == capture._token_hash(sequence.target_token_ids) + assert record["sequence_token_ids_sha256"] == capture._token_hash( + sequence.sequence_token_ids + ) + assert all( + len(record[field]) == 64 + for field in ( + "tokenizer_manifest_sha256", + "source_content_sha256", + "formatted_content_sha256", + ) + ) + assert span == { + "prefill_start": 0, + "prefill_stop": len(sequence.prompt_token_ids), + "scored_start": len(sequence.prompt_token_ids), + "scored_stop": len(sequence.sequence_token_ids), + "cache_exposed_start": len(sequence.prompt_token_ids) + 1, + "cache_exposed_stop": len(sequence.sequence_token_ids), + } + assert len(sequence.target_token_ids) >= 2 + assert sequence.cache_exposed_transition_count == len(sequence.target_token_ids) - 1 + assert materialized.lookup(sequence.identity_record_sha256) is sequence + + pg19 = [ + sequence + for sequence in materialized.sequences + if sequence.identity_record["family"] == "pg19" + ] + assert all(len(sequence.prompt_token_ids) == 4_096 for sequence in pg19) + assert all(len(sequence.target_token_ids) == 128 for sequence in pg19) + assert all(sequence.cache_exposed_transition_count == 127 for sequence in pg19) + + ruler_inventory = { + ( + sequence.identity_record["ruler_category"], + sequence.identity_record["config"], + sequence.identity_record["configured_length"], + sequence.identity_record["seed"], + ) + for sequence in materialized.sequences + if sequence.identity_record["family"] == "ruler" + } + assert ruler_inventory == set(resolver.RULER_STAGE_A_SCHEDULE) + for sequence in materialized.sequences: + record = sequence.identity_record + if record["family"] != "ruler": + continue + receipt = source.ruler_receipt( + category=record["ruler_category"], + config=record["config"], + configured_length=record["configured_length"], + seed=record["seed"], + ) + target, _semantics = capture._ruler_stage_a_target( + category=record["ruler_category"], + config=record["config"], + outputs=receipt["outputs"], + ) + assert sequence.prompt_token_ids == tuple( + FakeTokenizer().encode( + receipt["input"] + receipt["answer_prefix"], + add_special_tokens=False, + ) + ) + assert sequence.target_token_ids == tuple( + FakeTokenizer().encode(target, add_special_tokens=False) + ) + + humaneval_projection = {row.canonical_id: row.offset for row in source.humaneval_projection()} + for sequence in materialized.sequences: + record = sequence.identity_record + if record["family"] != "humaneval_plus": + continue + row = source.humaneval_row( + offset=humaneval_projection[record["canonical_id"]], + expected_task_id=record["canonical_id"], + ) + assert sequence.prompt_token_ids == tuple( + FakeTokenizer().encode(row["prompt"], add_special_tokens=True) + ) + assert sequence.target_token_ids == tuple( + FakeTokenizer().encode(row["canonical_solution"], add_special_tokens=False)[:128] + ) + assert all( + len(sequence.target_token_ids) <= 128 + for sequence in materialized.sequences + if sequence.identity_record["family"] == "humaneval_plus" + ) + assert len(materialized.token_sequence_manifest_sha256) == 64 + assert source.head_calls == 2 + + +def test_stage_a_materialization_accepts_exact_two_token_target() -> None: + source = FakeSource() + source.receipt_mutator = lambda receipt: ( + receipt.update({"outputs": ["xy"]}) + if receipt["config"] == "qa_1" and receipt["seed"] == 2_339 + else None + ) + frozen_bytes = _frozen_stage_a_identity(source) + + materialized = capture.materialize_stage_a_identity_sequences( + source=source, + frozen_stage_a_identity_artifact=frozen_bytes, + calibration_binding_artifact=_binding(), + ) + + qa = next( + sequence + for sequence in materialized.sequences + if sequence.identity_record["config"] == "qa_1" + ) + assert len(qa.target_token_ids) == 2 + assert qa.cache_exposed_transition_count == 1 + assert qa.identity_record["token_span"]["cache_exposed_start"] == ( + qa.identity_record["token_span"]["scored_start"] + 1 + ) + + +def test_stage_a_materialization_rejects_candidate_and_drift_before_use() -> None: + captured = capture.capture_identity_input( + phase="stage_a", + source=FakeSource(), + calibration_binding=_binding(), + ) + candidate = resolver.build_candidate( + captured, + expected_revisions=resolver.FROZEN_DATASET_REVISIONS, + calibration_binding_artifact=_binding(), + ) + source = FakeSource() + with pytest.raises(ValueError, match="frozen Stage-A identity"): + capture.materialize_stage_a_identity_sequences( + source=source, + frozen_stage_a_identity_artifact=resolver.canonical_json_bytes(candidate), + calibration_binding_artifact=_binding(), + ) + assert source.accesses == [] + + frozen_bytes = _frozen_stage_a_identity() + changed = FakeSource() + original_humaneval_row = changed.humaneval_row + + def changed_humaneval_row(*, offset: int, expected_task_id: str) -> dict[str, Any]: + row = original_humaneval_row(offset=offset, expected_task_id=expected_task_id) + row["canonical_solution"] += "\n# authenticated-content-drift" + return row + + changed.humaneval_row = changed_humaneval_row # type: ignore[method-assign] + with pytest.raises(ValueError, match="authenticated identity"): + capture.materialize_stage_a_identity_sequences( + source=changed, + frozen_stage_a_identity_artifact=frozen_bytes, + calibration_binding_artifact=_binding(), + ) + + +def test_stage_a_materialization_rejects_missing_duplicate_reordered_and_tampered() -> None: + frozen_bytes = _frozen_stage_a_identity() + materialized = capture.materialize_stage_a_identity_sequences( + source=FakeSource(), + frozen_stage_a_identity_artifact=frozen_bytes, + calibration_binding_artifact=_binding(), + ) + common = { + "tokenizer_manifest_sha256": materialized.tokenizer_manifest_sha256, + "capture_input_sha256": materialized.capture_input_sha256, + "frozen_identity_file_sha256": materialized.frozen_identity_file_sha256, + "frozen_identity_canonical_evidence_sha256": ( + materialized.frozen_identity_canonical_evidence_sha256 + ), + "calibration_binding_file_sha256": materialized.calibration_binding_file_sha256, + } + + with pytest.raises(ValueError, match="exactly 12"): + capture.StageAIdentityMaterialization( + sequences=materialized.sequences[:-1], + **common, + ) + with pytest.raises(ValueError, match="duplicate identities"): + capture.StageAIdentityMaterialization( + sequences=materialized.sequences[:-1] + (materialized.sequences[0],), + **common, + ) + with pytest.raises(ValueError, match="ordered by family then rank"): + capture.StageAIdentityMaterialization( + sequences=(materialized.sequences[1], materialized.sequences[0]) + + materialized.sequences[2:], + **common, + ) + + sequence = materialized.sequences[0] + tampered_record = sequence.identity_record + tampered_record["target_token_ids_sha256"] = "0" * 64 + tampered_record["identity_record_sha256"] = resolver.identity_record_sha256(tampered_record) + with pytest.raises(ValueError, match="only by authenticated v5 materialization"): + capture.MaterializedStageASequence( + _identity_record_bytes=capture.canonical_json_bytes(sequence.identity_record), + prompt_token_ids=sequence.prompt_token_ids, + target_token_ids=sequence.target_token_ids, + _authentication_seal=object(), + ) + with pytest.raises(ValueError, match="tokens differ from their identity record"): + capture.MaterializedStageASequence( + _identity_record_bytes=capture.canonical_json_bytes(tampered_record), + prompt_token_ids=sequence.prompt_token_ids, + target_token_ids=sequence.target_token_ids, + _authentication_seal=capture._STAGE_A_MATERIALIZATION_AUTHENTICATION_SEAL, + ) + + +def test_stage_a_materialization_identity_records_are_content_redacted() -> None: + frozen_bytes = _frozen_stage_a_identity() + materialized = capture.materialize_stage_a_identity_sequences( + source=FakeSource(), + frozen_stage_a_identity_artifact=frozen_bytes, + calibration_binding_artifact=_binding(), + ) + records = capture.canonical_json_bytes(materialized.identity_records) + + for forbidden in ( + b'"answer_prefix":', + b'"canonical_solution":', + b'"formatted_payload":', + b'"input":', + b'"outputs":', + b'"prompt":', + b'"source_payload":', + b'"text":', + b'"prompt_token_ids":', + b'"target_token_ids":', + ): + assert forbidden not in records + + def test_ruler_stage_a_target_includes_all_required_outputs_and_selects_one_qa_alternative() -> ( None ): @@ -1189,9 +1620,7 @@ def test_ruler_receipt_required_output_cardinality_and_uniqueness_fail_closed() ), ( "vt", - lambda receipt: receipt.update( - {"outputs": ["ZZZZZ", *receipt["outputs"][1:]]} - ), + lambda receipt: receipt.update({"outputs": ["ZZZZZ", *receipt["outputs"][1:]]}), "required answer is absent", ), ], @@ -1412,15 +1841,30 @@ def test_live_tokenizer_load_uses_only_authenticated_files_from_an_isolated_dire observed: dict[str, object] = {} class FakeApi: + def __init__(self, *, token: bool, endpoint: str) -> None: + assert token is False + assert endpoint == "https://huggingface.co" + @staticmethod - def list_repo_files(_repo_id: str, *, revision: str) -> list[str]: + def list_repo_files(_repo_id: str, *, revision: str, token: bool) -> list[str]: assert revision == resolver.PRIMARY_MODEL_REVISION + assert token is False return [*expected, "tokenizer.model", "model.safetensors"] - def fake_download(*, repo_id: str, filename: str, revision: str, cache_dir: Path) -> str: + def fake_download( + *, + repo_id: str, + filename: str, + revision: str, + cache_dir: Path, + token: bool, + endpoint: str, + ) -> str: assert repo_id == resolver.PRIMARY_MODEL_ID assert revision == resolver.PRIMARY_MODEL_REVISION assert cache_dir == (tmp_path / "cache").resolve() + assert token is False + assert endpoint == "https://huggingface.co" return str(snapshot / filename) class IsolatedTokenizer: @@ -1461,6 +1905,7 @@ def from_pretrained(cls, path: Path, **kwargs: object) -> IsolatedTokenizer: assert observed["path"] != snapshot assert observed["kwargs"] == { "local_files_only": True, + "token": False, "trust_remote_code": False, } assert not Path(observed["path"]).exists() @@ -1607,6 +2052,10 @@ def test_execution_artifact_decoders_run_before_file_hash_binding( "profile": experiment013_source.EXPERIMENT013_SOURCE_MANIFEST_PROFILE, "object_format": "sha1", "source_commit": "a" * 40, + "git_executable": { + "sha256": capture.sha256_bytes(FIXTURE_GIT_EXECUTABLE.read_bytes()), + "size_bytes": FIXTURE_GIT_EXECUTABLE.stat().st_size, + }, "repository_binding": { "schema": experiment013_source.EXPERIMENT013_REPOSITORY_BINDING_SCHEMA, "worktree_layout": "primary", @@ -1683,6 +2132,7 @@ def test_point_of_use_authentication_rechecks_source_runtime_modules_and_model( { "base_runtime_root": tmp_path / "runtime", "staged_interpreter": tmp_path / "runtime" / "python.exe", + "git_executable": FIXTURE_GIT_EXECUTABLE, "package_runtime_roots": {"packages": package_root}, "package_import_paths": {"packages": "Lib/site-packages"}, } @@ -1693,15 +2143,19 @@ def test_point_of_use_authentication_rechecks_source_runtime_modules_and_model( "model_file_manifest_file_sha256": b"model\n", "parquet_materialization_manifest_file_sha256": b"parquet\n", } - bindings = { - field: capture.sha256_bytes(data) for field, data in sorted(artifacts.items()) - } + bindings = {field: capture.sha256_bytes(data) for field, data in sorted(artifacts.items())} source_manifest = {"paths": []} class SourceModule: @staticmethod - def verify_experiment013_source_manifest(manifest: Any, *, repo_root: Path) -> Any: + def verify_experiment013_source_manifest( + manifest: Any, + *, + repo_root: Path, + git_executable: Path, + ) -> Any: assert repo_root == REPOSITORY_ROOT + assert git_executable == FIXTURE_GIT_EXECUTABLE events.append("source") return dict(manifest) @@ -1729,15 +2183,15 @@ def authenticate_calibration_runtime( base_runtime_root: Path, package_roots: Any, interpreter_path: Path, + git_executable_path: Path, ) -> Any: assert base_runtime_root == runtime_context.base_runtime_root assert package_roots == runtime_context.package_runtime_roots assert interpreter_path == runtime_context.staged_interpreter + assert git_executable_path == runtime_context.git_executable events.append("runtime") return SimpleNamespace( - manifest_file_sha256=bindings[ - "calibration_runtime_manifest_file_sha256" - ] + manifest_file_sha256=bindings["calibration_runtime_manifest_file_sha256"] ) @staticmethod @@ -1746,12 +2200,14 @@ def capture_model_file_manifest_from_hub( revision: str, *, transformers_version: str, + token: bool, ) -> bytes: assert (model_id, revision, transformers_version) == ( resolver.PRIMARY_MODEL_ID, resolver.PRIMARY_MODEL_REVISION, resolver.TRANSFORMERS_VERSION, ) + assert token is False events.append("model") return artifacts["model_file_manifest_file_sha256"] @@ -1819,9 +2275,7 @@ def test_loaded_calibration_runner_source_is_bound_to_source_manifest() -> None: "paths": [ { "path": "scripts/run_static_q468_calibration.py", - "raw_sha256": capture.sha256_bytes( - capture.CALIBRATION_RUNNER_PATH.read_bytes() - ), + "raw_sha256": capture.sha256_bytes(capture.CALIBRATION_RUNNER_PATH.read_bytes()), } ] } @@ -1838,9 +2292,7 @@ def test_loaded_calibration_runner_source_is_bound_to_source_manifest() -> None: lambda context: context.update({"extra": Path("unused")}), lambda context: context.update({"base_runtime_root": "not-a-path"}), lambda context: context.update({"package_runtime_roots": {"packages": False}}), - lambda context: context.update( - {"package_import_paths": {"packages": "../site-packages"}} - ), + lambda context: context.update({"package_import_paths": {"packages": "../site-packages"}}), lambda context: context.update( {"package_import_paths": {"packages": "Lib\\site-packages"}} ), @@ -1879,6 +2331,8 @@ def test_cli_requires_all_four_execution_artifact_files(tmp_path: Path) -> None: str(tmp_path / "runtime"), "--staged-interpreter", str(tmp_path / "runtime" / "python.exe"), + "--git-executable", + str(FIXTURE_GIT_EXECUTABLE), "--package-root", f"packages={tmp_path / 'packages'}", "--package-import-path", @@ -1894,13 +2348,19 @@ def test_live_source_probes_only_the_frozen_objects( observed_urls: list[str] = [] class FakeApi: + def __init__(self, *, token: bool, endpoint: str) -> None: + assert token is False + assert endpoint == "https://huggingface.co" + @staticmethod - def model_info(repo_id: str, *, revision: str) -> Any: + def model_info(repo_id: str, *, revision: str, token: bool) -> Any: + assert token is False observed_hf.append(("model", repo_id, revision)) return SimpleNamespace(sha=revision) @staticmethod - def dataset_info(repo_id: str, *, revision: str) -> Any: + def dataset_info(repo_id: str, *, revision: str, token: bool) -> Any: + assert token is False observed_hf.append(("dataset", repo_id, revision)) return SimpleNamespace(sha=revision) @@ -1947,3 +2407,412 @@ def test_required_ruler_receipt_inventory_is_exact_and_unique() -> None: assert sum(item["phase"] == "stage_a" for item in receipts) == 4 assert receipts[0]["filename"] == ("retrieval__niah_multiquery__l2048__s12339.json") assert receipts[-1]["filename"] == ("question_answering__qa_1__l4096__s2339.json") + + +def _parquet_bytes(columns: dict[str, list[str]]) -> bytes: + import pyarrow as arrow + import pyarrow.parquet as parquet + + sink = arrow.BufferOutputStream() + parquet.write_table(arrow.table(columns), sink) + return sink.getvalue().to_pybytes() + + +def _fixture_stage_a_input_bundle( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> SimpleNamespace: + from recurquant import experiment013_parquet + + frozen_bytes = _frozen_stage_a_identity() + frozen = resolver.deserialize_frozen_stage_a_identity_artifact( + frozen_bytes, + calibration_binding_artifact=_binding(), + ) + parquet_payloads = { + "pg19": _parquet_bytes( + { + "url": [ + f"http://www.gutenberg.org/ebooks/{200_000 + index}" for index in range(50) + ], + "text": [f"fixture PG19 text {index}" for index in range(50)], + } + ), + "humaneval_plus": _parquet_bytes( + { + "task_id": [f"HumanEval/{index}" for index in range(164)], + "prompt": [f"def task_{index}(x):\n" for index in range(164)], + "canonical_solution": [" return x\n" for _ in range(164)], + } + ), + } + selected: list[tuple[Any, Any]] = [] + datasets: dict[str, Any] = {} + for dataset, file in capture._bundle_expected_parquet_files(): + payload = parquet_payloads[dataset.key] + lfs_sha256 = capture.sha256_bytes(payload) + size_bytes = len(payload) + git_blob_oid = capture._git_blob_sha1( + capture._bundle_lfs_pointer_bytes( + sha256=lfs_sha256, + size_bytes=size_bytes, + ) + ) + fixture_file = replace( + file, + size_bytes=size_bytes, + git_blob_oid=git_blob_oid, + lfs_sha256=lfs_sha256, + lfs_size_bytes=size_bytes, + ) + fixture_dataset = replace( + dataset, + selected_splits=(file.logical_split,), + files=(fixture_file,), + ) + selected.append((fixture_dataset, fixture_file)) + datasets[dataset.key] = fixture_dataset + frozen_selected = tuple(selected) + monkeypatch.setattr( + capture, + "_bundle_expected_parquet_files", + lambda: frozen_selected, + ) + fixture_manifest = SimpleNamespace(dataset=lambda key: datasets[key]) + monkeypatch.setattr( + experiment013_parquet, + "load_experiment013_parquet_manifest", + lambda *_args, **_kwargs: fixture_manifest, + ) + + bundle_root = tmp_path / "stage-a-input-bundle" + bundle_root.mkdir() + records: list[dict[str, Any]] = [] + capture._bundle_add_object( + bundle_root, + records, + role="model_hub_manifest", + source_id=resolver.PRIMARY_MODEL_ID, + revision=resolver.PRIMARY_MODEL_REVISION, + logical_path="model-file-manifest.json", + payload=FIXTURE_EXECUTION_ARTIFACTS["model_file_manifest_file_sha256"], + ) + tokenizer_payloads = { + "tokenizer.json": b"fixture-tokenizer", + "tokenizer_config.json": b"fixture-tokenizer-config", + } + assert set(tokenizer_payloads) == set(capture.RULER_EXPECTED_TOKENIZER_ASSETS) + for name, payload in sorted(tokenizer_payloads.items()): + capture._bundle_add_object( + bundle_root, + records, + role="tokenizer", + source_id=resolver.PRIMARY_MODEL_ID, + revision=resolver.PRIMARY_MODEL_REVISION, + logical_path=name, + payload=payload, + ) + for path, payload in sorted(_fake_generator_files().items()): + capture._bundle_add_object( + bundle_root, + records, + role="ruler_generator", + source_id=resolver.RULER_SOURCE_ID, + revision=resolver.RULER_REVISION, + logical_path=path, + payload=payload, + git_blob_oid=capture.RULER_GENERATOR_GIT_BLOBS[path], + ) + capture._bundle_add_object( + bundle_root, + records, + role="ruler_generation_manifest", + source_id=resolver.RULER_SOURCE_ID, + revision=resolver.RULER_REVISION, + logical_path="generation-manifest.json", + payload=b"fixture generation manifest bytes\n", + ) + for item in capture.required_ruler_receipts(): + filename = str(item["filename"]) + capture._bundle_add_object( + bundle_root, + records, + role="ruler_receipt", + source_id=resolver.RULER_SOURCE_ID, + revision=resolver.RULER_REVISION, + logical_path=filename, + payload=f"fixture opaque receipt {filename}\n".encode(), + ) + snapshots: list[dict[str, Any]] = [] + parquet_logicals: dict[str, str] = {} + for dataset, file in frozen_selected: + logical = f"{dataset.key}/{file.logical_split}/{file.immutable_path}" + parquet_logicals[dataset.key] = logical + capture._bundle_add_object( + bundle_root, + records, + role="parquet", + source_id=dataset.dataset_id, + revision=dataset.conversion_revision, + logical_path=logical, + payload=parquet_payloads[dataset.key], + git_blob_oid=file.git_blob_oid, + lfs_sha256=file.lfs_sha256, + ) + snapshots.append( + { + "dataset_key": dataset.key, + "dataset_id": dataset.dataset_id, + "source_revision": dataset.source_revision, + "conversion_revision": dataset.conversion_revision, + "files": [ + { + "path": file.immutable_path, + "git_blob_oid": file.git_blob_oid, + "lfs_sha256": file.lfs_sha256, + "size_bytes": file.size_bytes, + } + ], + } + ) + records.sort(key=lambda item: (item["role"], item["source_id"], item["logical_path"])) + manifest = { + "schema": capture.STAGE_A_INPUT_BUNDLE_SCHEMA, + "phase": "stage_a", + "capture_version": capture.CAPTURE_VERSION, + "staging_profile": capture.STAGE_A_INPUT_BUNDLE_PROFILE, + "frozen_identity_file_sha256": frozen.file_sha256, + "calibration_binding_file_sha256": capture.sha256_bytes(_binding()), + "execution_bindings": dict(frozen.execution_bindings), + "source_heads": dict(capture.EXPECTED_SOURCE_HEADS), + "model_hub_manifest_file_sha256": capture.sha256_bytes( + FIXTURE_EXECUTION_ARTIFACTS["model_file_manifest_file_sha256"] + ), + "parquet_hub_snapshots": snapshots, + "objects": records, + } + manifest_path = bundle_root / capture.STAGE_A_INPUT_BUNDLE_FILENAME + manifest_path.write_bytes(capture.canonical_json_bytes(manifest)) + bundle = capture.authenticate_stage_a_input_bundle( + bundle_root, + frozen_stage_a_identity_artifact=frozen_bytes, + calibration_binding_artifact=_binding(), + execution_binding_artifacts=FIXTURE_EXECUTION_ARTIFACTS, + ) + return SimpleNamespace( + bundle=bundle, + root=bundle_root, + manifest_path=manifest_path, + frozen_bytes=frozen_bytes, + records=records, + selected=frozen_selected, + parquet_logicals=parquet_logicals, + ) + + +def _authenticate_fixture_bundle(fixture: SimpleNamespace) -> Any: + return capture.authenticate_stage_a_input_bundle( + fixture.root, + frozen_stage_a_identity_artifact=fixture.frozen_bytes, + calibration_binding_artifact=_binding(), + execution_binding_artifacts=FIXTURE_EXECUTION_ARTIFACTS, + ) + + +def test_stage_a_bundle_authenticates_both_lfs_pointer_and_payload_identities( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fixture = _fixture_stage_a_input_bundle(tmp_path, monkeypatch) + + assert len(fixture.selected) == 2 + for dataset, file in fixture.selected: + record = fixture.bundle.objects[("parquet", fixture.parquet_logicals[dataset.key])] + assert record["sha256"] == file.lfs_sha256 + assert record["size_bytes"] == file.size_bytes + assert record["git_blob_oid"] == capture._git_blob_sha1( + capture._bundle_lfs_pointer_bytes( + sha256=file.lfs_sha256, + size_bytes=file.size_bytes, + ) + ) + + collision_root = tmp_path / "bad-lfs" + collision_root.mkdir() + payload = b"valid LFS payload bytes\n" + with pytest.raises(ValueError, match="Git blob"): + capture._bundle_add_object( + collision_root, + [], + role="parquet", + source_id="fixture/dataset", + revision="a" * 40, + logical_path="default/test/0000.parquet", + payload=payload, + git_blob_oid="0" * 40, + lfs_sha256=capture.sha256_bytes(payload), + ) + + +def test_stage_a_bundle_rejects_object_tamper_and_extra_inventory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fixture = _fixture_stage_a_input_bundle(tmp_path, monkeypatch) + logical = fixture.parquet_logicals["pg19"] + record = fixture.bundle.objects[("parquet", logical)] + object_path = fixture.root / Path(str(record["relative_path"])) + object_path.write_bytes(object_path.read_bytes() + b"tampered") + with pytest.raises(ValueError, match="changed"): + _authenticate_fixture_bundle(fixture) + + +@pytest.mark.parametrize("location", ("root", "objects")) +def test_stage_a_bundle_rejects_extra_filesystem_entry( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + location: str, +) -> None: + fixture = _fixture_stage_a_input_bundle(tmp_path, monkeypatch) + parent = fixture.root if location == "root" else fixture.root / "objects" + (parent / "unexpected").write_bytes(b"unexpected\n") + with pytest.raises(ValueError, match="filesystem inventory drifted"): + _authenticate_fixture_bundle(fixture) + + +def test_stage_a_bundle_rejects_forged_semantic_provenance( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fixture = _fixture_stage_a_input_bundle(tmp_path, monkeypatch) + manifest = json.loads(fixture.manifest_path.read_bytes()) + record = next(item for item in manifest["objects"] if item["role"] == "model_hub_manifest") + record["revision"] = "f" * 40 + fixture.manifest_path.write_bytes(capture.canonical_json_bytes(manifest)) + + with pytest.raises(ValueError, match="frozen semantics"): + _authenticate_fixture_bundle(fixture) + + +def test_stage_a_bundle_rejects_linked_object( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fixture = _fixture_stage_a_input_bundle(tmp_path, monkeypatch) + record = next(iter(fixture.bundle.objects.values())) + object_path = fixture.root / Path(str(record["relative_path"])) + outside = tmp_path / "outside-object" + outside.write_bytes(object_path.read_bytes()) + object_path.unlink() + try: + object_path.symlink_to(outside) + except OSError as error: + pytest.skip(f"symlink creation is unavailable: {type(error).__name__}") + + with pytest.raises(ValueError, match="link|reparse"): + _authenticate_fixture_bundle(fixture) + + +def test_stage_a_bundle_path_status_errors_fail_closed(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="cannot authenticate"): + capture._bundle_is_link_or_reparse(tmp_path / "absent") + + +def test_existing_stage_a_bundle_still_requires_runtime_auth_and_isolation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fixture = _fixture_stage_a_input_bundle(tmp_path, monkeypatch) + ruler_root = tmp_path / "receipts" + ruler_root.mkdir() + + with pytest.raises(ValueError): + capture.stage_stage_a_input_bundle( + bundle_root=fixture.root, + cache_dir=tmp_path / "cache", + ruler_receipt_dir=ruler_root, + frozen_stage_a_identity_artifact=fixture.frozen_bytes, + calibration_binding_artifact=_binding(), + execution_binding_artifacts=FIXTURE_EXECUTION_ARTIFACTS, + runtime_authentication_context={"invalid": "context"}, + ) + with pytest.raises(ValueError, match="must not be nested"): + capture.stage_stage_a_input_bundle( + bundle_root=fixture.root, + cache_dir=tmp_path, + ruler_receipt_dir=ruler_root, + frozen_stage_a_identity_artifact=fixture.frozen_bytes, + calibration_binding_artifact=_binding(), + execution_binding_artifacts=FIXTURE_EXECUTION_ARTIFACTS, + runtime_authentication_context=FIXTURE_RUNTIME_CONTEXT, + ) + + +def test_staged_capture_source_is_offline_and_reads_only_authenticated_objects( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fixture = _fixture_stage_a_input_bundle(tmp_path, monkeypatch) + for name in capture._STAGE_A_FORBIDDEN_CREDENTIAL_ENVIRONMENT: + monkeypatch.delenv(name, raising=False) + for name, value in capture._STAGE_A_OFFLINE_ENVIRONMENT.items(): + monkeypatch.setenv(name, value) + + def reject_network(*_args: Any, **_kwargs: Any) -> Any: + raise AssertionError("offline staged capture attempted network access") + + monkeypatch.setattr(capture.urllib.request, "urlopen", reject_network) + source = capture.StagedCaptureSource(fixture.bundle) + assert source.source_heads() == capture.EXPECTED_SOURCE_HEADS + assert set(source.ruler_generator_files()) == set(capture.RULER_GENERATOR_GIT_BLOBS) + assert len(source.pg19_projection("validation")) == 50 + pg19 = source.pg19_row( + "validation", + offset=0, + expected_url="http://www.gutenberg.org/ebooks/200000", + ) + assert pg19["text"] == "fixture PG19 text 0" + assert len(source.humaneval_projection()) == 164 + humaneval = source.humaneval_row(offset=0, expected_task_id="HumanEval/0") + assert humaneval["canonical_solution"] == " return x\n" + + class FixtureAutoTokenizer: + @staticmethod + def from_pretrained(path: Path, **kwargs: Any) -> FakeTokenizer: + assert kwargs == { + "local_files_only": True, + "trust_remote_code": False, + "token": False, + } + assert {item.name for item in path.iterdir()} == set( + capture.RULER_EXPECTED_TOKENIZER_ASSETS + ) + return FakeTokenizer() + + monkeypatch.setitem( + sys.modules, + "transformers", + SimpleNamespace(AutoTokenizer=FixtureAutoTokenizer), + ) + tokenizer = source.tokenizer_material() + assert tokenizer.model_weights_loaded is False + assert set(tokenizer.files) == set(capture.RULER_EXPECTED_TOKENIZER_ASSETS) + + +def test_staged_capture_source_rejects_missing_offline_flag_or_credential( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fixture = _fixture_stage_a_input_bundle(tmp_path, monkeypatch) + for name in capture._STAGE_A_FORBIDDEN_CREDENTIAL_ENVIRONMENT: + monkeypatch.delenv(name, raising=False) + for name, value in capture._STAGE_A_OFFLINE_ENVIRONMENT.items(): + monkeypatch.setenv(name, value) + monkeypatch.delenv("HF_HUB_OFFLINE") + with pytest.raises(RuntimeError, match="offline-mode flags"): + capture.StagedCaptureSource(fixture.bundle) + + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + monkeypatch.setenv("HF_TOKEN", "forbidden-fixture-token") + with pytest.raises(RuntimeError, match="forbidden credential"): + capture.StagedCaptureSource(fixture.bundle) diff --git a/tests/test_experiment013_calibration_api.py b/tests/test_experiment013_calibration_api.py index 0e585c7..08bf8e7 100644 --- a/tests/test_experiment013_calibration_api.py +++ b/tests/test_experiment013_calibration_api.py @@ -11,6 +11,7 @@ AuthenticatedModelFiles, AuthenticatedSequence, CalibrationAdapter, + FisherStepObservation, ModelFileIdentity, StepObservation, ) @@ -85,6 +86,7 @@ def test_context_normalizes_and_freezes_runtime_authentication_paths() -> None: def test_adapter_facing_values_have_one_stable_importable_identity() -> None: sequence = AuthenticatedSequence((1, 2), "a" * 64, "b" * 64, None, "c" * 64) observation = StepObservation(0, 1, (0,), object(), None, (1,)) + fisher = FisherStepObservation(0, 1, 2, 2, 3, observation, object(), object(), 1.25) file_identity = ModelFileIdentity( "model.safetensors", 1, @@ -105,6 +107,8 @@ def test_adapter_facing_values_have_one_stable_importable_identity() -> None: assert sequence.token_ids == (1, 2) assert observation.successful_kernel_calls_per_layer == (1,) + assert (fisher.boundary_position, fisher.input_position, fisher.target_position) == (0, 1, 2) + assert fisher.step_observation is observation assert authenticated.files == (file_identity,) @@ -150,6 +154,10 @@ def step_token(self, model: object, **kwargs: object) -> object: del model, kwargs return object() + def step_token_with_fisher(self, model: object, **kwargs: object) -> object: + del model, kwargs + return object() + def end_sequence(self, model: object, record: object) -> None: del model, record diff --git a/tests/test_experiment013_parquet.py b/tests/test_experiment013_parquet.py index 18d5f5b..a7ed5be 100644 --- a/tests/test_experiment013_parquet.py +++ b/tests/test_experiment013_parquet.py @@ -3,9 +3,11 @@ import copy import hashlib import json +import sys from collections import defaultdict from dataclasses import replace from pathlib import Path +from types import SimpleNamespace import pytest @@ -169,9 +171,7 @@ def _parquet_backend( manifest = load_experiment013_parquet_manifest() dataset = manifest.dataset(dataset_key) schema_columns = ( - ("url", "text") - if dataset_key == "pg19" - else ("prompt", "task_id", "canonical_solution") + ("url", "text") if dataset_key == "pg19" else ("prompt", "task_id", "canonical_solution") ) layouts: dict[str, ParquetFileLayout] = {} for file in dataset.files: @@ -394,6 +394,71 @@ def test_implementation_contains_no_dataset_viewer_or_mutable_parquet_endpoint() assert "@~parquet" not in source +def test_hub_metadata_backend_forces_public_official_endpoint( + monkeypatch: pytest.MonkeyPatch, +) -> None: + revision = "a" * 40 + path = "data/train-00000-of-00001.parquet" + calls: list[tuple[str, object]] = [] + + class FakeApi: + def __init__(self, *, token: bool, endpoint: str) -> None: + calls.append(("api", (token, endpoint))) + assert token is False + assert endpoint == "https://huggingface.co" + + @staticmethod + def dataset_info(**kwargs: object) -> object: + calls.append(("dataset_info", dict(kwargs))) + assert kwargs["token"] is False + return SimpleNamespace( + sha=revision, + siblings=( + SimpleNamespace( + rfilename=path, + blob_id="b" * 40, + lfs={"sha256": "c" * 64, "size": 123}, + ), + ), + ) + + def fake_url(**kwargs: object) -> str: + calls.append(("url", dict(kwargs))) + assert kwargs["endpoint"] == "https://huggingface.co" + return "https://huggingface.co/public-object" + + def fake_head(url: str, *, token: bool) -> object: + calls.append(("head", (url, token))) + assert token is False + return SimpleNamespace( + commit_hash=revision, + size=123, + etag="c" * 64, + ) + + monkeypatch.setitem( + sys.modules, + "huggingface_hub", + SimpleNamespace( + HfApi=FakeApi, + get_hf_file_metadata=fake_head, + hf_hub_url=fake_url, + ), + ) + backend = parquet_module.HuggingFaceHubMetadataBackend(token=False) + + assert backend.resolve_dataset_revision(repo_id="public/dataset", revision=revision) == revision + snapshot = backend.snapshot_parquet_files( + repo_id="public/dataset", + revision=revision, + paths=(path,), + ) + + assert snapshot.commit_hash == revision + assert snapshot.files[0].lfs_sha256 == "c" * 64 + assert {name for name, _value in calls} >= {"api", "dataset_info", "url", "head"} + + def test_bulk_projection_is_ordered_counted_immutable_and_authenticated_once() -> None: events: list[str] = [] manifest = load_experiment013_parquet_manifest() diff --git a/tests/test_experiment013_qwen35_adapter.py b/tests/test_experiment013_qwen35_adapter.py index 00dd791..421c404 100644 --- a/tests/test_experiment013_qwen35_adapter.py +++ b/tests/test_experiment013_qwen35_adapter.py @@ -3,6 +3,8 @@ import gc import hashlib import json +import os +import subprocess import sys import types import weakref @@ -13,6 +15,7 @@ import torch from recurquant import experiment013_qwen35_adapter as adapter_module +from recurquant import experiment013_source as source_module from recurquant.experiment013_calibration_api import ( AdapterConstructionContext, AuthenticatedModelFiles, @@ -76,7 +79,7 @@ def _fake_capture_binding( def _source_manifest_bytes(payload: bytes) -> bytes: return json.dumps( { - "schema": adapter_module.SOURCE_MANIFEST_SCHEMA, + "schema": source_module.EXPERIMENT013_SOURCE_MANIFEST_SCHEMA, "paths": [ { "path": adapter_module.CAPTURE_SOURCE_PATH, @@ -87,6 +90,40 @@ def _source_manifest_bytes(payload: bytes) -> bytes: ).encode("utf-8") +def _git(root: Path, *arguments: str) -> None: + process = subprocess.run( + ["git", *arguments], + cwd=root, + check=False, + capture_output=True, + text=True, + env=os.environ.copy(), + ) + if process.returncode != 0: + raise AssertionError(process.stderr or process.stdout) + + +def _canonical_v2_source_manifest_bytes(root: Path, payload: bytes) -> bytes: + root.mkdir(parents=True) + _git(root, "init", "-b", "main") + _git(root, "config", "user.name", "Experiment 013 Adapter Test") + _git(root, "config", "user.email", "experiment013-adapter@example.invalid") + (root / ".gitattributes").write_text("* text eol=lf\n", encoding="utf-8", newline="\n") + (root / ".gitignore").write_text("artifacts/\n", encoding="utf-8", newline="\n") + for relative in source_module.EXPERIMENT013_SOURCE_PATHS: + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes( + payload + if relative == adapter_module.CAPTURE_SOURCE_PATH + else f"adapter-v2-fixture:{relative}\n".encode() + ) + _git(root, "add", "--", ".") + _git(root, "commit", "-m", "adapter v2 source fixture") + manifest = source_module.capture_experiment013_source_manifest(root) + return source_module.canonical_experiment013_source_manifest_bytes(manifest) + + def _identity_record(index: int, token_ids: tuple[int, ...]) -> dict[str, object]: return { "identity_record_sha256": _sha(index + 1), @@ -258,6 +295,39 @@ def test_capture_loader_executes_manifest_bound_bytes_without_leaving_a_module( adapter_module._verify_capture_binding(binding) +def test_capture_loader_accepts_canonical_v2_source_manifest(tmp_path: Path) -> None: + root = tmp_path / "repository" + payload = b"AUTHENTICATED_VALUE = 23\n" + manifest_bytes = _canonical_v2_source_manifest_bytes(root, payload) + manifest = json.loads(manifest_bytes) + + assert manifest["schema"] == "recurquant.experiment013.source-manifest.v2" + assert manifest["schema"] == source_module.EXPERIMENT013_SOURCE_MANIFEST_SCHEMA + + binding = adapter_module._load_capture_module(root, manifest_bytes) + + assert binding.module.AUTHENTICATED_VALUE == 23 + assert binding.raw_sha256 == hashlib.sha256(payload).hexdigest() + assert adapter_module.CAPTURE_MODULE_NAME not in sys.modules + adapter_module._verify_capture_binding(binding) + + +def test_capture_loader_rejects_retired_v1_source_manifest(tmp_path: Path) -> None: + root = tmp_path / "repository" + path = root / Path(adapter_module.CAPTURE_SOURCE_PATH) + path.parent.mkdir(parents=True) + payload = b"AUTHENTICATED_VALUE = 17\n" + path.write_bytes(payload) + manifest = json.loads(_source_manifest_bytes(payload)) + manifest["schema"] = "recurquant.experiment013.source-manifest.v1" + + with pytest.raises( + adapter_module.Experiment013AdapterError, + match="repository source manifest schema drifted", + ): + adapter_module._load_capture_module(root, json.dumps(manifest).encode("utf-8")) + + def test_capture_loader_rejects_every_preloaded_capture_module(tmp_path: Path) -> None: root = tmp_path / "repository" path = root / Path(adapter_module.CAPTURE_SOURCE_PATH) @@ -352,6 +422,7 @@ def materialize(**_kwargs: object) -> _FakeMaterialization: class _FakeCacheLayer: def __init__(self) -> None: self.recurrent_states: dict[int, torch.Tensor | None] = {0: None} + self.is_recurrent_states_initialized: dict[int, bool] = {0: False} class _FakeDynamicCache: @@ -371,6 +442,7 @@ def update_recurrent_state(self, state: torch.Tensor, layer_index: int) -> torch cached = torch.empty_like(state) self.layers[layer_index].recurrent_states[0] = cached cached.copy_(state) + self.layers[layer_index].is_recurrent_states_initialized[0] = True if layer_index == self.corrupt_layer: cached.add_(1) return cached @@ -388,6 +460,7 @@ def __init__(self, layer_idx: int) -> None: self.bad_query = False self.bad_query_dtype = False self.bad_state = False + self.mutate_source_state = False self.causal_conv1d_fn = None self.causal_conv1d_update = lambda *args, **kwargs: None self.chunk_gated_delta_rule = self._chunk @@ -401,6 +474,8 @@ def _state(self, initial_state: torch.Tensor | None) -> torch.Tensor: float(self.layer_idx + 1), dtype=torch.float32, ) + if self.mutate_source_state: + initial_state.data.add_(0.5) return initial_state + 1.0 def _chunk( @@ -529,6 +604,56 @@ def __init__(self) -> None: self.config = types.SimpleNamespace() self.model = _FakeTextModel() self.training = False + self._parameter = torch.nn.Parameter(torch.tensor(1.0), requires_grad=False) + self.fail_after_fisher_forward = False + + def parameters(self): + return iter((self._parameter,)) + + def __call__( + self, + *, + input_ids: torch.Tensor, + position_ids: torch.Tensor, + past_key_values: _FakeDynamicCache, + use_cache: bool, + logits_to_keep: int, + ) -> object: + assert tuple(input_ids.shape) == (1, 1) + assert tuple(position_ids.shape) == (1, 1) + assert use_cache is True + assert logits_to_keep == 1 + assert torch.is_grad_enabled() + position = int(position_ids.item()) + state_scalars: list[torch.Tensor] = [] + for layer_index in adapter_module.RECURRENT_LAYER_INDICES: + module = self.model.layers[layer_index].linear_attn + query = torch.full( + adapter_module.QUERY_SHAPE, + float(layer_index + position), + dtype=torch.bfloat16, + ) + cached = past_key_values.layers[layer_index].recurrent_states[0] + assert isinstance(cached, torch.Tensor) + output = module.recurrent_gated_delta_rule( + query, + torch.zeros_like(query), + torch.zeros_like(query), + g=torch.zeros((1, 1, 16), dtype=torch.float32), + beta=torch.zeros((1, 1, 16), dtype=torch.float32), + initial_state=cached, + output_final_state=True, + use_qk_l2norm_in_kernel=True, + ) + past_key_values.update_recurrent_state(output[1], layer_index) + state_scalars.append(output[1][0, 0, 0, 0]) + past_key_values.sequence_length += 1 + if self.fail_after_fisher_forward: + raise RuntimeError("fake differentiable forward failure") + score = torch.stack(state_scalars).sum() * 1e-4 + vocabulary_axis = torch.arange(32, dtype=torch.float32) + logits = (score * vocabulary_axis).reshape(1, 1, -1) + return types.SimpleNamespace(logits=logits, past_key_values=past_key_values) def _bind_fake_model( @@ -631,6 +756,223 @@ def test_one_token_chunk_then_recurrent_and_anchor_only_state(tmp_path: Path) -> adapter.close_model(model) +def test_h1_fisher_step_is_causal_functional_and_detached(tmp_path: Path) -> None: + adapter = adapter_module.create_adapter(_context(tmp_path)) + model, observer = _bind_fake_model(adapter) + record = _sequence_record(length=4) + adapter.begin_sequence(model, record) + adapter.step_token(model, token_id=5, position=0, capture_state=False) + + result = adapter.step_token_with_fisher( + model, + token_id=6, + position=1, + target_token_id=7, + capture_state=True, + ) + + assert (result.boundary_position, result.input_position, result.target_position) == (0, 1, 2) + assert (result.input_token_id, result.target_token_id) == (6, 7) + assert result.step_observation.position == 1 + assert result.step_observation.token_id == 6 + assert result.step_observation.recurrence_query.shape == (18, 16, 128) + assert result.step_observation.recurrent_state.shape == (18, 16, 128, 128) + assert result.source_recurrent_state.shape == (18, 16, 128, 128) + assert result.source_state_gradient.shape == (18, 16, 128, 128) + assert result.source_recurrent_state.dtype == torch.float32 + assert result.source_state_gradient.dtype == torch.float32 + assert torch.count_nonzero(result.source_state_gradient).item() == 18 + assert result.target_nll > 0.0 + score = (result.source_recurrent_state[:, 0, 0, 0] + 1.0).sum() * 1e-4 + vocabulary_axis = torch.arange(32, dtype=torch.float32) + probabilities = torch.softmax(score * vocabulary_axis, dim=0) + analytic_gradient = float( + ((probabilities * vocabulary_axis).sum() - result.target_token_id).item() * 1e-4 + ) + observed_gradient = float(result.source_state_gradient[0, 0, 0, 0].item()) + assert observed_gradient == pytest.approx(analytic_gradient, rel=2e-6, abs=1e-9) + + score_fp64 = float(score.item()) + axis_fp64 = torch.arange(32, dtype=torch.float64) + + def perturbed_nll(delta: float) -> float: + logits = (score_fp64 + delta * 1e-4) * axis_fp64 + return float((torch.logsumexp(logits, dim=0) - logits[7]).item()) + + epsilon = 1e-2 + finite_difference = (perturbed_nll(epsilon) - perturbed_nll(-epsilon)) / (2 * epsilon) + assert observed_gradient == pytest.approx(finite_difference, rel=2e-5, abs=1e-9) + assert adapter.runtime_metadata()["fisher_step_count"] == 1 + assert "update_recurrent_state" not in adapter._sequence.cache.__dict__ # type: ignore[union-attr] + assert model._parameter.requires_grad is False + assert model._parameter.grad is None + for layer_index in adapter_module.RECURRENT_LAYER_INDICES: + cached = adapter._sequence.cache.layers[layer_index].recurrent_states[0] # type: ignore[union-attr] + assert isinstance(cached, torch.Tensor) + assert cached.requires_grad is False + assert cached.grad_fn is None + + continued = adapter.step_token(model, token_id=7, position=2, capture_state=True) + assert continued.position == 2 + assert continued.recurrent_state is not None + assert torch.equal( + continued.recurrent_state[:, 0, 0, 0], + result.step_observation.recurrent_state[:, 0, 0, 0] + 1.0, + ) + assert adapter._sequence is not None + assert adapter._sequence.next_position == 3 + assert adapter._sequence.cache.get_seq_length() == 3 + assert observer.is_idle + adapter.end_sequence(model, record) + adapter.close_model(model) + + +def test_h1_fisher_failure_rolls_back_cache_and_invalidates_sequence(tmp_path: Path) -> None: + adapter = adapter_module.create_adapter(_context(tmp_path)) + model, observer = _bind_fake_model(adapter) + record = _sequence_record(length=3) + adapter.begin_sequence(model, record) + adapter.step_token(model, token_id=5, position=0, capture_state=False) + assert adapter._sequence is not None + before = { + layer_index: adapter._sequence.cache.layers[layer_index].recurrent_states[0].clone() + for layer_index in adapter_module.RECURRENT_LAYER_INDICES + } + model.fail_after_fisher_forward = True + + with pytest.raises(RuntimeError, match="fake differentiable forward failure"): + adapter.step_token_with_fisher( + model, + token_id=6, + position=1, + target_token_id=7, + capture_state=False, + ) + + assert adapter._sequence_failed is True + assert adapter._sequence.cache.get_seq_length() == 1 + assert "update_recurrent_state" not in adapter._sequence.cache.__dict__ + for layer_index, expected in before.items(): + restored = adapter._sequence.cache.layers[layer_index].recurrent_states[0] + assert torch.equal(restored, expected) + assert restored.requires_grad is False + assert restored.grad_fn is None + assert observer.is_idle + assert model._parameter.grad is None + adapter.end_sequence(model, record) + adapter.close_model(model) + + +def test_h1_fisher_rejects_in_place_source_state_mutation_and_rolls_back( + tmp_path: Path, +) -> None: + adapter = adapter_module.create_adapter(_context(tmp_path)) + model, observer = _bind_fake_model(adapter) + record = _sequence_record(length=3) + adapter.begin_sequence(model, record) + adapter.step_token(model, token_id=5, position=0, capture_state=False) + assert adapter._sequence is not None + before = { + layer_index: adapter._sequence.cache.layers[layer_index].recurrent_states[0].clone() + for layer_index in adapter_module.RECURRENT_LAYER_INDICES + } + for layer_index in adapter_module.RECURRENT_LAYER_INDICES: + model.model.layers[layer_index].linear_attn.mutate_source_state = True + + with pytest.raises( + adapter_module.Experiment013AdapterError, + match="mutated source state", + ): + adapter.step_token_with_fisher( + model, + token_id=6, + position=1, + target_token_id=7, + capture_state=False, + ) + + assert adapter._sequence_failed is True + assert adapter._sequence.cache.get_seq_length() == 1 + for layer_index, expected in before.items(): + restored = adapter._sequence.cache.layers[layer_index].recurrent_states[0] + assert torch.equal(restored, expected) + assert restored.requires_grad is False + assert restored.grad_fn is None + assert observer.is_idle + adapter.end_sequence(model, record) + adapter.close_model(model) + + +def test_h1_fisher_detach_failure_rolls_back_without_advancing_counters( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + adapter = adapter_module.create_adapter(_context(tmp_path)) + model, observer = _bind_fake_model(adapter) + record = _sequence_record(length=3) + adapter.begin_sequence(model, record) + adapter.step_token(model, token_id=5, position=0, capture_state=False) + assert adapter._sequence is not None + before = { + layer_index: adapter._sequence.cache.layers[layer_index].recurrent_states[0].clone() + for layer_index in adapter_module.RECURRENT_LAYER_INDICES + } + + def fail_detach(_cache: object) -> None: + raise RuntimeError("synthetic detach failure") + + monkeypatch.setattr(adapter_module, "_detach_cache_tensors", fail_detach) + with pytest.raises(RuntimeError, match="synthetic detach failure"): + adapter.step_token_with_fisher( + model, + token_id=6, + position=1, + target_token_id=7, + capture_state=False, + ) + + assert adapter._sequence_failed is True + assert adapter._sequence.next_position == 1 + assert adapter._sequence.cache.get_seq_length() == 1 + assert adapter.runtime_metadata()["fisher_step_count"] == 0 + for layer_index, expected in before.items(): + restored = adapter._sequence.cache.layers[layer_index].recurrent_states[0] + assert torch.equal(restored, expected) + assert observer.is_idle + adapter.end_sequence(model, record) + adapter.close_model(model) + + +def test_h1_fisher_requires_warm_boundary_and_future_target(tmp_path: Path) -> None: + adapter = adapter_module.create_adapter(_context(tmp_path)) + model, _observer = _bind_fake_model(adapter) + warm_record = _sequence_record(length=3) + adapter.begin_sequence(model, warm_record) + with pytest.raises(adapter_module.Experiment013AdapterError, match="warm S_b"): + adapter.step_token_with_fisher( + model, + token_id=5, + position=0, + target_token_id=6, + capture_state=False, + ) + adapter.end_sequence(model, warm_record) + + short_record = _sequence_record(length=2) + adapter.begin_sequence(model, short_record) + adapter.step_token(model, token_id=5, position=0, capture_state=False) + with pytest.raises(adapter_module.Experiment013AdapterError, match=r"x_\(b\+2\) target"): + adapter.step_token_with_fisher( + model, + token_id=6, + position=1, + target_token_id=7, + capture_state=False, + ) + adapter.end_sequence(model, short_record) + adapter.close_model(model) + + @pytest.mark.parametrize( "failure", [ diff --git a/tests/test_experiment013_source.py b/tests/test_experiment013_source.py index 67a35fb..5e7e71d 100644 --- a/tests/test_experiment013_source.py +++ b/tests/test_experiment013_source.py @@ -1,6 +1,9 @@ from __future__ import annotations +import ast import copy +import hashlib +import importlib.util import json import os import subprocess @@ -13,6 +16,7 @@ from recurquant.experiment013_source import ( EXPERIMENT013_SOURCE_PATHS, Experiment013SourceError, + authenticate_git_executable, canonical_experiment013_source_manifest_bytes, canonical_experiment013_source_manifest_sha256, capture_experiment013_source_manifest, @@ -73,9 +77,12 @@ def test_frozen_inventory_covers_all_experiment013_surfaces_without_hash_constan "scripts/capture_static_q468_identity_input.py", "scripts/generate_static_q468_ruler_receipts.py", "scripts/launch_static_q468_calibration.py", + "scripts/launch_static_q468_stage_a.py", "scripts/resolve_static_q468_identity.py", "scripts/run_static_q468_calibration.py", + "scripts/screen_static_q468_stage_a.py", "requirements/experiment013-ruler.txt", + "src/recurquant/cache.py", "src/recurquant/static_q468.py", "src/recurquant/static_q468_cache.py", "src/recurquant/static_q468_calibration.py", @@ -83,11 +90,14 @@ def test_frozen_inventory_covers_all_experiment013_surfaces_without_hash_constan "src/recurquant/experiment013_parquet.py", "src/recurquant/experiment013_qwen35_adapter.py", "src/recurquant/experiment013_source.py", + "src/recurquant/experiment013_stage_a.py", "tests/test_capture_static_q468_identity_input.py", "tests/test_generate_static_q468_ruler_receipts.py", "tests/test_launch_static_q468_calibration.py", + "tests/test_launch_static_q468_stage_a.py", "tests/test_resolve_static_q468_identity.py", "tests/test_run_static_q468_calibration.py", + "tests/test_screen_static_q468_stage_a.py", "tests/test_static_q468.py", "tests/test_static_q468_cache.py", "tests/test_static_q468_calibration.py", @@ -95,6 +105,7 @@ def test_frozen_inventory_covers_all_experiment013_surfaces_without_hash_constan "tests/test_experiment013_parquet.py", "tests/test_experiment013_qwen35_adapter.py", "tests/test_experiment013_source.py", + "tests/test_experiment013_stage_a.py", } assert required <= set(EXPERIMENT013_SOURCE_PATHS) @@ -104,6 +115,50 @@ def test_frozen_inventory_covers_all_experiment013_surfaces_without_hash_constan assert 'canonical_manifest_sha256 = "' not in source.read_text(encoding="utf-8") +def test_frozen_inventory_is_an_explicit_string_literal_set() -> None: + source = Path(__file__).resolve().parents[1] / "src" / "recurquant" / "experiment013_source.py" + tree = ast.parse(source.read_text(encoding="utf-8")) + assignment = next( + node + for node in tree.body + if isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and node.target.id == "EXPERIMENT013_SOURCE_PATHS" + ) + assert isinstance(assignment.value, ast.Call) + outer = assignment.value + assert isinstance(outer.func, ast.Name) and outer.func.id == "tuple" + assert len(outer.args) == 1 and isinstance(outer.args[0], ast.Call) + ordered = outer.args[0] + assert isinstance(ordered.func, ast.Name) and ordered.func.id == "sorted" + assert len(ordered.args) == 1 and isinstance(ordered.args[0], ast.Set) + literals = ordered.args[0].elts + assert literals + assert all(isinstance(item, ast.Constant) and isinstance(item.value, str) for item in literals) + rendered = tuple(item.value for item in literals if isinstance(item, ast.Constant)) + assert rendered == tuple(sorted(rendered)) + assert len(rendered) == len(set(rendered)) + assert rendered == EXPERIMENT013_SOURCE_PATHS + + +def test_stage_a_runner_required_source_paths_are_frozen() -> None: + root = Path(__file__).resolve().parents[1] + module_name = "_recurquant_experiment013_stage_a_source_inventory_test" + path = root / "scripts" / "screen_static_q468_stage_a.py" + spec = importlib.util.spec_from_file_location(module_name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + try: + spec.loader.exec_module(module) + stage_a_required = module.REQUIRED_SOURCE_PATHS + finally: + sys.modules.pop(module_name, None) + + assert isinstance(stage_a_required, frozenset) + assert stage_a_required <= set(EXPERIMENT013_SOURCE_PATHS) + + def test_capture_is_portable_complete_and_allows_ignored_artifacts(tmp_path: Path) -> None: root = _init_repository(tmp_path / "repository") _write(root / "artifacts" / "ignored-result.json", "{}\n") @@ -168,6 +223,52 @@ def test_capture_scrubs_inherited_git_index_redirection( assert manifest["source_commit"] == _git(root, "rev-parse", "HEAD") +def test_explicit_authenticated_git_ignores_a_fake_path_precedence( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = _init_repository(tmp_path / "repository") + authenticated_git = authenticate_git_executable() + fake_bin = tmp_path / "fake-bin" + fake_bin.mkdir() + (fake_bin / "git.exe").write_bytes(b"not an executable\n") + monkeypatch.setenv("PATH", f"{fake_bin}{os.pathsep}{os.environ.get('PATH', '')}") + + manifest = capture_experiment013_source_manifest( + root, + git_executable=authenticated_git.path, + ) + + assert ( + verify_experiment013_source_manifest( + manifest, + root, + git_executable=authenticated_git.path, + ) + == manifest + ) + assert manifest["git_executable"] == { + "sha256": authenticated_git.sha256, + "size_bytes": authenticated_git.size_bytes, + } + manifest_bytes = canonical_experiment013_source_manifest_bytes(manifest) + assert str(authenticated_git.path).encode() not in manifest_bytes + + +def test_git_for_windows_cmd_shim_is_canonicalized_to_real_binary(tmp_path: Path) -> None: + cmd_git = tmp_path / "Git" / "cmd" / "git.exe" + real_git = tmp_path / "Git" / "mingw64" / "bin" / "git.exe" + cmd_git.parent.mkdir(parents=True) + real_git.parent.mkdir(parents=True) + cmd_git.write_bytes(b"shim\n") + real_git.write_bytes(b"actual Git implementation\n") + + identity = authenticate_git_executable(cmd_git) + + assert identity.path == real_git.resolve(strict=True) + assert identity.sha256 == hashlib.sha256(real_git.read_bytes()).hexdigest() + assert identity.size_bytes == real_git.stat().st_size + + def test_capture_rejects_unsafe_local_git_config(tmp_path: Path) -> None: root = _init_repository(tmp_path / "repository") _git(root, "config", "core.fsmonitor", "true") diff --git a/tests/test_experiment013_stage_a.py b/tests/test_experiment013_stage_a.py new file mode 100644 index 0000000..68c2e52 --- /dev/null +++ b/tests/test_experiment013_stage_a.py @@ -0,0 +1,560 @@ +from __future__ import annotations + +import hashlib +import json +from collections.abc import Callable, Mapping, Sequence +from dataclasses import replace + +import pytest + +from recurquant.evidence import canonical_json_bytes +from recurquant.experiment013_stage_a import ( + BOOTSTRAP_QUANTILE_ALGORITHM, + BOOTSTRAP_SAMPLES, + BOOTSTRAP_SEED, + DECISION_COMPARATOR_ORDER, + FP32_METHOD, + STAGE_A_FAMILY_ORDER, + STAGE_A_METHOD_ORDER, + UNIFORM_RHT_Q4_METHOD, + UNIFORM_RHT_Q8_METHOD, + StageAExample, + StageATokenRow, + build_stage_a_evidence_artifact, + deserialize_stage_a_evidence_artifact, + stratified_bootstrap_upper_bound, +) +from recurquant.static_q468 import ( + STATIC_Q48_COMPARATOR_METHOD, + STATIC_Q468_ABLATION_METHOD, + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + STATIC_Q468_MSE_METHOD, + STATIC_Q468_PRIMARY_METHOD, +) +from recurquant.static_q468_cache import DYNAMIC_Q468_BASELINE_METHOD + +IDENTITY_FILE_SHA256 = "a" * 64 +BINDING_FILE_SHA256 = "b" * 64 + +Excess = Callable[[StageAExample, str, int], float] +Agreement = Callable[[StageAExample, str, int], bool] + + +def _examples(*, transitions: int | Callable[[str, int], int] = 1) -> tuple[StageAExample, ...]: + result: list[StageAExample] = [] + for family in STAGE_A_FAMILY_ORDER: + for rank in range(4): + count = transitions(family, rank) if callable(transitions) else transitions + identity_hash = hashlib.sha256(f"{family}:{rank}".encode()).hexdigest() + result.append( + StageAExample( + family=family, + canonical_id=f"{family}/{rank}", + selection_rank=rank, + continuation_token_count=count + 1, + identity_record_sha256=identity_hash, + ) + ) + return tuple(result) + + +def _default_excess(example: StageAExample, method: str, transition: int) -> float: + del example, transition + return { + FP32_METHOD: 0.0, + UNIFORM_RHT_Q4_METHOD: 0.20, + UNIFORM_RHT_Q8_METHOD: 0.10, + STATIC_Q48_COMPARATOR_METHOD: 0.08, + STATIC_Q468_ABLATION_METHOD: 100.0, + DYNAMIC_Q468_BASELINE_METHOD: 0.015, + STATIC_Q468_MSE_METHOD: 0.014, + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD: 0.013, + STATIC_Q468_PRIMARY_METHOD: 0.020, + }[method] + + +def _rows( + examples: tuple[StageAExample, ...], + *, + excess: Excess = _default_excess, + agreement: Agreement | None = None, +) -> tuple[StageATokenRow, ...]: + result: list[StageATokenRow] = [] + for example in examples: + for method in STAGE_A_METHOD_ORDER: + for transition in range(example.transition_count): + delta = excess(example, method, transition) + is_fp32 = method == FP32_METHOD + result.append( + StageATokenRow( + family=example.family, + canonical_id=example.canonical_id, + selection_rank=example.selection_rank, + identity_record_sha256=example.identity_record_sha256, + method_id=method, + transition_index=transition, + reference_nll=0.0, + method_nll=delta, + kl=0.0 if is_fp32 else max(delta, 0.0) + 0.001, + top1_agreement=( + True + if is_fp32 or agreement is None + else agreement(example, method, transition) + ), + ) + ) + return tuple(result) + + +def _build( + examples: tuple[StageAExample, ...], + rows: tuple[StageATokenRow, ...], +) -> bytes: + return build_stage_a_evidence_artifact( + examples, + rows, + stage_a_identity_file_sha256=IDENTITY_FILE_SHA256, + stage_a_calibration_binding_file_sha256=BINDING_FILE_SHA256, + ) + + +def _decision(raw: bytes) -> dict[str, object]: + artifact = deserialize_stage_a_evidence_artifact(raw) + return dict(artifact.evidence["decision"]) + + +def _comparator(decision: dict[str, object], method: str) -> dict[str, object]: + comparisons = decision["noninferiority_comparators"] + assert isinstance(comparisons, Sequence) + return dict(next(item for item in comparisons if item["comparator_method"] == method)) + + +def test_frozen_method_order_and_m_equals_two_yields_one_transition() -> None: + assert STAGE_A_METHOD_ORDER == ( + "fp32_reference", + "rht_q468_uniform_q4", + "rht_q468_uniform_q8", + "rht_q48_static_p14739", + "rht_q468_static_k27030", + "rht_q468_dynamic_k27030", + "rht_q468_static_mse_k29334", + "rht_q468_static_diag_empirical_fisher_h1_k29334", + "rht_q468_static_k29334", + ) + examples = _examples(transitions=1) + raw = _build(examples, _rows(examples)) + artifact = deserialize_stage_a_evidence_artifact( + raw, + expected_file_sha256=hashlib.sha256(raw).hexdigest(), + expected_canonical_evidence_sha256=json.loads(raw)["canonical_evidence_sha256"], + expected_stage_a_identity_file_sha256=IDENTITY_FILE_SHA256, + expected_stage_a_calibration_binding_file_sha256=BINDING_FILE_SHA256, + ) + + assert artifact.passed is True + assert len(artifact.evidence["token_rows"]) == 12 * 9 + assert len(artifact.evidence["example_summaries"]) == 12 * 9 + summaries = artifact.evidence["method_summaries"] + assert [summary["method_id"] for summary in summaries] == list(STAGE_A_METHOD_ORDER) + assert all( + family["token_count"] == 4 for summary in summaries for family in summary["by_family"] + ) + + +def test_per_example_excess_top1_and_existing_cvar95_definition_are_published() -> None: + examples = _examples(transitions=21) + rows = list(_rows(examples)) + first = examples[0] + for index, row in enumerate(rows): + if row.canonical_id == first.canonical_id and row.method_id == STATIC_Q468_PRIMARY_METHOD: + rows[index] = replace(row, kl=float(row.transition_index)) + artifact = deserialize_stage_a_evidence_artifact(_build(examples, tuple(rows))) + summary = next( + item + for item in artifact.evidence["example_summaries"] + if item["canonical_id"] == first.canonical_id + and item["method_id"] == STATIC_Q468_PRIMARY_METHOD + ) + + assert summary["metrics"]["excess_nll"] == pytest.approx(0.020) + assert summary["metrics"]["top1_agreement"] == 1.0 + # ceil(0.05 * 21) = 2, so CVaR95 is the mean of token KL 20 and 19. + assert summary["metrics"]["cvar95_kl"] == pytest.approx(19.5) + + +@pytest.mark.parametrize("malformation", ["missing", "duplicate", "reordered"]) +def test_token_grid_fails_closed_on_missing_duplicate_or_reordered_rows( + malformation: str, +) -> None: + examples = _examples() + rows = list(_rows(examples)) + if malformation == "missing": + rows.pop() + elif malformation == "duplicate": + rows[-1] = rows[-2] + else: + rows[1], rows[2] = rows[2], rows[1] + + with pytest.raises(ValueError, match="row count|missing, duplicate, or reordered"): + _build(examples, tuple(rows)) + + +def test_example_grid_rejects_duplicate_and_reordered_identities() -> None: + examples = list(_examples()) + examples[0], examples[1] = examples[1], examples[0] + with pytest.raises(ValueError, match="ordered by frozen family order"): + _build(tuple(examples), _rows(_examples())) + + examples = list(_examples()) + examples[1] = StageAExample( + family=examples[1].family, + canonical_id=examples[0].canonical_id, + selection_rank=examples[1].selection_rank, + continuation_token_count=examples[1].continuation_token_count, + identity_record_sha256=examples[1].identity_record_sha256, + ) + with pytest.raises(ValueError, match="duplicate Stage-A example"): + _build(tuple(examples), _rows(_examples())) + + +def test_reconstructing_verifier_rejects_tampered_rows_and_derived_metrics() -> None: + examples = _examples() + raw = _build(examples, _rows(examples)) + document = json.loads(raw) + document["evidence"]["token_rows"][-1]["method_nll"] += 0.25 + document["canonical_evidence_sha256"] = hashlib.sha256( + canonical_json_bytes(document["evidence"]) + ).hexdigest() + tampered = canonical_json_bytes(document) + + with pytest.raises(ValueError, match="deterministic reconstruction"): + deserialize_stage_a_evidence_artifact(tampered) + + changed_rows = list(_rows(examples)) + changed_rows[-1] = replace( + changed_rows[-1], + method_nll=changed_rows[-1].method_nll + 0.25, + ) + changed = _build(examples, tuple(changed_rows)) + with pytest.raises(ValueError, match="file SHA-256 differs"): + deserialize_stage_a_evidence_artifact( + changed, + expected_file_sha256=hashlib.sha256(raw).hexdigest(), + ) + + +def test_duplicate_json_keys_fail_closed() -> None: + with pytest.raises(ValueError, match="duplicate key"): + deserialize_stage_a_evidence_artifact(b'{"artifact_kind":"x","artifact_kind":"y"}') + + +def test_verified_evidence_tree_is_immutable_and_passed_is_stable() -> None: + examples = _examples() + raw = _build(examples, _rows(examples)) + artifact = deserialize_stage_a_evidence_artifact(raw) + decision = artifact.evidence["decision"] + assert isinstance(decision, Mapping) + comparisons = decision["noninferiority_comparators"] + assert isinstance(comparisons, tuple) + + with pytest.raises(TypeError): + decision["stage_a_passed"] = False # type: ignore[index] + with pytest.raises(TypeError): + comparisons[0]["passed"] = False # type: ignore[index] + + assert artifact.passed is True + assert artifact.serialized_bytes == raw + assert artifact.file_sha256 == hashlib.sha256(raw).hexdigest() + + +def test_negative_kl_is_rejected() -> None: + examples = _examples() + rows = list(_rows(examples)) + rows[-1] = replace(rows[-1], kl=-1e-6) + + with pytest.raises(ValueError, match="kl must be nonnegative"): + _build(examples, tuple(rows)) + + +def test_cvar95_uses_reviewed_fp32_tail_mean_semantics_exactly() -> None: + examples = _examples(transitions=21) + rows = list(_rows(examples)) + first = examples[0] + values = [16_777_216.0, 1.0, *([0.0] * 19)] + for index, row in enumerate(rows): + if row.canonical_id == first.canonical_id and row.method_id == UNIFORM_RHT_Q4_METHOD: + rows[index] = replace(row, kl=values[row.transition_index]) + + artifact = deserialize_stage_a_evidence_artifact(_build(examples, tuple(rows))) + summary = next( + item + for item in artifact.evidence["example_summaries"] + if item["canonical_id"] == first.canonical_id and item["method_id"] == UNIFORM_RHT_Q4_METHOD + ) + + # FP64 arithmetic would produce 8,388,608.5. Torch's reviewed FP32 + # fidelity path rounds that midpoint to 8,388,608.0. + assert summary["metrics"]["cvar95_kl"] == 8_388_608.0 + + +def test_cvar95_rejects_values_that_overflow_during_fp32_conversion() -> None: + examples = _examples() + rows = list(_rows(examples)) + rows[-1] = replace(rows[-1], kl=1e100) + + with pytest.raises(ValueError, match="remain finite after FP32 conversion"): + _build(examples, tuple(rows)) + + +def test_bootstrap_is_deterministic_family_equal_and_nearest_rank() -> None: + differences = { + "pg19": [1.0, 1.0, 1.0, 1.0], + "ruler": [2.0, 2.0, 2.0, 2.0], + "humaneval_plus": [6.0, 6.0, 6.0, 6.0], + } + first = stratified_bootstrap_upper_bound(differences) + second = stratified_bootstrap_upper_bound(differences) + + assert first == second + assert first["bootstrap_samples"] == BOOTSTRAP_SAMPLES == 10_000 + assert first["seed"] == BOOTSTRAP_SEED == 2_339 + assert first["family_equal_point_estimate"] == pytest.approx(3.0) + assert first["one_sided_95_upper_bound"] == pytest.approx(3.0) + assert first["quantile_algorithm"] == BOOTSTRAP_QUANTILE_ALGORITHM + assert "ceil(q*N)-1" in BOOTSTRAP_QUANTILE_ALGORITHM + + +def test_token_micro_is_diagnostic_and_family_macro_weights_examples_equally() -> None: + examples = _examples( + transitions=lambda family, rank: 10 if (family, rank) == ("pg19", 0) else 1 + ) + + def excess(example: StageAExample, method: str, transition: int) -> float: + del transition + if method == FP32_METHOD: + return 0.0 + if method == STATIC_Q468_PRIMARY_METHOD: + return 0.08 if (example.family, example.selection_rank) == ("pg19", 0) else 0.0 + if method == STATIC_Q48_COMPARATOR_METHOD: + return 0.20 + if method in DECISION_COMPARATOR_ORDER: + return 0.10 + return 0.30 + + artifact = deserialize_stage_a_evidence_artifact( + _build(examples, _rows(examples, excess=excess)) + ) + summaries = {summary["method_id"]: summary for summary in artifact.evidence["method_summaries"]} + primary = summaries[STATIC_Q468_PRIMARY_METHOD] + + # One of four PG19 examples is 0.08, so PG19 macro is 0.02. The other + # family means are zero, making the family-equal macro 0.02/3. + assert primary["family_macro"]["excess_nll"] == pytest.approx(0.02 / 3.0) + assert primary["token_micro_diagnostic"]["excess_nll"] > 0.02 + + +def test_bootstrap_upper_conjunct_is_required() -> None: + examples = _examples() + + def excess(example: StageAExample, method: str, transition: int) -> float: + del transition + candidate = 0.020 + if method == FP32_METHOD: + return 0.0 + if method == STATIC_Q468_PRIMARY_METHOD: + return candidate + if method == DYNAMIC_Q468_BASELINE_METHOD: + difference = 0.015 if example.selection_rank < 3 else -0.015 + return candidate - difference + if method in ( + STATIC_Q468_MSE_METHOD, + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + ): + return candidate + 0.020 + if method == STATIC_Q48_COMPARATOR_METHOD: + return candidate + 0.050 + return 0.10 + + decision = _decision(_build(examples, _rows(examples, excess=excess))) + comparison = _comparator(decision, DYNAMIC_Q468_BASELINE_METHOD) + assert comparison["checks"] == { + "one_sided_95_upper_bound_at_most_0_010": False, + "every_family_point_at_most_0_015": True, + "family_macro_top1_trail_at_most_0_005": True, + } + assert decision["stage_a_passed"] is False + + +def test_every_family_point_conjunct_is_required() -> None: + examples = _examples() + + def excess(example: StageAExample, method: str, transition: int) -> float: + del transition + candidate = 0.020 + if method == FP32_METHOD: + return 0.0 + if method == STATIC_Q468_PRIMARY_METHOD: + return candidate + if method == DYNAMIC_Q468_BASELINE_METHOD: + difference = 0.0151 if example.family == "pg19" else -0.020 + return candidate - difference + if method in ( + STATIC_Q468_MSE_METHOD, + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + ): + return candidate + 0.020 + if method == STATIC_Q48_COMPARATOR_METHOD: + return candidate + 0.050 + return 0.10 + + decision = _decision(_build(examples, _rows(examples, excess=excess))) + comparison = _comparator(decision, DYNAMIC_Q468_BASELINE_METHOD) + assert comparison["checks"] == { + "one_sided_95_upper_bound_at_most_0_010": True, + "every_family_point_at_most_0_015": False, + "family_macro_top1_trail_at_most_0_005": True, + } + assert decision["stage_a_passed"] is False + + +def test_family_macro_top1_conjunct_is_required() -> None: + examples = _examples() + + def excess(example: StageAExample, method: str, transition: int) -> float: + del example, transition + if method == FP32_METHOD: + return 0.0 + if method == STATIC_Q468_PRIMARY_METHOD: + return 0.020 + if method == STATIC_Q48_COMPARATOR_METHOD: + return 0.080 + if method in DECISION_COMPARATOR_ORDER: + return 0.040 + return 0.10 + + def agreement(example: StageAExample, method: str, transition: int) -> bool: + del example, transition + if method == DYNAMIC_Q468_BASELINE_METHOD: + return True + return method not in ( + STATIC_Q468_PRIMARY_METHOD, + STATIC_Q468_MSE_METHOD, + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + ) + + decision = _decision(_build(examples, _rows(examples, excess=excess, agreement=agreement))) + comparison = _comparator(decision, DYNAMIC_Q468_BASELINE_METHOD) + assert comparison["checks"] == { + "one_sided_95_upper_bound_at_most_0_010": True, + "every_family_point_at_most_0_015": True, + "family_macro_top1_trail_at_most_0_005": False, + } + assert decision["stage_a_passed"] is False + + +@pytest.mark.parametrize( + "comparator", + [STATIC_Q468_MSE_METHOD, STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD], +) +def test_each_frozen_static_selector_comparator_can_fail_the_screen(comparator: str) -> None: + examples = _examples() + + def excess(example: StageAExample, method: str, transition: int) -> float: + del example, transition + if method == FP32_METHOD: + return 0.0 + if method == STATIC_Q468_PRIMARY_METHOD: + return 0.020 + if method == comparator: + return 0.0 + if method in DECISION_COMPARATOR_ORDER: + return 0.040 + if method == STATIC_Q48_COMPARATOR_METHOD: + return 0.080 + return 0.10 + + decision = _decision(_build(examples, _rows(examples, excess=excess))) + assert _comparator(decision, comparator)["passed"] is False + assert decision["stage_a_passed"] is False + + +def test_all_nonlinearity_boundaries_pass_at_equality() -> None: + examples = _examples(transitions=50) + difference_by_family = {"pg19": 0.015, "ruler": 0.010, "humaneval_plus": 0.005} + + def excess(example: StageAExample, method: str, transition: int) -> float: + del transition + candidate = 0.020 + if method == FP32_METHOD: + return 0.0 + if method == STATIC_Q468_PRIMARY_METHOD: + return candidate + if method in DECISION_COMPARATOR_ORDER: + return candidate - difference_by_family[example.family] + if method == STATIC_Q48_COMPARATOR_METHOD: + return candidate + 0.050 + return 0.10 + + def agreement(example: StageAExample, method: str, transition: int) -> bool: + if method == STATIC_Q468_PRIMARY_METHOD: + # One miss in one of four 50-transition examples per family gives + # an exact family-macro trail of 1/(4*50) = 0.005. + return example.selection_rank != 0 or transition != 0 + return True + + decision = _decision(_build(examples, _rows(examples, excess=excess, agreement=agreement))) + for comparator in DECISION_COMPARATOR_ORDER: + comparison = _comparator(decision, comparator) + assert comparison["paired_family_stratified_bootstrap"][ + "one_sided_95_upper_bound" + ] == pytest.approx(0.010) + assert max( + point["candidate_minus_comparator_excess_nll"] for point in comparison["family_points"] + ) == pytest.approx(0.015) + assert comparison["candidate_top1_trail"] == pytest.approx(0.005) + assert comparison["checks"] == { + "one_sided_95_upper_bound_at_most_0_010": True, + "every_family_point_at_most_0_015": True, + "family_macro_top1_trail_at_most_0_005": True, + } + assert decision["stage_a_passed"] is True + + +def test_q48_requires_strictly_lower_excess_nll_in_every_family() -> None: + examples = _examples() + + def excess(example: StageAExample, method: str, transition: int) -> float: + del transition + candidate = 0.020 + if method == FP32_METHOD: + return 0.0 + if method == STATIC_Q468_PRIMARY_METHOD: + return candidate + if method == STATIC_Q48_COMPARATOR_METHOD: + return candidate if example.family == "ruler" else candidate + 0.010 + if method in DECISION_COMPARATOR_ORDER: + return candidate + 0.020 + return 0.10 + + decision = _decision(_build(examples, _rows(examples, excess=excess))) + q48 = decision["q48_every_family_strict_superiority"] + assert q48["passed"] is False + ruler = next(point for point in q48["family_points"] if point["family"] == "ruler") + assert ruler["q48_minus_candidate_excess_nll"] == 0.0 + assert ruler["candidate_strictly_lower"] is False + assert decision["stage_a_passed"] is False + + +def test_static_k27030_is_diagnostic_and_cannot_decide_passage() -> None: + examples = _examples() + + def excess(example: StageAExample, method: str, transition: int) -> float: + value = _default_excess(example, method, transition) + return 10_000.0 if method == STATIC_Q468_ABLATION_METHOD else value + + decision = _decision(_build(examples, _rows(examples, excess=excess))) + + assert decision["stage_a_passed"] is True + assert decision["excluded_from_decision"][STATIC_Q468_ABLATION_METHOD] == ( + "selection-step-matched diagnostic only" + ) diff --git a/tests/test_generate_static_q468_ruler_receipts.py b/tests/test_generate_static_q468_ruler_receipts.py index 3d45576..4af3151 100644 --- a/tests/test_generate_static_q468_ruler_receipts.py +++ b/tests/test_generate_static_q468_ruler_receipts.py @@ -5,6 +5,8 @@ import subprocess import sys from pathlib import Path +from types import SimpleNamespace +from typing import Any import pytest @@ -490,6 +492,54 @@ def test_subprocess_environment_removes_python_injection(monkeypatch) -> None: assert env["TEST_MARKER"] == "bound" +def test_ruler_checkout_uses_explicit_git_and_ignores_path( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ruler_root = tmp_path / "ruler" + ruler_root.mkdir() + authenticated_git = tmp_path / "toolchain" / "git.exe" + authenticated_git.parent.mkdir() + authenticated_git.write_bytes(b"authenticated Git fixture\n") + fake_path = tmp_path / "fake-path" + fake_path.mkdir() + (fake_path / "git.exe").write_bytes(b"must not execute\n") + monkeypatch.setenv("PATH", str(fake_path)) + blob_data = b"authenticated blob\n" + blob_id = ruler._git_blob_sha1(blob_data) + observed: list[list[str]] = [] + + def fake_run(argv: list[str], **kwargs: object) -> subprocess.CompletedProcess[Any]: + observed.append(argv) + assert "PATH" not in {name.upper() for name in kwargs["env"]} # type: ignore[index] + if argv[-2:] == ["rev-parse", "HEAD"]: + return subprocess.CompletedProcess(argv, 0, stdout="a" * 40 + "\n", stderr="") + return subprocess.CompletedProcess(argv, 0, stdout=blob_data, stderr=b"") + + monkeypatch.setattr(ruler.subprocess, "run", fake_run) + monkeypatch.setattr(ruler, "_verify_task_specs_against_source", lambda **kwargs: None) + capture = SimpleNamespace( + resolver=SimpleNamespace(RULER_REVISION="a" * 40), + RULER_GENERATOR_GIT_BLOBS={ + "scripts/synthetic.yaml": blob_id, + "scripts/data/synthetic/constants.py": blob_id, + }, + _ruler_generator_manifest=lambda files: (), + ) + + checkout = ruler.verify_ruler_checkout( + ruler_root, + capture, + git_executable_path=authenticated_git, + ) + + assert checkout.source_files == { + "scripts/synthetic.yaml": blob_data, + "scripts/data/synthetic/constants.py": blob_data, + } + assert observed and all(command[0] == str(authenticated_git.resolve()) for command in observed) + + def test_isolated_stage_contains_only_verified_blob_and_corpus_bytes(tmp_path) -> None: checkout = ruler.VerifiedRulerCheckout( source_manifest=(), diff --git a/tests/test_launch_static_q468_calibration.py b/tests/test_launch_static_q468_calibration.py index 28d89ab..94a9640 100644 --- a/tests/test_launch_static_q468_calibration.py +++ b/tests/test_launch_static_q468_calibration.py @@ -12,11 +12,7 @@ import pytest -SCRIPT = ( - Path(__file__).resolve().parents[1] - / "scripts" - / "launch_static_q468_calibration.py" -) +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "launch_static_q468_calibration.py" SPEC = importlib.util.spec_from_file_location("launch_static_q468_calibration", SCRIPT) assert SPEC is not None and SPEC.loader is not None launcher = importlib.util.module_from_spec(SPEC) @@ -38,6 +34,8 @@ def _sealed_fixture(tmp_path: Path) -> dict[str, Any]: packages = tmp_path / "packages" repository = tmp_path / "repository" artifacts = tmp_path / "artifacts" + git_executable = _write(tmp_path / "toolchain" / "git.exe", b"fixture git executable\n") + git_executable, git_record = launcher._authenticated_git_executable(git_executable) interpreter = _write(base / "python.exe", b"staged interpreter\n") _write(base / "Lib" / "marker.py", b"BASE = True\n") @@ -45,11 +43,7 @@ def _sealed_fixture(tmp_path: Path) -> dict[str, Any]: _write(import_root / "demo" / "__init__.py", b"VALUE = 1\n") metadata = b"Metadata-Version: 2.1\nName: demo\nVersion: 1.0\n\n" _write(import_root / "demo-1.0.dist-info" / "METADATA", metadata) - record = ( - b"demo-1.0.dist-info/METADATA,,\n" - b"demo-1.0.dist-info/RECORD,,\n" - b"demo/__init__.py,,\n" - ) + record = b"demo-1.0.dist-info/METADATA,,\ndemo-1.0.dist-info/RECORD,,\ndemo/__init__.py,,\n" _write(import_root / "demo-1.0.dist-info" / "RECORD", record) package_roots = {"packages": packages.resolve(strict=True)} @@ -64,10 +58,9 @@ def _sealed_fixture(tmp_path: Path) -> dict[str, Any]: runtime = { "artifact_kind": launcher.RUNTIME_MANIFEST_KIND, "base_runtime_root": launcher.BASE_RUNTIME_ROOT_NAME, - "base_sys_path": ["Lib"], - "distributions": list( - launcher._distribution_inventory(package_roots, import_paths) - ), + "base_sys_path": ["Lib", "."], + "distributions": list(launcher._distribution_inventory(package_roots, import_paths)), + "git_executable": git_record, "interpreter": { "relative_path": "python.exe", "root": launcher.BASE_RUNTIME_ROOT_NAME, @@ -82,9 +75,7 @@ def _sealed_fixture(tmp_path: Path) -> dict[str, Any]: "pointer_bits": 8 * struct.calcsize("P"), "system": platform.system(), }, - "package_roots": [ - {"import_path": "Lib/site-packages", "name": "packages"} - ], + "package_roots": [{"import_path": "Lib/site-packages", "name": "packages"}], "python": { "abi_flags": getattr(sys, "abiflags", ""), "cache_tag": sys.implementation.cache_tag, @@ -106,11 +97,16 @@ def _sealed_fixture(tmp_path: Path) -> dict[str, Any]: repository / launcher.RUNNER_SOURCE_PATH, ( b"def sealed_main(argv, *, base_runtime_root, package_roots, " - b"package_import_paths, interpreter_path, pycache_prefix):\n" + b"package_import_paths, interpreter_path, pycache_prefix, " + b"git_executable_path):\n" b" return 0\n" ), ) source_payload = { + "git_executable": { + "sha256": git_record["sha256"], + "size_bytes": git_record["size_bytes"], + }, "object_format": "sha1", "paths": [ { @@ -122,16 +118,14 @@ def _sealed_fixture(tmp_path: Path) -> dict[str, Any]: "worktree_blob_oid": "b" * 40, } ], - "profile": "experiment-013-static-q468-frozen-source-v1", + "profile": "experiment-013-static-q468-frozen-source-v2", "repository_binding": {}, - "schema": "recurquant.experiment013.source-manifest.v1", + "schema": "recurquant.experiment013.source-manifest.v2", "source_commit": "a" * 40, } source = { **source_payload, - "canonical_manifest_sha256": _sha256( - launcher._pretty_json_bytes(source_payload) - ), + "canonical_manifest_sha256": _sha256(launcher._pretty_json_bytes(source_payload)), } source_path = _write( artifacts / "source.json", @@ -142,9 +136,7 @@ def _sealed_fixture(tmp_path: Path) -> dict[str, Any]: bindings = { "calibration_runtime_manifest_file_sha256": _sha256(runtime_path.read_bytes()), "model_file_manifest_file_sha256": _sha256(model_path.read_bytes()), - "parquet_materialization_manifest_file_sha256": _sha256( - parquet_path.read_bytes() - ), + "parquet_materialization_manifest_file_sha256": _sha256(parquet_path.read_bytes()), "repository_source_manifest_file_sha256": _sha256(source_path.read_bytes()), } evidence = { @@ -156,9 +148,7 @@ def _sealed_fixture(tmp_path: Path) -> dict[str, Any]: "status": "frozen", } identity = { - "canonical_evidence_sha256": _sha256( - launcher._canonical_json_bytes(evidence) - ), + "canonical_evidence_sha256": _sha256(launcher._canonical_json_bytes(evidence)), "evidence": evidence, } identity_path = _write( @@ -166,6 +156,7 @@ def _sealed_fixture(tmp_path: Path) -> dict[str, Any]: launcher._canonical_json_bytes(identity), ) runner_arguments = [ + "--fisher-h1-smoke", "--frozen-identity", str(identity_path), "--repository-source-manifest", @@ -188,6 +179,8 @@ def _sealed_fixture(tmp_path: Path) -> dict[str, Any]: host_arguments = [ "--base-runtime-root", str(base), + "--git-executable", + str(git_executable), "--package-root", f"packages={packages.resolve(strict=True)}", "--runtime-manifest", @@ -198,6 +191,7 @@ def _sealed_fixture(tmp_path: Path) -> dict[str, Any]: return { "base": base, "bindings": bindings, + "git_executable": git_executable, "host_arguments": host_arguments, "model_path": model_path, "packages": packages, @@ -208,6 +202,52 @@ def _sealed_fixture(tmp_path: Path) -> dict[str, Any]: } +def _run_embedded_manifest_boundary( + fixture: dict[str, Any], + *, + pycache: Path, + extra_environment: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[bytes]: + pycache.mkdir() + scratch = pycache.parent / f"{pycache.name}-scratch" + scratch.mkdir() + command = [ + sys.executable, + "-I", + "-S", + "-B", + "-X", + f"pycache_prefix={pycache}", + "-X", + "utf8", + "-c", + launcher.SEALED_STDIN_LOADER, + str(fixture["runtime_path"]), + str(fixture["base"]), + json.dumps( + {"packages": str(fixture["packages"])}, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ), + str(pycache), + str(fixture["git_executable"]), + str(scratch), + *fixture["runner_arguments"], + ] + environment = launcher._sealed_environment(scratch_directory=scratch) + if extra_environment: + environment.update(extra_environment) + return subprocess.run( + command, + check=False, + capture_output=True, + cwd=fixture["base"], + env=environment, + input=launcher.SEALED_BOOTSTRAP_BYTES, + ) + + def test_launch_uses_exact_isolated_command_and_reauthenticates( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -215,8 +255,9 @@ def test_launch_uses_exact_isolated_command_and_reauthenticates( fixture = _sealed_fixture(tmp_path) monkeypatch.setenv("VIRTUAL_ENV", str(tmp_path / "untrusted-venv")) monkeypatch.setenv("VIRTUAL_ENV_PROMPT", "untrusted") + monkeypatch.setenv("PATH", str(tmp_path / "fake-path")) events: list[str] = [] - commands: list[tuple[list[str], Path, dict[str, str]]] = [] + commands: list[tuple[list[str], Path, dict[str, str], bytes]] = [] verify_bound = launcher._verify_bound_artifacts verify_runtime = launcher._verify_runtime verify_pycache = launcher._verify_empty_pycache @@ -239,10 +280,11 @@ def run_wrapper( check: bool, cwd: Path, env: dict[str, str], + input: bytes, ) -> subprocess.CompletedProcess[str]: events.append("run") assert check is False - commands.append((command, cwd, env)) + commands.append((command, cwd, env, input)) return subprocess.CompletedProcess(command, 0) monkeypatch.setattr(launcher, "_verify_bound_artifacts", bound_wrapper) @@ -262,22 +304,69 @@ def run_wrapper( "pycache", ] assert len(commands) == 1 - command, cwd, environment = commands[0] + command, cwd, environment, stdin_payload = commands[0] assert command[0] == str((fixture["base"] / "python.exe").resolve(strict=True)) assert command[1:5] == ["-I", "-S", "-B", "-X"] assert command[5].startswith("pycache_prefix=") - assert command[6:10] == ["-X", "utf8", "-c", launcher.SEALED_BOOTSTRAP] + assert command[6:10] == ["-X", "utf8", "-c", launcher.SEALED_STDIN_LOADER] + assert launcher.SEALED_BOOTSTRAP not in command + assert len(subprocess.list2cmdline(command)) < 32_767 assert command[10] == str(fixture["runtime_path"].resolve(strict=True)) assert command[11] == str(fixture["base"].resolve(strict=True)) - assert json.loads(command[12]) == { - "packages": str(fixture["packages"].resolve(strict=True)) - } - assert command[14:] == fixture["runner_arguments"] + assert json.loads(command[12]) == {"packages": str(fixture["packages"].resolve(strict=True))} + assert command[14] == str(fixture["git_executable"]) + assert command[16:] == fixture["runner_arguments"] assert not Path(command[13]).exists() + assert not Path(command[15]).exists() assert cwd == fixture["base"].resolve(strict=True) assert all(not key.upper().startswith("PYTHON") for key in environment) assert "VIRTUAL_ENV" not in environment assert "VIRTUAL_ENV_PROMPT" not in environment + assert "PATH" not in environment + assert environment["TEMP"] == environment["TMP"] == command[15] + assert stdin_payload == launcher.SEALED_BOOTSTRAP_BYTES + + +def test_sealed_environment_omits_auth_network_and_compute_modifiers( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + scratch = tmp_path / "scratch" + scratch.mkdir() + forbidden = { + "HF_TOKEN": "secret", + "HUGGING_FACE_HUB_TOKEN": "secret", + "HF_ENDPOINT": "https://attacker.invalid", + "HTTPS_PROXY": "http://proxy.invalid", + "NO_PROXY": "*", + "CUDA_VISIBLE_DEVICES": "7", + "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", + "TORCH_LOGS": "+all", + "NCCL_DEBUG": "INFO", + "OMP_NUM_THREADS": "99", + } + for name, value in forbidden.items(): + monkeypatch.setenv(name, value) + + environment = launcher._sealed_environment(scratch_directory=scratch) + + assert not set(forbidden).intersection({name.upper() for name in environment}) + assert environment["TEMP"] == environment["TMP"] == str(scratch.resolve(strict=True)) + assert environment["LANG"] == environment["LC_ALL"] == "C" + assert environment["TZ"] == "UTC" + + +def test_embedded_bootstrap_rejects_an_extra_credential_variable(tmp_path: Path) -> None: + fixture = _sealed_fixture(tmp_path) + + completed = _run_embedded_manifest_boundary( + fixture, + pycache=tmp_path / "credential-pycache", + extra_environment={"HF_TOKEN": "must-not-cross"}, + ) + + assert completed.returncode != 0 + assert b"sealed child environment differs from the minimal contract" in completed.stderr def test_help_does_not_require_the_runner_separator(capsys: pytest.CaptureFixture[str]) -> None: @@ -285,6 +374,66 @@ def test_help_does_not_require_the_runner_separator(capsys: pytest.CaptureFixtur assert "exact run_static_q468_calibration.py arguments" in capsys.readouterr().out +def test_full_launch_requires_both_prior_fisher_smoke_paths_before_subprocess( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fixture = _sealed_fixture(tmp_path) + arguments = list(fixture["host_arguments"]) + arguments.remove("--fisher-h1-smoke") + monkeypatch.setattr( + launcher.subprocess, + "run", + lambda *_args, **_kwargs: pytest.fail("subprocess must not start"), + ) + + with pytest.raises(launcher.SealedLaunchError, match="requires both prior Fisher"): + launcher.launch(arguments) + + +def test_full_launch_authenticates_prior_fisher_smoke_report_and_marker( + tmp_path: Path, +) -> None: + fixture = _sealed_fixture(tmp_path) + evidence = {"status": "fisher_h1_smoke_passed"} + report = launcher._canonical_json_bytes( + { + "artifact_kind": launcher.RUN_REPORT_KIND, + "canonical_evidence_sha256": _sha256(launcher._canonical_json_bytes(evidence)), + "evidence": evidence, + "schema_version": launcher.RUN_REPORT_SCHEMA, + } + ) + report_path = _write(tmp_path / "smoke" / "report.json", report) + marker_path = _write( + tmp_path / "smoke" / "FISHER_H1_SMOKE_COMPLETE", + launcher.FISHER_SMOKE_COMPLETE_BYTES, + ) + runner_arguments = list(fixture["runner_arguments"]) + runner_arguments.remove("--fisher-h1-smoke") + runner_arguments.extend( + [ + "--prior-fisher-h1-smoke-report", + str(report_path), + "--prior-fisher-h1-smoke-complete-marker", + str(marker_path), + ] + ) + + options = launcher._extract_runner_options(runner_arguments) + launcher._verify_bound_artifacts( + options, + runtime_manifest_path=fixture["runtime_path"], + ) + + +def test_embedded_bootstrap_authenticates_smoke_prerequisite_before_runner_load() -> None: + assert "_smoke(_runner_options)" in launcher.SEALED_BOOTSTRAP + assert "full calibration requires prior Fisher H=1 smoke prerequisites" in ( + launcher.SEALED_BOOTSTRAP + ) + + @pytest.mark.parametrize( ("field", "value"), [("schema_version", 3.0), ("dont_write_bytecode", True)], @@ -305,6 +454,68 @@ def test_runtime_parser_rejects_equality_compatible_json_types( launcher._parse_runtime_manifest(launcher._canonical_json_bytes(document)) +def test_runtime_parser_accepts_exact_dot_only_as_base_sys_path_sentinel( + tmp_path: Path, +) -> None: + fixture = _sealed_fixture(tmp_path) + document = json.loads(fixture["runtime_path"].read_bytes()) + + parsed = launcher._parse_runtime_manifest(launcher._canonical_json_bytes(document)) + + assert parsed["base_sys_path"] == ["Lib", "."] + + document["package_roots"][0]["import_path"] = "." + with pytest.raises(launcher.SealedLaunchError, match="canonical relative path"): + launcher._parse_runtime_manifest(launcher._canonical_json_bytes(document)) + + +def test_embedded_runtime_parser_applies_the_same_exact_dot_boundary( + tmp_path: Path, +) -> None: + fixture = _sealed_fixture(tmp_path) + + accepted = _run_embedded_manifest_boundary( + fixture, + pycache=tmp_path / "accepted-pycache", + ) + assert accepted.returncode != 0 + assert b"base sys.path is not a canonical relative path" not in accepted.stderr + assert b"point-used interpreter path drifted" in accepted.stderr + + document = json.loads(fixture["runtime_path"].read_bytes()) + document["package_roots"][0]["import_path"] = "." + fixture["runtime_path"].write_bytes(launcher._canonical_json_bytes(document)) + rejected_elsewhere = _run_embedded_manifest_boundary( + fixture, + pycache=tmp_path / "elsewhere-pycache", + ) + assert rejected_elsewhere.returncode != 0 + assert b"import path is not a canonical relative path" in rejected_elsewhere.stderr + + document["package_roots"][0]["import_path"] = "Lib/site-packages" + document["base_sys_path"] = ["Lib", "./"] + fixture["runtime_path"].write_bytes(launcher._canonical_json_bytes(document)) + rejected_unsafe = _run_embedded_manifest_boundary( + fixture, + pycache=tmp_path / "unsafe-pycache", + ) + assert rejected_unsafe.returncode != 0 + assert b"base sys.path is not a canonical relative path" in rejected_unsafe.stderr + + +@pytest.mark.parametrize("entry", ["./", "./Lib", "..", "Lib/.."]) +def test_runtime_parser_rejects_noncanonical_base_sys_path_dot_forms( + tmp_path: Path, + entry: str, +) -> None: + fixture = _sealed_fixture(tmp_path) + document = json.loads(fixture["runtime_path"].read_bytes()) + document["base_sys_path"] = ["Lib", entry] + + with pytest.raises(launcher.SealedLaunchError, match="canonical relative path"): + launcher._parse_runtime_manifest(launcher._canonical_json_bytes(document)) + + def test_identity_parser_rejects_float_schema_version(tmp_path: Path) -> None: fixture = _sealed_fixture(tmp_path) identity_path = fixture["runtime_path"].parent / "identity.json" @@ -318,6 +529,11 @@ def test_identity_parser_rejects_float_schema_version(tmp_path: Path) -> None: launcher._parse_identity(launcher._canonical_json_bytes(document)) +def test_launcher_and_embedded_bootstrap_require_identity_v5() -> None: + assert launcher.IDENTITY_SCHEMA == 5 + assert 'evidence.get("schema_version") != 5' in launcher.SEALED_BOOTSTRAP + + def test_embedded_bootstrap_repeats_exact_json_type_checks() -> None: assert 'type(evidence.get("schema_version")) is not int' in launcher.SEALED_BOOTSTRAP assert '_typed(root["launch_policy"], _policy' in launcher.SEALED_BOOTSTRAP @@ -384,20 +600,66 @@ def test_nonempty_pycache_is_rejected(tmp_path: Path) -> None: def test_bootstrap_rejects_flag_drift() -> None: completed = subprocess.run( - [sys.executable, "-S", "-c", launcher.SEALED_BOOTSTRAP], + [sys.executable, "-S", "-c", launcher.SEALED_STDIN_LOADER], + check=False, + capture_output=True, + input=launcher.SEALED_BOOTSTRAP_BYTES, + ) + + assert completed.returncode != 0 + assert b"sealed bootstrap startup flags drifted" in completed.stderr + + +def test_authenticated_stdin_loader_crosses_a_real_child_boundary( + tmp_path: Path, +) -> None: + pycache = tmp_path / "empty-pycache" + pycache.mkdir() + payload = b"import sys\nprint('authenticated-child:' + sys.argv[1])\nraise SystemExit(23)\n" + loader = launcher._authenticated_stdin_loader(payload) + + completed = subprocess.run( + [ + sys.executable, + "-I", + "-S", + "-B", + "-X", + f"pycache_prefix={pycache}", + "-X", + "utf8", + "-c", + loader, + "ok", + ], + check=False, + capture_output=True, + input=payload, + ) + + assert completed.returncode == 23 + assert completed.stdout.splitlines() == [b"authenticated-child:ok"] + assert completed.stderr == b"" + assert not any(pycache.iterdir()) + + +def test_authenticated_stdin_loader_rejects_modified_payload() -> None: + completed = subprocess.run( + [sys.executable, "-I", "-S", "-B", "-c", launcher.SEALED_STDIN_LOADER], check=False, capture_output=True, - text=True, + input=launcher.SEALED_BOOTSTRAP_BYTES + b"\n", ) assert completed.returncode != 0 - assert "sealed bootstrap startup flags drifted" in completed.stderr + assert b"sealed bootstrap stdin authentication failed" in completed.stderr def test_bootstrap_rejects_preloaded_sensitive_module(tmp_path: Path) -> None: pycache = tmp_path / "empty-pycache" pycache.mkdir() prefix = "import sys; sys.modules['_virtualenv'] = object()\n" + payload = (prefix + launcher.SEALED_BOOTSTRAP).encode("utf-8") completed = subprocess.run( [ sys.executable, @@ -409,15 +671,15 @@ def test_bootstrap_rejects_preloaded_sensitive_module(tmp_path: Path) -> None: "-X", "utf8", "-c", - prefix + launcher.SEALED_BOOTSTRAP, + launcher._authenticated_stdin_loader(payload), ], check=False, capture_output=True, - text=True, + input=payload, ) assert completed.returncode != 0 - assert "sealed bootstrap found a preloaded sensitive module" in completed.stderr + assert b"sealed bootstrap found a preloaded sensitive module" in completed.stderr assert not any(pycache.iterdir()) diff --git a/tests/test_launch_static_q468_stage_a.py b/tests/test_launch_static_q468_stage_a.py new file mode 100644 index 0000000..fb91084 --- /dev/null +++ b/tests/test_launch_static_q468_stage_a.py @@ -0,0 +1,731 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import json +import platform +import shutil +import struct +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +MODULE_PATH = ROOT / "scripts" / "launch_static_q468_stage_a.py" +SPEC = importlib.util.spec_from_file_location("launch_static_q468_stage_a_test", MODULE_PATH) +assert SPEC is not None and SPEC.loader is not None +launcher = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = launcher +SPEC.loader.exec_module(launcher) + + +def _digest(value: bytes | str) -> str: + if isinstance(value, str): + value = value.encode() + return hashlib.sha256(value).hexdigest() + + +def _write(path: Path, data: bytes) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return path + + +def _identity(bindings: dict[str, str]) -> bytes: + evidence = { + "schema_version": 5, + "identity_schema": "recurquant.experiment013.identity-frozen.v5", + "status": "frozen", + "phase": "stage_a", + "identity_only": True, + "promotion_required": False, + "promotion": {"explicit": True}, + "execution_bindings": bindings, + } + return launcher._canonical_json_bytes( + { + "canonical_evidence_sha256": _digest(launcher._canonical_json_bytes(evidence)), + "evidence": evidence, + } + ) + + +def _source_manifest(root: Path) -> bytes: + git_bytes = b"fixture Git executable\n" + _write(root / "toolchain" / "git.exe", git_bytes) + paths = [] + for relative in sorted( + { + launcher.RUNNER_SOURCE_PATH, + launcher.LAUNCHER_SOURCE_PATH, + launcher.CALIBRATION_LAUNCHER_SOURCE_PATH, + } + ): + path = root / Path(relative) + path.parent.mkdir(parents=True, exist_ok=True) + payload = relative.encode() + path.write_bytes(payload) + oid = "1" * 40 + paths.append( + { + "git_blob_oid": oid, + "index_blob_oid": oid, + "mode": "100644", + "path": relative, + "raw_sha256": _digest(payload), + "worktree_blob_oid": oid, + } + ) + document = { + "schema": "recurquant.experiment013.source-manifest.v2", + "profile": "experiment-013-static-q468-frozen-source-v2", + "object_format": "sha1", + "source_commit": "2" * 40, + "git_executable": { + "sha256": _digest(git_bytes), + "size_bytes": len(git_bytes), + }, + "repository_binding": {}, + "paths": paths, + } + document["canonical_manifest_sha256"] = _digest(launcher._pretty_json_bytes(document)) + return launcher._pretty_json_bytes(document) + + +def _runner_arguments(paths: dict[str, Path], digests: dict[str, str]) -> list[str]: + values = { + "--frozen-identity": paths["identity"], + "--stage-a-calibration-binding": paths["binding"], + "--repository-root": paths["root"], + "--source-commit": "2" * 40, + "--identity-commit": "3" * 40, + "--model-root": paths["root"], + "--cache-root": paths["root"], + "--input-bundle-root": paths["root"] / "bundle", + "--ruler-root": paths["root"], + "--output-dir": paths["root"] / "out", + "--runtime-manifest": paths["runtime"], + "--model-file-manifest": paths["model"], + "--parquet-materialization-manifest": paths["parquet"], + "--repository-source-manifest": paths["source"], + "--expected-runtime-manifest-sha256": digests["runtime"], + "--expected-model-file-manifest-sha256": digests["model"], + "--expected-parquet-materialization-manifest-sha256": digests["parquet"], + } + result = ["preflight"] + for option in sorted(values): + result.extend((option, str(values[option]))) + return result + + +def _load_calibration_launcher(name: str) -> Any: + path = ROOT / launcher.CALIBRATION_LAUNCHER_SOURCE_PATH + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + try: + spec.loader.exec_module(module) + except BaseException: + sys.modules.pop(name, None) + raise + return module + + +def _copy_test_runtime(destination: Path) -> Path: + source = Path(sys.base_prefix).resolve(strict=True) + destination.mkdir(parents=True) + interpreter_source = Path(getattr(sys, "_base_executable", sys.executable)).resolve(strict=True) + interpreter = destination / interpreter_source.name + shutil.copy2(interpreter_source, interpreter) + for path in source.iterdir(): + if path.is_file() and path.suffix.casefold() == ".dll": + shutil.copy2(path, destination / path.name) + + def ignore(_directory: str, names: list[str]) -> set[str]: + return { + name + for name in names + if name == "__pycache__" + or name == "site-packages" + or Path(name).suffix.casefold() in {".pyc", ".pyo", ".pth"} + } + + shutil.copytree(source / "Lib", destination / "Lib", ignore=ignore) + shutil.copytree(source / "DLLs", destination / "DLLs", ignore=ignore) + return interpreter + + +def _embedded_bootstrap_fixture(tmp_path: Path) -> dict[str, Any]: + calibration = _load_calibration_launcher("calibration_launcher_stage_a_embedded_test") + base = tmp_path / "base" + interpreter = _copy_test_runtime(base) + package_root = tmp_path / "packages" + git_executable = _write(tmp_path / "toolchain" / "git.exe", b"fixture Git executable\n") + git_executable, git_record = calibration._authenticated_git_executable(git_executable) + import_root = package_root / "Lib" / "site-packages" + _write(import_root / "demo" / "__init__.py", b"VALUE = 1\n") + _write( + import_root / "demo-1.0.dist-info" / "METADATA", + b"Metadata-Version: 2.1\nName: demo\nVersion: 1.0\n\n", + ) + _write( + import_root / "demo-1.0.dist-info" / "RECORD", + (b"demo-1.0.dist-info/METADATA,,\ndemo-1.0.dist-info/RECORD,,\ndemo/__init__.py,,\n"), + ) + + probe = subprocess.run( + [ + str(interpreter), + "-I", + "-S", + "-B", + "-X", + "utf8", + "-c", + "import json,sys;print(json.dumps(sys.path))", + ], + check=True, + capture_output=True, + text=True, + ) + base_resolved = base.resolve(strict=True) + base_sys_path: list[str] = [] + for raw_path in json.loads(probe.stdout): + path = Path(raw_path).resolve(strict=False) + if path == base_resolved: + base_sys_path.append(".") + else: + base_sys_path.append(path.relative_to(base_resolved).as_posix()) + + package_roots = {"packages": package_root.resolve(strict=True)} + import_paths = {"packages": "Lib/site-packages"} + interpreter_record = calibration._stable_file_record( + interpreter, + relative=interpreter.name, + context="Stage-A embedded-test interpreter", + ) + runtime = { + "artifact_kind": calibration.RUNTIME_MANIFEST_KIND, + "base_runtime_root": calibration.BASE_RUNTIME_ROOT_NAME, + "base_sys_path": base_sys_path, + "distributions": list(calibration._distribution_inventory(package_roots, import_paths)), + "git_executable": git_record, + "interpreter": { + "relative_path": interpreter.name, + "root": calibration.BASE_RUNTIME_ROOT_NAME, + "sha256": interpreter_record["sha256"], + "size_bytes": interpreter_record["size_bytes"], + }, + "launch_policy": calibration.SEALED_LAUNCH_POLICY, + "machine": { + "architecture": platform.architecture()[0], + "byteorder": sys.byteorder, + "machine": platform.machine(), + "pointer_bits": 8 * struct.calcsize("P"), + "system": platform.system(), + }, + "package_roots": [{"import_path": import_paths["packages"], "name": "packages"}], + "python": { + "abi_flags": getattr(sys, "abiflags", ""), + "cache_tag": sys.implementation.cache_tag, + "implementation": platform.python_implementation(), + "version": platform.python_version(), + }, + "runtime_trees": [ + { + "files": list(calibration._tree_files(base, context="embedded-test base")), + "kind": "base-runtime", + "name": "base-runtime", + }, + { + "files": list( + calibration._tree_files(package_root, context="embedded-test packages") + ), + "kind": "packages", + "name": "packages", + }, + ], + "schema_version": calibration.RUNTIME_MANIFEST_SCHEMA, + } + artifacts = tmp_path / "artifacts" + runtime_path = _write(artifacts / "runtime.json", calibration._canonical_json_bytes(runtime)) + + repository = tmp_path / "repository" + sentinel = tmp_path / "runner-boundary-reached.txt" + _write( + repository / launcher.RUNNER_SOURCE_PATH, + ( + "from pathlib import Path\n" + "def sealed_main(argv, *, base_runtime_root, package_roots, " + "package_import_paths, interpreter_path, pycache_prefix, " + "git_executable_path):\n" + " options = dict(zip(argv[1::2], argv[2::2], strict=True))\n" + f" Path({str(sentinel)!r}).write_text('stage-a-boundary\\n', encoding='utf-8')\n" + " return 37\n" + ).encode(), + ) + _write(repository / launcher.LAUNCHER_SOURCE_PATH, b"# authenticated fixture\n") + _write( + repository / launcher.CALIBRATION_LAUNCHER_SOURCE_PATH, + b"# authenticated fixture\n", + ) + source_paths = [] + for relative in sorted( + { + launcher.RUNNER_SOURCE_PATH, + launcher.LAUNCHER_SOURCE_PATH, + launcher.CALIBRATION_LAUNCHER_SOURCE_PATH, + } + ): + path = repository / Path(relative) + source_paths.append( + { + "git_blob_oid": "b" * 40, + "index_blob_oid": "b" * 40, + "mode": "100644", + "path": relative, + "raw_sha256": _digest(path.read_bytes()), + "worktree_blob_oid": "b" * 40, + } + ) + source_payload = { + "git_executable": { + "sha256": git_record["sha256"], + "size_bytes": git_record["size_bytes"], + }, + "object_format": "sha1", + "paths": source_paths, + "profile": "experiment-013-static-q468-frozen-source-v2", + "repository_binding": {}, + "schema": "recurquant.experiment013.source-manifest.v2", + "source_commit": "2" * 40, + } + source = { + **source_payload, + "canonical_manifest_sha256": _digest(launcher._pretty_json_bytes(source_payload)), + } + source_path = _write(artifacts / "source.json", launcher._pretty_json_bytes(source)) + model_path = _write(artifacts / "model.json", b'{"model":"fixture"}\n') + parquet_path = _write(artifacts / "parquet.json", b'{"parquet":"fixture"}\n') + binding_path = _write(artifacts / "binding.json", b'{"binding":"fixture"}\n') + bindings = { + "calibration_runtime_manifest_file_sha256": _digest(runtime_path.read_bytes()), + "model_file_manifest_file_sha256": _digest(model_path.read_bytes()), + "parquet_materialization_manifest_file_sha256": _digest(parquet_path.read_bytes()), + "repository_source_manifest_file_sha256": _digest(source_path.read_bytes()), + } + identity_path = _write(artifacts / "identity.json", _identity(bindings)) + paths = { + "binding": binding_path, + "identity": identity_path, + "model": model_path, + "parquet": parquet_path, + "root": repository, + "runtime": runtime_path, + "source": source_path, + } + runner_arguments = _runner_arguments( + paths, + { + "model": bindings["model_file_manifest_file_sha256"], + "parquet": bindings["parquet_materialization_manifest_file_sha256"], + "runtime": bindings["calibration_runtime_manifest_file_sha256"], + }, + ) + return { + "base": base, + "bootstrap": launcher._stage_a_bootstrap(calibration).encode("utf-8"), + "calibration": calibration, + "interpreter": interpreter, + "git_executable": git_executable, + "identity_path": identity_path, + "model_path": model_path, + "package_root": package_root, + "repository": repository, + "runner_arguments": runner_arguments, + "runner_path": repository / launcher.RUNNER_SOURCE_PATH, + "runtime_path": runtime_path, + "sentinel": sentinel, + "source_path": source_path, + } + + +def test_identity_parser_accepts_only_promoted_stage_a_v5() -> None: + bindings = {name: _digest(name) for name in launcher._BOUND_ARTIFACT_OPTIONS} + assert launcher._parse_identity(_identity(bindings)) == { + name: bindings[name] for name in sorted(bindings) + } + + root = launcher._strict_json(_identity(bindings), context="identity") + root["evidence"]["phase"] = "calibration" + root["canonical_evidence_sha256"] = _digest(launcher._canonical_json_bytes(root["evidence"])) + with pytest.raises(launcher.SealedStageALaunchError, match="promoted Stage-A"): + launcher._parse_identity(launcher._canonical_json_bytes(root)) + + +def test_runner_cli_forbids_method_seed_threshold_and_policy_options(tmp_path: Path) -> None: + paths = { + name: tmp_path / name + for name in ("identity", "binding", "runtime", "model", "parquet", "source") + } + paths["root"] = tmp_path + digests = {name: _digest(name) for name in ("runtime", "model", "parquet")} + arguments = _runner_arguments(paths, digests) + assert set(launcher._extract_options(arguments)) == launcher._REQUIRED_OPTIONS + for option in ("--methods", "--seed", "--threshold", "--q48-policy", "--uniform-policy"): + with pytest.raises(launcher.SealedStageALaunchError, match="frozen CLI"): + launcher._extract_options([*arguments, option, "bad"]) + + +def test_source_bootstrap_and_verifier_detect_byte_tamper(tmp_path: Path) -> None: + data = _source_manifest(tmp_path) + decoded = launcher._parse_source(data) + runner = launcher._verify_source(decoded, tmp_path) + assert runner == (tmp_path / Path(launcher.RUNNER_SOURCE_PATH)).resolve() + runner.write_bytes(b"tampered") + with pytest.raises(launcher.SealedStageALaunchError, match="source bytes drifted"): + launcher._verify_source(decoded, tmp_path) + + +def test_bound_artifacts_are_checked_before_runner_load(tmp_path: Path) -> None: + runtime = b"runtime" + model = b"model" + parquet = b"parquet" + source = _source_manifest(tmp_path) + files = { + "runtime": runtime, + "model": model, + "parquet": parquet, + "source": source, + "binding": b"binding-v3", + } + paths = {name: tmp_path / f"{name}.json" for name in files} + paths["root"] = tmp_path + for name, payload in files.items(): + paths[name].write_bytes(payload) + bindings = { + "calibration_runtime_manifest_file_sha256": _digest(runtime), + "model_file_manifest_file_sha256": _digest(model), + "parquet_materialization_manifest_file_sha256": _digest(parquet), + "repository_source_manifest_file_sha256": _digest(source), + } + paths["identity"] = tmp_path / "identity.json" + paths["identity"].write_bytes(_identity(bindings)) + arguments = _runner_arguments( + paths, + {"runtime": _digest(runtime), "model": _digest(model), "parquet": _digest(parquet)}, + ) + options = launcher._extract_options(arguments) + parsed, _source, runner = launcher._verify_bound_inputs( + options, runtime_manifest_path=paths["runtime"] + ) + assert parsed == bindings + assert runner.name == "screen_static_q468_stage_a.py" + + paths["model"].write_bytes(b"tampered") + with pytest.raises(launcher.SealedStageALaunchError, match="identity binding mismatch"): + launcher._verify_bound_inputs(options, runtime_manifest_path=paths["runtime"]) + + +def test_bootstrap_derivation_switches_schema_phase_runner_and_keeps_isolation() -> None: + calibration_path = ROOT / launcher.CALIBRATION_LAUNCHER_SOURCE_PATH + spec = importlib.util.spec_from_file_location( + "calibration_launcher_stage_a_test", calibration_path + ) + assert spec is not None and spec.loader is not None + calibration = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = calibration + try: + spec.loader.exec_module(calibration) + bootstrap = launcher._stage_a_bootstrap(calibration) + finally: + sys.modules.pop(spec.name, None) + assert 'evidence.get("schema_version") != 5' in bootstrap + assert 'evidence.get("phase") != "stage_a"' in bootstrap + assert launcher.RUNNER_SOURCE_PATH in bootstrap + assert "scripts/run_static_q468_calibration.py" not in bootstrap + assert "_stage_a_options" in bootstrap + assert "--stage-a-calibration-binding" in bootstrap + assert "--fisher-h1-smoke" not in bootstrap + assert "--prior-fisher-h1-smoke-report" not in bootstrap + assert "full calibration" not in bootstrap + assert 'promotion.get("explicit") is not True' in bootstrap + assert "isolated != 1" in bootstrap + assert "no_site != 1" in bootstrap + assert "dont_write_bytecode != 1" in bootstrap + + +def test_authenticated_stdin_loader_rejects_modified_bootstrap() -> None: + payload = b"raise SystemExit(23)" + loader = launcher._authenticated_stdin_loader(payload) + accepted = subprocess.run( + [sys.executable, "-I", "-S", "-B", "-c", loader], + check=False, + input=payload, + capture_output=True, + ) + assert accepted.returncode == 23 + + rejected = subprocess.run( + [sys.executable, "-I", "-S", "-B", "-c", loader], + check=False, + input=payload + b"\n", + capture_output=True, + ) + assert rejected.returncode != 23 + assert b"sealed Stage-A bootstrap stdin authentication failed" in rejected.stderr + + +@pytest.mark.skipif(sys.platform != "win32", reason="sealed runtime fixture is Windows-only") +def test_embedded_bootstrap_reaches_runner_boundary_and_rejects_bound_tamper( + tmp_path: Path, +) -> None: + fixture = _embedded_bootstrap_fixture(tmp_path) + + def run_bootstrap( + pycache: Path, + *, + offline: bool = True, + ) -> subprocess.CompletedProcess[bytes]: + pycache.mkdir() + scratch = pycache.parent / f"{pycache.name}-scratch" + scratch.mkdir() + command = launcher._sealed_argv( + interpreter=fixture["interpreter"], + bootstrap=fixture["bootstrap"], + runtime_manifest=fixture["runtime_path"], + base_runtime_root=fixture["base"], + package_roots={"packages": fixture["package_root"]}, + git_executable=fixture["git_executable"], + pycache_prefix=pycache, + scratch_directory=scratch, + runner_arguments=fixture["runner_arguments"], + ) + return subprocess.run( + command, + check=False, + cwd=fixture["base"], + env=launcher._sealed_environment(scratch_directory=scratch, offline=offline), + capture_output=True, + input=fixture["bootstrap"], + ) + + completed = run_bootstrap(tmp_path / "pycache-valid") + assert completed.returncode == 37, completed.stderr + assert fixture["sentinel"].read_text(encoding="utf-8") == "stage-a-boundary\n" + + fixture["sentinel"].unlink() + missing_offline = run_bootstrap(tmp_path / "pycache-missing-offline", offline=False) + assert missing_offline.returncode != 37 + assert b"mode-specific minimal contract" in missing_offline.stderr + assert not fixture["sentinel"].exists() + + fixture["model_path"].write_bytes(b"tampered\n") + rejected = run_bootstrap(tmp_path / "pycache-tampered") + assert rejected.returncode != 37 + assert b"identity binding mismatch: --model-file-manifest" in rejected.stderr + assert not fixture["sentinel"].exists() + + +def test_mode_specific_child_environments_strip_credentials_and_proxies( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + scratch = tmp_path / "scratch" + scratch.mkdir() + for name in ( + "GITHUB_TOKEN", + "HF_TOKEN", + "HTTPS_PROXY", + "HTTP_PROXY", + "ALL_PROXY", + ): + monkeypatch.setenv(name, "must-not-cross-child-boundary") + + networked = launcher._sealed_environment(scratch_directory=scratch, offline=False) + offline = launcher._sealed_environment(scratch_directory=scratch, offline=True) + + assert not {"HF_DATASETS_OFFLINE", "HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"} & set(networked) + assert { + "HF_DATASETS_OFFLINE": "1", + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + }.items() <= offline.items() + forbidden = {"GITHUB_TOKEN", "HF_TOKEN", "HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY"} + assert not forbidden & set(networked) + assert not forbidden & set(offline) + + +@pytest.mark.skipif(sys.platform != "win32", reason="sealed runtime fixture is Windows-only") +def test_offline_child_fatally_rejects_socket_connect_before_runner_side_effect( + tmp_path: Path, +) -> None: + fixture = _embedded_bootstrap_fixture(tmp_path) + fixture["runner_path"].write_bytes( + ( + "import socket\n" + "from pathlib import Path\n" + "def sealed_main(argv, *, base_runtime_root, package_roots, " + "package_import_paths, interpreter_path, pycache_prefix, " + "git_executable_path):\n" + " connection = socket.socket()\n" + " try:\n" + " connection.connect(('127.0.0.1', 9))\n" + " finally:\n" + " connection.close()\n" + f" Path({str(fixture['sentinel'])!r}).write_text('network-ran\\n')\n" + " return 37\n" + ).encode() + ) + source = launcher._strict_json(fixture["source_path"].read_bytes(), context="source") + for entry in source["paths"]: + if entry["path"] == launcher.RUNNER_SOURCE_PATH: + entry["raw_sha256"] = _digest(fixture["runner_path"].read_bytes()) + source_payload = dict(source) + source_payload.pop("canonical_manifest_sha256") + source["canonical_manifest_sha256"] = _digest(launcher._pretty_json_bytes(source_payload)) + fixture["source_path"].write_bytes(launcher._pretty_json_bytes(source)) + identity = launcher._strict_json( + fixture["identity_path"].read_bytes(), + context="identity", + ) + identity["evidence"]["execution_bindings"]["repository_source_manifest_file_sha256"] = _digest( + fixture["source_path"].read_bytes() + ) + identity["canonical_evidence_sha256"] = _digest( + launcher._canonical_json_bytes(identity["evidence"]) + ) + fixture["identity_path"].write_bytes(launcher._canonical_json_bytes(identity)) + pycache = tmp_path / "network-pycache" + scratch = tmp_path / "network-scratch" + pycache.mkdir() + scratch.mkdir() + command = launcher._sealed_argv( + interpreter=fixture["interpreter"], + bootstrap=fixture["bootstrap"], + runtime_manifest=fixture["runtime_path"], + base_runtime_root=fixture["base"], + package_roots={"packages": fixture["package_root"]}, + git_executable=fixture["git_executable"], + pycache_prefix=pycache, + scratch_directory=scratch, + runner_arguments=fixture["runner_arguments"], + ) + + completed = subprocess.run( + command, + check=False, + cwd=fixture["base"], + env=launcher._sealed_environment(scratch_directory=scratch, offline=True), + capture_output=True, + input=fixture["bootstrap"], + ) + + assert completed.returncode != 37 + assert b"forbids network access: socket.connect" in completed.stderr + assert not fixture["sentinel"].exists() + + +def test_split_launcher_arguments_requires_one_separator() -> None: + with pytest.raises(launcher.SealedStageALaunchError, match="exactly one"): + launcher._split_arguments(["a"]) + with pytest.raises(launcher.SealedStageALaunchError, match="exactly one"): + launcher._split_arguments(["a", "--", "b", "--", "c"]) + assert launcher._split_arguments(["host", "--", "preflight"]) == ( + ["host"], + ["preflight"], + ) + + +def test_launch_reauthenticates_after_child_and_uses_isolated_argv( + tmp_path: Path, monkeypatch: Any +) -> None: + runtime_path = tmp_path / "runtime.json" + runtime_path.write_bytes(b"runtime") + repository = tmp_path / "repo" + repository.mkdir() + base = tmp_path / "base" + packages = tmp_path / "packages" + base.mkdir() + packages.mkdir() + git_executable = _write(tmp_path / "toolchain" / "git.exe", b"fixture Git executable\n") + git_record = { + "absolute_path_sha256": _digest(str(git_executable.resolve()).casefold()), + "sha256": _digest(git_executable.read_bytes()), + "size_bytes": git_executable.stat().st_size, + } + source_git_record = { + "sha256": git_record["sha256"], + "size_bytes": git_record["size_bytes"], + } + runner_args = ["preflight"] + runner_args.extend( + pair + for option in sorted(launcher._REQUIRED_OPTIONS) + for pair in (option, str(repository) if option != "--source-commit" else "2" * 40) + ) + calls: list[str] = [] + options = { + option: value for option, value in zip(runner_args[1::2], runner_args[2::2], strict=True) + } + options["--repository-root"] = str(repository) + monkeypatch.setattr(launcher, "_extract_options", lambda args: options) + monkeypatch.setattr( + launcher, + "_verify_bound_inputs", + lambda *args, **kwargs: ( + calls.append("verify") + or ({}, {"paths": [], "git_executable": source_git_record}, Path("runner")) + ), + ) + fake_calibration = SimpleNamespace( + _parse_runtime_manifest=lambda data: {"git_executable": git_record}, + _verify_runtime=lambda *args, **kwargs: ( + base, + {"packages": packages}, + {"packages": "."}, + Path(sys.executable), + git_executable, + ), + _verify_empty_scratch=lambda path: None, + _assert_scratch_tree_has_no_reparse=lambda path: None, + ) + monkeypatch.setattr(launcher, "_load_calibration_launcher", lambda *args: fake_calibration) + monkeypatch.setattr(launcher, "_stage_a_bootstrap", lambda module: "raise SystemExit(0)") + child_modes: list[str] = [] + + def fake_run(command: list[str], **kwargs: Any) -> Any: + calls.append("subprocess") + child_modes.append(command[16]) + assert command[1:4] == ["-I", "-S", "-B"] + assert command[9] == launcher._authenticated_stdin_loader(b"raise SystemExit(0)") + assert kwargs["input"] == b"raise SystemExit(0)" + if command[16] == "prepare-inputs": + assert "HF_HUB_OFFLINE" not in kwargs["env"] + else: + assert kwargs["env"]["HF_HUB_OFFLINE"] == "1" + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(launcher.subprocess, "run", fake_run) + result = launcher.launch( + [ + "--base-runtime-root", + str(base), + "--git-executable", + str(git_executable), + "--package-root", + f"packages={packages}", + "--runtime-manifest", + str(runtime_path), + "--", + *runner_args, + ] + ) + assert result == 0 + assert calls == ["verify", "subprocess", "verify", "subprocess", "verify"] + assert child_modes == ["prepare-inputs", "preflight"] diff --git a/tests/test_resolve_static_q468_identity.py b/tests/test_resolve_static_q468_identity.py index 738c4d1..ba568e5 100644 --- a/tests/test_resolve_static_q468_identity.py +++ b/tests/test_resolve_static_q468_identity.py @@ -4,12 +4,19 @@ import importlib.util import json import sys +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import replace from pathlib import Path from types import SimpleNamespace from typing import Any from unittest.mock import patch import pytest +import torch + +from recurquant import static_q468 +from recurquant import static_q468_calibration as calibration REPOSITORY_ROOT = Path(__file__).resolve().parents[1] SCRIPT_PATH = REPOSITORY_ROOT / "scripts" / "resolve_static_q468_identity.py" @@ -29,9 +36,12 @@ FIXTURE_BINDING = { "calibration_identity_file_sha256": resolver.sha256_bytes(b"calibration-file"), "calibration_score_artifact_file_sha256": resolver.sha256_bytes(b"calibration-scores"), + "comparator_score_artifact_file_sha256": resolver.sha256_bytes(b"comparator-scores"), "split_half_stability_artifact_file_sha256": resolver.sha256_bytes(b"split-half"), + "static_fisher_k29334_policy_file_sha256": resolver.sha256_bytes(b"fisher-k29334-policy"), "static_k27030_policy_file_sha256": resolver.sha256_bytes(b"k27030-policy"), "static_k29334_policy_file_sha256": resolver.sha256_bytes(b"k29334-policy"), + "static_mse_k29334_policy_file_sha256": resolver.sha256_bytes(b"mse-k29334-policy"), } FIXTURE_EXECUTION_BINDINGS = { "repository_source_manifest_file_sha256": resolver.sha256_bytes(b"source-manifest"), @@ -47,6 +57,314 @@ def _hash(label: str) -> str: return resolver.sha256_bytes(label.encode()) +def _exact_code_vector(steps: int, *, from_end: bool = False) -> torch.Tensor: + rows = static_q468.FROZEN_QWEN35_STATIC_Q468_GEOMETRY.total_rows + codes = torch.zeros(rows, dtype=torch.uint8) + if from_end: + codes[-steps:] = 1 + else: + codes[:steps] = 1 + return codes + + +def _fake_policy( + *, + method_id: str, + marginal_steps: int, + codes: torch.Tensor, + identity_sha256: str, + tokenizer_manifest_sha256: str, + calibration_manifest_sha256: str, + calibration_scores_sha256: str, + source_commit: str, +) -> SimpleNamespace: + geometry = static_q468.FROZEN_QWEN35_STATIC_Q468_GEOMETRY + code_map_sha256 = calibration.static_q468_code_map_sha256( + codes, + geometry=geometry, + marginal_steps=marginal_steps, + ) + return SimpleNamespace( + method_id=method_id, + marginal_steps=marginal_steps, + geometry=geometry, + model_id=static_q468.PRIMARY_MODEL_ID, + model_revision=static_q468.PRIMARY_MODEL_REVISION, + tokenizer_id=static_q468.PRIMARY_TOKENIZER_ID, + tokenizer_revision=static_q468.PRIMARY_TOKENIZER_REVISION, + tokenizer_manifest_sha256=tokenizer_manifest_sha256, + transformers_version=static_q468.FROZEN_TRANSFORMERS_VERSION, + identity_artifact_sha256=identity_sha256, + source_commit=source_commit, + calibration_manifest_sha256=calibration_manifest_sha256, + calibration_scores_sha256=calibration_scores_sha256, + code_map_sha256=code_map_sha256, + precision_codes=lambda: codes.reshape( + geometry.layers, + geometry.heads, + geometry.key_rows, + ).clone(), + ) + + +@contextmanager +def _binding_v3_fixture() -> Iterator[SimpleNamespace]: + geometry = static_q468.FROZEN_QWEN35_STATIC_Q468_GEOMETRY + identity_bytes = b"frozen-identity-v5" + score_bytes = b"candidate-calibration-scores" + split_bytes = b"split-half-stability" + comparator_bytes = b"combined-comparator-scores" + policy_bytes = { + static_q468.STATIC_Q468_ABLATION_METHOD: b"static-k27030-policy", + static_q468.STATIC_Q468_PRIMARY_METHOD: b"static-k29334-policy", + static_q468.STATIC_Q468_MSE_METHOD: b"static-mse-k29334-policy", + static_q468.STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD: (b"static-fisher-k29334-policy"), + } + dependencies = { + "frozen_identity_artifact": identity_bytes, + "calibration_score_artifact": score_bytes, + "split_half_stability_artifact": split_bytes, + "static_k27030_policy_artifact": policy_bytes[static_q468.STATIC_Q468_ABLATION_METHOD], + "static_k29334_policy_artifact": policy_bytes[static_q468.STATIC_Q468_PRIMARY_METHOD], + "comparator_score_artifact": comparator_bytes, + "static_fisher_k29334_policy_artifact": policy_bytes[ + static_q468.STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD + ], + "static_mse_k29334_policy_artifact": policy_bytes[static_q468.STATIC_Q468_MSE_METHOD], + } + identity_sha256 = resolver.sha256_bytes(identity_bytes) + tokenizer_manifest_sha256 = _hash("binding-tokenizer-manifest") + identity_record_manifest_sha256 = _hash("complete-identity-v5-record-manifest") + source_commit_h0 = "a" * 40 + candidate_sequence_manifest_sha256 = _hash("candidate-sequence-manifest") + candidate_score_sha256 = _hash("candidate-raw-distortion-scores") + k27030_codes = _exact_code_vector(static_q468.FROZEN_STATIC_Q468_ABLATION_STEPS) + k29334_codes = _exact_code_vector(static_q468.FROZEN_STATIC_Q468_PRIMARY_STEPS) + + identity = SimpleNamespace( + file_sha256=identity_sha256, + canonical_evidence_sha256=_hash("identity-canonical-evidence"), + records=({},), + assignment=(), + assignment_sha256=_hash("identity-assignment"), + tokenizer_manifest_sha256=tokenizer_manifest_sha256, + ) + candidate_scores = SimpleNamespace( + artifact_kind=calibration.CALIBRATION_SCORE_ARTIFACT_KIND, + file_sha256=resolver.sha256_bytes(score_bytes), + calibration_identity_sha256=identity_sha256, + calibration_scores_sha256=candidate_score_sha256, + aggregate=SimpleNamespace( + identity_record_manifest_sha256=identity_record_manifest_sha256, + sequence_score_manifest_sha256=candidate_sequence_manifest_sha256, + ), + allocations=( + ( + static_q468.FROZEN_STATIC_Q468_ABLATION_STEPS, + k27030_codes, + calibration.static_q468_code_map_sha256( + k27030_codes, + geometry=geometry, + marginal_steps=static_q468.FROZEN_STATIC_Q468_ABLATION_STEPS, + ), + ), + ( + static_q468.FROZEN_STATIC_Q468_PRIMARY_STEPS, + k29334_codes, + calibration.static_q468_code_map_sha256( + k29334_codes, + geometry=geometry, + marginal_steps=static_q468.FROZEN_STATIC_Q468_PRIMARY_STEPS, + ), + ), + ), + ) + split = SimpleNamespace( + file_sha256=resolver.sha256_bytes(split_bytes), + identity_file_sha256=identity_sha256, + canonical_identity_sha256=identity.canonical_evidence_sha256, + resolver_assignment_sha256=identity.assignment_sha256, + full_sequence_score_manifest_sha256=candidate_sequence_manifest_sha256, + full_calibration_scores_sha256=candidate_score_sha256, + half_a_aggregate=SimpleNamespace( + identity_record_manifest_sha256=_hash("half-a-record-manifest") + ), + half_b_aggregate=SimpleNamespace( + identity_record_manifest_sha256=_hash("half-b-record-manifest") + ), + ) + + row = torch.arange(geometry.total_rows, dtype=torch.float64) + selector_specs = ( + ( + calibration.FROZEN_UNWEIGHTED_MSE_PROFILE, + ( + ((row + 1) % 997) / 997, + ((row + 5) % 991) / 991, + ((row + 9) % 983) / 983, + ), + False, + ), + ( + calibration.FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE, + ( + ((row + 13) % 977) / 977, + ((row + 17) % 971) / 971, + ((row + 21) % 967) / 967, + ), + True, + ), + ) + selectors: dict[str, SimpleNamespace] = {} + policies: dict[str, SimpleNamespace] = { + static_q468.STATIC_Q468_ABLATION_METHOD: _fake_policy( + method_id=static_q468.STATIC_Q468_ABLATION_METHOD, + marginal_steps=static_q468.FROZEN_STATIC_Q468_ABLATION_STEPS, + codes=k27030_codes, + identity_sha256=identity_sha256, + tokenizer_manifest_sha256=tokenizer_manifest_sha256, + calibration_manifest_sha256=candidate_sequence_manifest_sha256, + calibration_scores_sha256=candidate_score_sha256, + source_commit=source_commit_h0, + ), + static_q468.STATIC_Q468_PRIMARY_METHOD: _fake_policy( + method_id=static_q468.STATIC_Q468_PRIMARY_METHOD, + marginal_steps=static_q468.FROZEN_STATIC_Q468_PRIMARY_STEPS, + codes=k29334_codes, + identity_sha256=identity_sha256, + tokenizer_manifest_sha256=tokenizer_manifest_sha256, + calibration_manifest_sha256=candidate_sequence_manifest_sha256, + calibration_scores_sha256=candidate_score_sha256, + source_commit=source_commit_h0, + ), + } + for method_id, scores, reverse_codes in selector_specs: + codes = _exact_code_vector( + static_q468.FROZEN_STATIC_Q468_PRIMARY_STEPS, + from_end=reverse_codes, + ) + aggregate_score_sha256 = _hash(f"{method_id}-aggregate-evidence") + aggregate = SimpleNamespace( + identity_record_manifest_sha256=identity_record_manifest_sha256, + sequence_score_manifest_sha256=_hash(f"{method_id}-sequence-manifest"), + aggregate_scores_sha256=aggregate_score_sha256, + scores=lambda values=scores: values, + ) + selector = SimpleNamespace( + method_id=method_id, + aggregate=aggregate, + calibration_scores_sha256=aggregate_score_sha256, + marginal_steps=static_q468.FROZEN_STATIC_Q468_PRIMARY_STEPS, + precision_codes=codes, + code_map_sha256=calibration.static_q468_code_map_sha256( + codes, + geometry=geometry, + marginal_steps=static_q468.FROZEN_STATIC_Q468_PRIMARY_STEPS, + ), + ) + selectors[method_id] = selector + policies[method_id] = _fake_policy( + method_id=method_id, + marginal_steps=static_q468.FROZEN_STATIC_Q468_PRIMARY_STEPS, + codes=codes, + identity_sha256=identity_sha256, + tokenizer_manifest_sha256=tokenizer_manifest_sha256, + calibration_manifest_sha256=aggregate.sequence_score_manifest_sha256, + calibration_scores_sha256=static_q468.static_q468_distortion_sha256( + *scores, + geometry=geometry, + ), + source_commit=source_commit_h0, + ) + comparator_scores = SimpleNamespace( + selectors=selectors, + calibration_identity_sha256=identity_sha256, + file_sha256=resolver.sha256_bytes(comparator_bytes), + ) + policies_by_bytes = {policy_bytes[method_id]: policy for method_id, policy in policies.items()} + + def deserialize_policy(data: bytes) -> SimpleNamespace: + try: + return policies_by_bytes[data] + except KeyError as error: + raise ValueError("unknown policy fixture bytes") from error + + def rebuild_policy(*_scores: torch.Tensor, method_id: str, **_kwargs: object) -> object: + return policies[method_id] + + def serialize_policy(policy: SimpleNamespace) -> bytes: + return policy_bytes[policy.method_id] + + state = SimpleNamespace( + dependencies=dependencies, + identity=identity, + candidate_scores=candidate_scores, + split=split, + comparator_scores=comparator_scores, + policies=policies, + policy_bytes=policy_bytes, + identity_record_manifest_sha256=identity_record_manifest_sha256, + ) + with ( + patch.object( + resolver, + "deserialize_frozen_calibration_identity_artifact", + return_value=identity, + ), + patch.object( + resolver, + "_identity_half_record_manifests", + return_value={ + "a": split.half_a_aggregate.identity_record_manifest_sha256, + "b": split.half_b_aggregate.identity_record_manifest_sha256, + }, + ), + patch.object( + calibration, + "calibration_identity_record_manifest_sha256", + return_value=identity_record_manifest_sha256, + ), + patch.object( + calibration, + "deserialize_calibration_score_artifact", + return_value=candidate_scores, + ), + patch.object( + calibration, + "deserialize_frozen_split_half_stability_artifact", + return_value=split, + ), + patch.object( + calibration, + "deserialize_comparator_score_artifact", + return_value=comparator_scores, + ), + patch.object( + static_q468, + "deserialize_static_rht_q468_policy", + side_effect=deserialize_policy, + ), + patch.object( + static_q468, + "build_static_rht_q468_policy", + side_effect=rebuild_policy, + ), + patch.object( + static_q468, + "serialize_static_rht_q468_policy", + side_effect=serialize_policy, + ), + ): + yield state + + +def _reauthenticated_binding_bytes(document: dict[str, Any]) -> bytes: + document["canonical_evidence_sha256"] = resolver.sha256_bytes( + resolver.canonical_json_bytes(document["evidence"]) + ) + return resolver.canonical_json_bytes(document) + + def _datasets() -> list[dict[str, Any]]: return [ { @@ -57,8 +375,8 @@ def _datasets() -> list[dict[str, Any]]: "split": "train", "canonical_id_field": "task_id", "canonical_id_manifest_sha256": resolver.mbpp_calibration_identity()[1], - "formatter_id": "recurquant.mbpp.v0.2", - "formatter_sha256": _hash("mbpp-formatter"), + "formatter_id": resolver.FROZEN_FORMATTER_IDS["mbpp"], + "formatter_sha256": resolver.FROZEN_STATIC_FORMATTER_SHA256["mbpp"], }, { "key": "pg19", @@ -68,8 +386,8 @@ def _datasets() -> list[dict[str, Any]]: "split": "validation", "canonical_id_field": "url", "canonical_id_manifest_sha256": _hash("pg19-id-manifest"), - "formatter_id": "recurquant.pg19.contiguous.v1", - "formatter_sha256": _hash("pg19-formatter"), + "formatter_id": resolver.FROZEN_FORMATTER_IDS["pg19"], + "formatter_sha256": resolver.FROZEN_STATIC_FORMATTER_SHA256["pg19"], }, { "key": "ruler", @@ -79,7 +397,7 @@ def _datasets() -> list[dict[str, Any]]: "split": "generated", "canonical_id_field": "configuration_id", "canonical_id_manifest_sha256": _hash("ruler-id-manifest"), - "formatter_id": "recurquant.ruler.official.v1", + "formatter_id": resolver.FROZEN_FORMATTER_IDS["ruler"], "formatter_sha256": _hash("ruler-formatter"), }, { @@ -90,8 +408,8 @@ def _datasets() -> list[dict[str, Any]]: "split": "test", "canonical_id_field": "task_id", "canonical_id_manifest_sha256": _hash("humaneval-id-manifest"), - "formatter_id": "recurquant.humaneval-plus.v1", - "formatter_sha256": _hash("humaneval-formatter"), + "formatter_id": resolver.FROZEN_FORMATTER_IDS["humaneval_plus"], + "formatter_sha256": resolver.FROZEN_STATIC_FORMATTER_SHA256["humaneval_plus"], }, ] @@ -138,6 +456,7 @@ def _record( }[family] label = f"{family}-{canonical_id}-{config}-{seed}-{sequence_length}" sequence_hash = _hash(f"sequence-tokens-{label}") + sequence_token_ids = tuple(range(sequence_length)) token_span = { "prefill_start": 0, "prefill_stop": prefill_stop, @@ -172,6 +491,7 @@ def _record( sequence_token_ids_sha256_value=sequence_hash, token_span=token_span, ), + "fisher_boundary": resolver.build_fisher_boundary_contract(sequence_token_ids), } record["identity_record_sha256"] = resolver.identity_record_sha256(record) return record @@ -184,6 +504,9 @@ def _refresh_record_lineage(record: dict[str, Any]) -> None: sequence_token_ids_sha256_value=record["sequence_token_ids_sha256"], token_span=record["token_span"], ) + record["fisher_boundary"] = resolver.build_fisher_boundary_contract( + tuple(range(record["sequence_length"])) + ) record["identity_record_sha256"] = resolver.identity_record_sha256(record) @@ -193,7 +516,7 @@ def _stage_a_source() -> dict[str, Any]: records.append( _record( family="pg19", - canonical_id=f"book-{rank}", + canonical_id=f"http://www.gutenberg.org/ebooks/{10_000 + rank}", config="default", rank=rank, seed=None, @@ -212,7 +535,12 @@ def _stage_a_source() -> dict[str, Any]: records.append( _record( family="ruler", - canonical_id=f"{config}-4096-2339", + canonical_id=resolver.ruler_canonical_id( + category=category, + config=config, + configured_length=4_096, + seed=2_339, + ), config=config, rank=rank, seed=2_339, @@ -304,6 +632,146 @@ def test_stage_a_candidate_is_deterministic_and_complete() -> None: ) +def test_fisher_boundary_roundtrip_binds_h1_positions_and_ordered_token_hashes() -> None: + token_ids = tuple(100 + index for index in range(19)) + + boundary = resolver.build_fisher_boundary_contract(token_ids) + normalized = resolver._normalize_fisher_boundary( + copy.deepcopy(boundary), + sequence_length=len(token_ids), + context="fixture.fisher_boundary", + ) + + expected_boundaries = list(resolver.anchor_positions(len(token_ids) - 2)) + expected_inputs = [position + 1 for position in expected_boundaries] + expected_targets = [position + 2 for position in expected_boundaries] + assert normalized == boundary + assert boundary["schema"] == resolver.FISHER_BOUNDARY_SCHEMA + assert boundary["horizon"] == 1 + assert boundary["boundary_positions"] == expected_boundaries + assert boundary["input_positions"] == expected_inputs + assert boundary["target_positions"] == expected_targets + assert boundary["input_token_ids_sha256"] == resolver._fisher_boundary_token_ids_sha256( + [token_ids[position] for position in expected_inputs], role="input" + ) + assert boundary["target_token_ids_sha256"] == resolver._fisher_boundary_token_ids_sha256( + [token_ids[position] for position in expected_targets], role="target" + ) + assert boundary["fisher_boundary_sha256"] == resolver.fisher_boundary_sha256(boundary) + assert "input_token_ids" not in boundary + assert "target_token_ids" not in boundary + + +def test_fisher_boundary_self_hash_and_record_hash_tampering_fail_closed() -> None: + source = _stage_a_source() + row = source["records"][0] + row["fisher_boundary"]["fisher_boundary_sha256"] = "0" * 64 + with pytest.raises(ValueError, match="self-hash drifted"): + _build_candidate(source) + + source = _stage_a_source() + row = source["records"][0] + row["fisher_boundary"]["input_token_ids_sha256"] = "0" * 64 + row["fisher_boundary"]["fisher_boundary_sha256"] = resolver.fisher_boundary_sha256( + row["fisher_boundary"] + ) + with pytest.raises(ValueError, match="identity record SHA-256 drifted"): + _build_candidate(source) + + +def test_fisher_boundary_off_by_one_is_rejected_after_rehashing_both_layers() -> None: + source = _stage_a_source() + row = source["records"][0] + row["fisher_boundary"]["boundary_positions"][0] += 1 + row["fisher_boundary"]["fisher_boundary_sha256"] = resolver.fisher_boundary_sha256( + row["fisher_boundary"] + ) + row["identity_record_sha256"] = resolver.identity_record_sha256(row) + + with pytest.raises(ValueError, match=r"B\(T\)=anchor_positions\(T-2\)"): + _build_candidate(source) + + +def test_fisher_boundary_rejects_boolean_horizon_and_malformed_hash() -> None: + source = _stage_a_source() + row = source["records"][0] + row["fisher_boundary"]["horizon"] = True + row["fisher_boundary"]["fisher_boundary_sha256"] = resolver.fisher_boundary_sha256( + row["fisher_boundary"] + ) + row["identity_record_sha256"] = resolver.identity_record_sha256(row) + with pytest.raises(ValueError, match="horizon must be an integer"): + _build_candidate(source) + + source = _stage_a_source() + row = source["records"][0] + row["fisher_boundary"]["target_token_ids_sha256"] = "not-a-sha256" + row["fisher_boundary"]["fisher_boundary_sha256"] = resolver.fisher_boundary_sha256( + row["fisher_boundary"] + ) + row["identity_record_sha256"] = resolver.identity_record_sha256(row) + with pytest.raises(ValueError, match="target_token_ids_sha256 must be a lowercase SHA-256"): + _build_candidate(source) + + +def test_fisher_boundary_rejects_sequences_shorter_than_three_tokens() -> None: + with pytest.raises(ValueError, match="at least three tokens"): + resolver.build_fisher_boundary_contract((1, 2)) + + boundary = resolver.build_fisher_boundary_contract((1, 2, 3)) + with pytest.raises(ValueError, match="at least three tokens"): + resolver._normalize_fisher_boundary( + boundary, + sequence_length=2, + context="fixture.fisher_boundary", + ) + + +def test_v4_input_candidate_and_frozen_schemas_are_rejected() -> None: + source = _stage_a_source() + source["schema"] = "recurquant.experiment013.identity-input.v4" + with pytest.raises(ValueError, match="identity input schema drifted"): + _build_candidate(source) + + candidate = _build_candidate(_stage_a_source()) + candidate["evidence"]["identity_schema"] = "recurquant.experiment013.identity-candidate.v4" + candidate["canonical_evidence_sha256"] = resolver.sha256_bytes( + resolver.canonical_json_bytes(candidate["evidence"]) + ) + with pytest.raises(ValueError, match="candidate identity_schema drifted"): + resolver.validate_candidate_artifact(candidate) + + candidate = _build_candidate(_stage_a_source()) + candidate_hash = resolver.sha256_bytes(resolver.canonical_json_bytes(candidate)) + verified = SimpleNamespace(binding=dict(FIXTURE_BINDING)) + with patch.object( + resolver, + "deserialize_stage_a_calibration_binding_artifact", + return_value=verified, + ): + frozen = resolver.promote_candidate( + candidate, + candidate_file_sha256=candidate_hash, + calibration_binding_artifact=FIXTURE_BINDING_ARTIFACT, + ) + frozen["evidence"]["identity_schema"] = "recurquant.experiment013.identity-frozen.v4" + frozen["canonical_evidence_sha256"] = resolver.sha256_bytes( + resolver.canonical_json_bytes(frozen["evidence"]) + ) + with ( + patch.object( + resolver, + "deserialize_stage_a_calibration_binding_artifact", + return_value=verified, + ), + pytest.raises(ValueError, match="frozen Stage-A identity contract drifted"), + ): + resolver.deserialize_frozen_stage_a_identity_artifact( + resolver.canonical_json_bytes(frozen), + calibration_binding_artifact=FIXTURE_BINDING_ARTIFACT, + ) + + @pytest.mark.parametrize( ("mutate", "message"), [ @@ -339,6 +807,197 @@ def test_stage_a_candidate_requires_and_matches_a_verified_binding_artifact() -> _build_candidate(source) +def test_stage_a_binding_v3_round_trips_exact_eight_embedded_dependencies() -> None: + with _binding_v3_fixture() as fixture: + artifact = resolver.build_stage_a_calibration_binding_artifact(**fixture.dependencies) + decoded = resolver.deserialize_stage_a_calibration_binding_artifact(artifact) + + document = json.loads(artifact) + expected_dependencies = { + "calibration_score_artifact", + "comparator_score_artifact", + "frozen_identity_artifact", + "split_half_stability_artifact", + "static_fisher_k29334_policy_artifact", + "static_k27030_policy_artifact", + "static_k29334_policy_artifact", + "static_mse_k29334_policy_artifact", + } + assert resolver.STAGE_A_BINDING_ARTIFACT_SCHEMA_VERSION == 3 + assert resolver.STAGE_A_BINDING_ARTIFACT_REVISION.endswith("-v3") + assert document["schema_version"] == 3 + assert document["evidence"]["artifact_revision"].endswith("-v3") + assert set(document["evidence"]["dependencies_base64"]) == expected_dependencies + assert set(document["evidence"]["dependency_file_sha256"]) == expected_dependencies + assert set(decoded.binding) == resolver.CALIBRATION_BINDING_FIELDS + assert decoded.binding["comparator_score_artifact_file_sha256"] == resolver.sha256_bytes( + fixture.dependencies["comparator_score_artifact"] + ) + assert decoded.binding["static_fisher_k29334_policy_file_sha256"] == ( + resolver.sha256_bytes(fixture.dependencies["static_fisher_k29334_policy_artifact"]) + ) + assert decoded.binding["static_mse_k29334_policy_file_sha256"] == resolver.sha256_bytes( + fixture.dependencies["static_mse_k29334_policy_artifact"] + ) + with pytest.raises(TypeError): + decoded.binding["calibration_identity_file_sha256"] = "0" * 64 + with pytest.raises(TypeError): + decoded.dependency_file_sha256["frozen_identity_artifact"] = "0" * 64 + with pytest.raises(AttributeError): + decoded.file_sha256 = "0" * 64 + + +def test_stage_a_binding_rejects_v2_missing_extra_and_one_byte_dependency_tamper() -> None: + with _binding_v3_fixture() as fixture: + artifact = resolver.build_stage_a_calibration_binding_artifact(**fixture.dependencies) + + legacy = json.loads(artifact) + legacy["schema_version"] = 2 + legacy["evidence"]["artifact_revision"] = "experiment-013-stage-a-calibration-binding-v2" + with pytest.raises(ValueError, match="kind or schema drifted"): + resolver.deserialize_stage_a_calibration_binding_artifact( + _reauthenticated_binding_bytes(legacy) + ) + + missing = json.loads(artifact) + del missing["evidence"]["dependencies_base64"]["comparator_score_artifact"] + with pytest.raises(ValueError, match="dependencies fields drifted"): + resolver.deserialize_stage_a_calibration_binding_artifact( + _reauthenticated_binding_bytes(missing) + ) + + extra = json.loads(artifact) + extra["evidence"]["dependencies_base64"]["ninth_dependency"] = resolver._canonical_b64( + b"forbidden", context="fixture" + ) + with pytest.raises(ValueError, match="dependencies fields drifted"): + resolver.deserialize_stage_a_calibration_binding_artifact( + _reauthenticated_binding_bytes(extra) + ) + + tampered = json.loads(artifact) + original = resolver._decode_canonical_b64( + tampered["evidence"]["dependencies_base64"]["comparator_score_artifact"], + context="fixture", + ) + changed = bytes([original[0] ^ 1]) + original[1:] + tampered["evidence"]["dependencies_base64"]["comparator_score_artifact"] = ( + resolver._canonical_b64(changed, context="fixture") + ) + with pytest.raises(ValueError, match="dependency bytes differ"): + resolver.deserialize_stage_a_calibration_binding_artifact( + _reauthenticated_binding_bytes(tampered) + ) + + +def test_stage_a_binding_rejects_cross_profile_policy_swap() -> None: + with _binding_v3_fixture() as fixture: + dependencies = dict(fixture.dependencies) + dependencies["static_fisher_k29334_policy_artifact"] = fixture.dependencies[ + "static_mse_k29334_policy_artifact" + ] + dependencies["static_mse_k29334_policy_artifact"] = fixture.dependencies[ + "static_fisher_k29334_policy_artifact" + ] + with pytest.raises(ValueError, match="does not satisfy its frozen K29334 geometry"): + resolver.build_stage_a_calibration_binding_artifact(**dependencies) + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + ( + lambda fixture, policy: setattr(policy, "model_id", "Qwen/wrong-model"), + "frozen model contract drifted", + ), + ( + lambda fixture, policy: setattr( + policy, + "tokenizer_manifest_sha256", + _hash("wrong-tokenizer-manifest"), + ), + "frozen identity binding drifted", + ), + ( + lambda fixture, policy: setattr(policy, "source_commit", "b" * 40), + "source commit differs from H0", + ), + ( + lambda fixture, policy: setattr( + policy, + "marginal_steps", + static_q468.FROZEN_STATIC_Q468_PRIMARY_STEPS - 1, + ), + "frozen K29334 geometry", + ), + ( + lambda fixture, policy: setattr( + policy, + "geometry", + replace( + static_q468.FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + target_resident_bytes=(static_q468.FROZEN_STATELEASE_RESIDENT_BYTES + 8), + ), + ), + "frozen K29334 geometry", + ), + ( + lambda fixture, policy: setattr( + policy, + "identity_artifact_sha256", + _hash("wrong-frozen-identity"), + ), + "frozen identity binding drifted", + ), + ( + lambda fixture, policy: setattr( + policy, + "calibration_manifest_sha256", + _hash("wrong-comparator-sequence-manifest"), + ), + "decoded selector scores", + ), + ( + lambda fixture, policy: setattr( + policy, + "calibration_scores_sha256", + _hash("wrong-raw-distortion-hash"), + ), + "raw distortion hash", + ), + ( + lambda fixture, policy: setattr( + policy, + "code_map_sha256", + _hash("wrong-comparator-code-map"), + ), + "code map differs from its exact allocation", + ), + ], +) +def test_stage_a_binding_rederives_every_comparator_policy_contract( + mutation: Any, + message: str, +) -> None: + with _binding_v3_fixture() as fixture: + policy = fixture.policies[static_q468.STATIC_Q468_MSE_METHOD] + mutation(fixture, policy) + with pytest.raises(ValueError, match=message): + resolver.build_stage_a_calibration_binding_artifact(**fixture.dependencies) + + +def test_stage_a_binding_rejects_incomplete_comparator_identity_manifest() -> None: + with _binding_v3_fixture() as fixture: + selector = fixture.comparator_scores.selectors[ + calibration.FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE + ] + selector.aggregate.identity_record_manifest_sha256 = _hash( + "incomplete-comparator-identity-record-manifest" + ) + with pytest.raises(ValueError, match="complete frozen identity"): + resolver.build_stage_a_calibration_binding_artifact(**fixture.dependencies) + + def test_raw_content_and_unknown_fields_fail_closed() -> None: source = _stage_a_source() source["records"][0]["prompt"] = "raw protected text" @@ -410,6 +1069,105 @@ def test_dataset_revision_must_match_explicit_cli_contract() -> None: _build_candidate(source) +@pytest.mark.parametrize( + ("phase", "pg19_split"), + [("calibration", "train"), ("stage_a", "validation")], +) +def test_dataset_contracts_bind_phase_splits_and_exact_formatters( + phase: str, + pg19_split: str, +) -> None: + datasets = _datasets() + next(item for item in datasets if item["key"] == "pg19")["split"] = pg19_split + normalized = resolver._validate_dataset_contracts( + datasets, + expected_revisions=REVISIONS, + phase=phase, + ) + assert {item["key"]: item["split"] for item in normalized} == ( + resolver.FROZEN_DATASET_SPLITS[phase] + ) + assert {item["key"]: item["formatter_id"] for item in normalized} == ( + resolver.FROZEN_FORMATTER_IDS + ) + + next(item for item in datasets if item["key"] == "pg19")["split"] = ( + "validation" if phase == "calibration" else "train" + ) + with pytest.raises(ValueError, match=rf"{phase} pg19 dataset split must be"): + resolver._validate_dataset_contracts( + datasets, + expected_revisions=REVISIONS, + phase=phase, + ) + + datasets = _datasets() + next(item for item in datasets if item["key"] == "pg19")["split"] = pg19_split + next(item for item in datasets if item["key"] == "ruler")["formatter_id"] = ( + "recurquant.ruler-attacker-controlled.v1" + ) + with pytest.raises(ValueError, match="ruler formatter ID must be"): + resolver._validate_dataset_contracts( + datasets, + expected_revisions=REVISIONS, + phase=phase, + ) + + datasets = _datasets() + next(item for item in datasets if item["key"] == "pg19")["split"] = pg19_split + next(item for item in datasets if item["key"] == "pg19")["formatter_sha256"] = "0" * 64 + with pytest.raises(ValueError, match="pg19 formatter SHA-256 drifted"): + resolver._validate_dataset_contracts( + datasets, + expected_revisions=REVISIONS, + phase=phase, + ) + + +@pytest.mark.parametrize( + ("family", "bad_id", "message"), + [ + ("pg19", "https://www.gutenberg.org/ebooks/10000", "exact http"), + ("pg19", "http://www.gutenberg.org/ebooks/010000", "exact http"), + ("pg19", "http://www.gutenberg.org/ebooks/10000?raw=prompt", "exact http"), + ("humaneval_plus", "HumanEvalPlus/0", "HumanEval/0..163"), + ("humaneval_plus", "HumanEval/01", "HumanEval/0..163"), + ("humaneval_plus", "HumanEval/164", "HumanEval/0..163"), + ], +) +def test_canonical_dataset_identifier_shapes_fail_closed( + family: str, + bad_id: str, + message: str, +) -> None: + source = _stage_a_source() + row = next(item for item in source["records"] if item["family"] == family) + row["canonical_id"] = bad_id + + with pytest.raises(ValueError, match=message): + _build_candidate(source) + + +@pytest.mark.parametrize( + ("bad_id", "message"), + [ + ("http://www.gutenberg.org/ebooks/10000\nraw-prompt", "control character"), + ("x" * (resolver.MAX_METADATA_STRING_LENGTH + 1), "metadata length limit"), + ("def solve(): return 'raw prompt'", "whitespace or raw content"), + ], +) +def test_metadata_strings_reject_controls_overlong_values_and_raw_content( + bad_id: str, + message: str, +) -> None: + source = _stage_a_source() + row = next(item for item in source["records"] if item["family"] == "pg19") + row["canonical_id"] = bad_id + + with pytest.raises(ValueError, match=message): + _build_candidate(source) + + def test_ruler_category_config_and_actual_length_are_independently_bound() -> None: source = _stage_a_source() ruler = next(row for row in source["records"] if row["family"] == "ruler") @@ -420,6 +1178,17 @@ def test_ruler_category_config_and_actual_length_are_independently_bound() -> No source = _stage_a_source() ruler = next(row for row in source["records"] if row["family"] == "ruler") ruler["configured_length"] = 4_095 + ruler["canonical_id"] = resolver.ruler_canonical_id( + category=ruler["ruler_category"], + config=ruler["config"], + configured_length=ruler["configured_length"], + seed=ruler["seed"], + ) + ruler["selection_sha256"] = resolver.selection_sha256( + resolver.RULER_STAGE_A_SELECTION_NAMESPACE, + ruler["canonical_id"], + ) + _refresh_record_lineage(ruler) with pytest.raises(ValueError, match="exceeds the RULER configured length"): _build_candidate(source) @@ -430,6 +1199,61 @@ def test_ruler_category_config_and_actual_length_are_independently_bound() -> No _build_candidate(source) +def test_ruler_canonical_id_is_derived_from_the_complete_generation_tuple() -> None: + source = _stage_a_source() + ruler = next(row for row in source["records"] if row["family"] == "ruler") + ruler["canonical_id"] = "forged-ruler-id" + ruler["selection_sha256"] = resolver.selection_sha256( + resolver.RULER_STAGE_A_SELECTION_NAMESPACE, + ruler["canonical_id"], + ) + _refresh_record_lineage(ruler) + + with pytest.raises(ValueError, match="RULER canonical ID drifted"): + _build_candidate(source) + + +@pytest.mark.parametrize("family", ["pg19", "humaneval_plus"]) +def test_non_ruler_record_configs_are_exact(family: str) -> None: + source = _stage_a_source() + row = next(item for item in source["records"] if item["family"] == family) + row["config"] = "forged-config" + _refresh_record_lineage(row) + + with pytest.raises(ValueError, match=rf"{family} config must be 'default'"): + _build_candidate(source) + + +def test_duplicate_stage_a_canonical_ids_fail_after_complete_rehash() -> None: + source = _stage_a_source() + pg19 = [row for row in source["records"] if row["family"] == "pg19"] + pg19[1]["canonical_id"] = pg19[0]["canonical_id"] + pg19[1]["selection_sha256"] = resolver.selection_sha256( + resolver.PG19_VALIDATION_NAMESPACE, + pg19[1]["canonical_id"], + ) + ranked = sorted( + pg19, + key=lambda row: (row["selection_sha256"], row["canonical_id"]), + ) + for rank, row in enumerate(ranked): + row["selection_rank"] = rank + _refresh_record_lineage(row) + + with pytest.raises(ValueError, match="duplicate canonical selection keys"): + _build_candidate(source) + + +def test_sha_rank_order_rejects_duplicate_keys_for_calibration_and_stage_a() -> None: + rows = ( + {"selection_sha256": "a" * 64, "canonical_id": "same", "selection_rank": 0}, + {"selection_sha256": "a" * 64, "canonical_id": "same", "selection_rank": 1}, + ) + for context in ("calibration PG19", "Stage-A PG19"): + with pytest.raises(ValueError, match="duplicate canonical selection keys"): + resolver._validate_sha_rank_order(rows, context=context) + + def test_stage_a_requires_two_continuation_tokens_for_one_cache_prediction() -> None: source = _stage_a_source() row = source["records"][0] @@ -619,6 +1443,18 @@ def test_frozen_stage_a_decoder_reauthenticates_promotion_records_and_binding() decoded.parquet_materialization_manifest_file_sha256 == resolver.PARQUET_MATERIALIZATION_MANIFEST_FILE_SHA256 ) + original_canonical_id = decoded.records[0]["canonical_id"] + original_boundaries = tuple(decoded.records[0]["fisher_boundary"]["boundary_positions"]) + with pytest.raises(TypeError): + decoded.records[0]["canonical_id"] = "HumanEval/99" + with pytest.raises(TypeError, match="immutable"): + decoded.records[0]["fisher_boundary"]["boundary_positions"].append(999) + with pytest.raises(TypeError): + decoded.execution_bindings["repository_source_manifest_file_sha256"] = "0" * 64 + with pytest.raises(AttributeError): + decoded.records = () + assert decoded.records[0]["canonical_id"] == original_canonical_id + assert tuple(decoded.records[0]["fisher_boundary"]["boundary_positions"]) == original_boundaries tampered = copy.deepcopy(frozen) tampered["evidence"]["records"][0]["source_content_sha256"] = "0" * 64 @@ -637,7 +1473,6 @@ def test_frozen_stage_a_decoder_reauthenticates_promotion_records_and_binding() resolver.canonical_json_bytes(tampered), calibration_binding_artifact=FIXTURE_BINDING_ARTIFACT, ) - wrong_binding = dict(FIXTURE_BINDING) wrong_binding["static_k29334_policy_file_sha256"] = "0" * 64 with ( @@ -672,6 +1507,42 @@ def test_frozen_stage_a_decoder_reauthenticates_promotion_records_and_binding() ) +def test_calibration_identity_dto_recursively_freezes_verified_data() -> None: + record = {"nested": {"positions": [1, 2, 3]}} + assignment = {"identity": ["pg19", "http://www.gutenberg.org/ebooks/10000"]} + dto = resolver.FrozenCalibrationIdentityArtifact( + file_sha256="1" * 64, + canonical_evidence_sha256="2" * 64, + records=(record,), + assignment=(assignment,), + assignment_sha256="3" * 64, + tokenizer_manifest_sha256="4" * 64, + parquet_materialization_manifest_file_sha256=( + resolver.PARQUET_MATERIALIZATION_MANIFEST_FILE_SHA256 + ), + execution_bindings=dict(FIXTURE_EXECUTION_BINDINGS), + ) + + record["nested"]["positions"].append(4) + assignment["identity"].append("attacker") + with pytest.raises(TypeError, match="immutable"): + dto.records[0]["nested"]["positions"].append(5) + with pytest.raises(TypeError): + dto.assignment[0]["identity"][0] = "ruler" + with pytest.raises(TypeError): + dto.execution_bindings["repository_source_manifest_file_sha256"] = "0" * 64 + with pytest.raises(AttributeError): + dto.assignment = () + assert dto.records[0]["nested"]["positions"] == [1, 2, 3] + assert dto.assignment[0]["identity"] == [ + "pg19", + "http://www.gutenberg.org/ebooks/10000", + ] + assert json.loads(resolver.canonical_json_bytes(dto.records[0])) == { + "nested": {"positions": [1, 2, 3]} + } + + def test_handcrafted_incomplete_candidate_cannot_be_promoted() -> None: evidence = { "identity_schema": resolver.CANDIDATE_SCHEMA, diff --git a/tests/test_run_static_q468_calibration.py b/tests/test_run_static_q468_calibration.py index e217735..102d527 100644 --- a/tests/test_run_static_q468_calibration.py +++ b/tests/test_run_static_q468_calibration.py @@ -24,6 +24,17 @@ api = importlib.util.module_from_spec(API_SPEC) sys.modules[API_SPEC.name] = api API_SPEC.loader.exec_module(api) +RESOLVER_SCRIPT = ( + Path(__file__).resolve().parents[1] / "scripts" / "resolve_static_q468_identity.py" +) +RESOLVER_SPEC = importlib.util.spec_from_file_location( + "resolve_static_q468_identity_for_runner_tests", + RESOLVER_SCRIPT, +) +assert RESOLVER_SPEC is not None and RESOLVER_SPEC.loader is not None +identity_resolver = importlib.util.module_from_spec(RESOLVER_SPEC) +sys.modules[RESOLVER_SPEC.name] = identity_resolver +RESOLVER_SPEC.loader.exec_module(identity_resolver) SPEC = importlib.util.spec_from_file_location("run_static_q468_calibration", SCRIPT) assert SPEC is not None and SPEC.loader is not None runner = importlib.util.module_from_spec(SPEC) @@ -50,6 +61,16 @@ def current_head() -> str: return process.stdout.strip() +def authenticated_git_path() -> Path: + return runner._authenticate_git_executable(None).path + + +def fisher_boundary_contract( + token_ids: tuple[int, ...] = (1, 2, 3), +) -> dict[str, object]: + return identity_resolver.build_fisher_boundary_contract(token_ids) + + def record(token_ids: tuple[int, ...] = (1, 2, 3)) -> dict[str, object]: prompt_stop = max(1, len(token_ids) - 1) return { @@ -57,6 +78,7 @@ def record(token_ids: tuple[int, ...] = (1, 2, 3)) -> dict[str, object]: "config": "default", "family": "mbpp", "formatted_content_sha256": "b" * 64, + "fisher_boundary": fisher_boundary_contract(token_ids), "generator_receipt_sha256": None, "prompt_token_ids_sha256": token_digest(token_ids[:prompt_stop]), "ruler_category": None, @@ -132,7 +154,7 @@ def bootstrap_identity_bytes( "identity_only": True, "phase": "calibration", "promotion_required": False, - "schema_version": 4, + "schema_version": runner.FROZEN_IDENTITY_SCHEMA_VERSION, "status": "frozen", } return runner.canonical_json_bytes( @@ -184,7 +206,31 @@ def model_manifest_bytes(files: Mapping[str, bytes]) -> bytes: return runner.canonical_json_bytes(payload) +def staged_model_files() -> dict[str, bytes]: + return { + "config.json": b"{}", + "model.safetensors": b"authenticated-weight-placeholder", + "model.safetensors.index.json": b"{}", + } + + +def model_staging_authorization( + files: Mapping[str, bytes] | None = None, +) -> Any: + payload = model_manifest_bytes(files or staged_model_files()) + manifest = runner.parse_model_file_manifest(payload) + frozen = identity((), model_manifest_sha256=digest(payload)) + return runner.ModelStagingAuthorization( + identity=frozen, + model_manifest=manifest, + frozen_identity_file_sha256="d" * 64, + identity_commit="3" * 40, + source_commit="1" * 40, + ) + + def runtime_manifest_bytes() -> bytes: + git_executable = runner._authenticate_git_executable(None) interpreter_sha256, interpreter_size = runner._stream_file_sha256( Path(sys.executable).resolve(strict=True) ) @@ -231,6 +277,11 @@ def runtime_manifest_bytes() -> bytes: "sha256": interpreter_sha256, "size_bytes": interpreter_size, }, + "git_executable": { + "absolute_path_sha256": git_executable.absolute_path_sha256, + "sha256": git_executable.sha256, + "size_bytes": git_executable.size_bytes, + }, "launch_policy": dict(runner.SEALED_LAUNCH_POLICY), "machine": dict( zip( @@ -289,7 +340,24 @@ def test_bootstrap_identity_rejects_float_schema_version() -> None: parquet_manifest_sha256="a" * 64, ) document = json.loads(data) - document["evidence"]["schema_version"] = 4.0 + document["evidence"]["schema_version"] = 5.0 + document["canonical_evidence_sha256"] = digest( + runner.canonical_json_bytes(document["evidence"]) + ) + + with pytest.raises(runner.CalibrationRunError, match="state"): + runner._bootstrap_identity_bindings(runner.canonical_json_bytes(document)) + + +def test_bootstrap_identity_rejects_obsolete_schema_v4() -> None: + data = bootstrap_identity_bytes( + source_manifest_sha256="7" * 64, + runtime_manifest_sha256="8" * 64, + model_manifest_sha256="9" * 64, + parquet_manifest_sha256="a" * 64, + ) + document = json.loads(data) + document["evidence"]["schema_version"] = 4 document["canonical_evidence_sha256"] = digest( runner.canonical_json_bytes(document["evidence"]) ) @@ -331,11 +399,13 @@ def __init__( *, stability_passed: bool = True, decode_error: BaseException | None = None, + expected_source_commit: str | None = None, ) -> None: self.frozen_identity = frozen_identity self.events = events self.stability_passed = stability_passed self.decode_error = decode_error + self.expected_source_commit = expected_source_commit or current_head() self.captured: list[Any] = [] def decode_identity(self, data: bytes) -> Any: @@ -364,7 +434,7 @@ def finalize( ) -> Any: self.events.append("finalize") assert len(scores) == len(identity.records) - assert source_commit == current_head() + assert source_commit == self.expected_source_commit stability = { "checks": [{"name": "fake", "passed": self.stability_passed}], "passed": self.stability_passed, @@ -377,14 +447,15 @@ def finalize( ) artifacts = runner.CalibrationArtifacts( score=b"score", + comparator_score=b"comparator-score", split_half=b"split", static_k27030=b"k27030", static_k29334=b"k29334", + static_mse_k29334=b"mse-k29334", + static_fisher_k29334=b"fisher-k29334", static_q48=b"q48", stage_a_binding=b"binding", stability=stability, - calibration_scores_sha256="3" * 64, - sequence_score_manifest_sha256="4" * 64, ) return runner.FinalizationResult( passed=True, @@ -406,6 +477,8 @@ def __init__( self.invalid_kernel_receipt = invalid_kernel_receipt self.closed = False self.capture_flags: list[bool] = [] + self.fisher_calls: list[tuple[int, int, int]] = [] + self.geometry = FakeBackend.geometry def materialize_sequence(self, item: Mapping[str, object]) -> Any: self.events.append("materialize_sequence") @@ -431,7 +504,7 @@ def step_token( del model self.events.append("step_token") self.capture_flags.append(capture_state) - geometry = FakeBackend.geometry + geometry = self.geometry return api.StepObservation( position=position, token_id=token_id, @@ -456,6 +529,42 @@ def step_token( ), ) + def step_token_with_fisher( + self, + model: object, + *, + token_id: int, + position: int, + target_token_id: int, + capture_state: bool, + ) -> object: + self.events.append("step_token_with_fisher") + self.fisher_calls.append((position, token_id, target_token_id)) + observation = self.step_token( + model, + token_id=token_id, + position=position, + capture_state=capture_state, + ) + geometry = self.geometry + source = torch.ones( + geometry.layers, + geometry.heads, + geometry.key_rows, + geometry.value_width, + ) + return api.FisherStepObservation( + boundary_position=position - 1, + input_position=position, + target_position=position + 1, + input_token_id=token_id, + target_token_id=target_token_id, + step_observation=observation, + source_recurrent_state=source, + source_state_gradient=torch.full_like(source, 0.25), + target_nll=1.0, + ) + def end_sequence(self, model: object, item: Mapping[str, object]) -> None: del model, item self.events.append("end_sequence") @@ -468,6 +577,7 @@ def close_model(self, model: object) -> None: def runtime_metadata(self) -> Mapping[str, object]: self.events.append("runtime_metadata") return { + "fisher_step_count": len(self.fisher_calls), "model_open": not self.closed, "name": "fake", "one_token_calls": self.events.count("step_token"), @@ -485,18 +595,33 @@ def fake_distortions(state: torch.Tensor, geometry: Any) -> Any: return tuple(torch.full(shape, value, dtype=torch.float64) for value in (3.0, 2.0, 1.0)) -def source_verifier(events: list[str], *, fail_on_call: int | None = None) -> Any: +def fake_fisher_distortions( + state: torch.Tensor, + gradient: torch.Tensor, + geometry: Any, +) -> Any: + assert state.shape == gradient.shape + return fake_distortions(state, geometry) + + +def source_verifier( + events: list[str], + *, + fail_on_call: int | None = None, + source_commit: str | None = None, +) -> Any: calls = 0 + expected_source_commit = source_commit or current_head() def verify(expected: Mapping[str, object], root: Path) -> Any: nonlocal calls calls += 1 events.append("verify_source") - assert expected == {"manifest": "expected", "source_commit": current_head()} + assert expected == {"manifest": "expected", "source_commit": expected_source_commit} assert root == SCRIPT.parents[1] if fail_on_call == calls: raise runner.CalibrationRunError("source drift") - return {"manifest": "expected", "source_commit": current_head()}, "5" * 64 + return {"manifest": "expected", "source_commit": expected_source_commit}, "5" * 64 return verify @@ -508,11 +633,17 @@ def configured_run( stability_passed: bool = True, source_fail_on_call: int | None = None, decode_error: BaseException | None = None, + source_commit: str | None = None, ) -> tuple[Any, Any, Any, Any]: selected_records = list(records or [record()]) - files = {"config.json": b"{}", "model.safetensors": b"safe-test-placeholder"} + expected_source_commit = source_commit or current_head() + files = { + "config.json": b"{}", + "model.safetensors": b"safe-test-placeholder", + "model.safetensors.index.json": b"{}", + } source_bytes = runner.canonical_json_bytes( - {"manifest": "expected", "source_commit": current_head()} + {"manifest": "expected", "source_commit": expected_source_commit} ) model_bytes = model_manifest_bytes(files) runtime_bytes = runtime_manifest_bytes() @@ -532,6 +663,7 @@ def configured_run( events, stability_passed=stability_passed, decode_error=decode_error, + expected_source_commit=expected_source_commit, ) sequence_map = { str(item["canonical_id"]): materialized( @@ -554,6 +686,9 @@ def authenticate_runtime(manifest: Any) -> Any: python_version=manifest.python_version, python_cache_tag=manifest.python_cache_tag, interpreter_sha256=manifest.interpreter_sha256, + git_executable_absolute_path_sha256=(manifest.git_executable_absolute_path_sha256), + git_executable_sha256=manifest.git_executable_sha256, + git_executable_size_bytes=manifest.git_executable_size_bytes, machine_name=manifest.machine_name, base_runtime_file_count=len(manifest.runtime_trees[0].files), package_root_count=len(manifest.package_roots), @@ -565,9 +700,15 @@ def authenticate_runtime(manifest: Any) -> Any: services = runner.RunnerServices( backend=backend, calibration_api=api, - verify_repository_source=source_verifier(events, fail_on_call=source_fail_on_call), + identity_resolver=identity_resolver, + verify_repository_source=source_verifier( + events, + fail_on_call=source_fail_on_call, + source_commit=expected_source_commit, + ), validate_adapter=lambda _adapter: events.append("validate_adapter"), distortion_function=fake_distortions, + fisher_distortion_function=fake_fisher_distortions, authenticate_model_files=authenticate, authenticate_runtime=authenticate_runtime, ) @@ -579,13 +720,65 @@ def authenticate_runtime(manifest: Any) -> Any: runtime_manifest_bytes=runtime_bytes, model_root=model_root, repository_root=SCRIPT.parents[1], - expected_source_commit=current_head(), + expected_source_commit=expected_source_commit, expected_model_file_manifest_sha256=digest(model_bytes), expected_parquet_materialization_manifest_sha256=digest(parquet_bytes), expected_runtime_manifest_sha256=digest(runtime_bytes), output_dir=tmp_path / "output", require_cuda=False, ) + parsed_runtime = runner.parse_calibration_runtime_manifest(runtime_bytes) + authenticated_runtime = authenticate_runtime(parsed_runtime) + events.clear() + parsed_model = runner.parse_model_file_manifest(model_bytes) + authenticated_model = runner.authenticate_local_model_files( + model_root, + parsed_model, + calibration_api=api, + ) + first_record = frozen.records[0] + first_token_count = int(first_record["sequence_length"]) + first_fisher_count = len(first_record["fisher_boundary"]["boundary_positions"]) + smoke_report = runner._report_bytes( + status="fisher_h1_smoke_passed", + identity=frozen, + source_commit=expected_source_commit, + source_manifest_sha256="5" * 64, + source_manifest_file_sha256=digest(source_bytes), + model_files=authenticated_model, + sequence_count=1, + token_count=first_token_count, + post_token_anchor_count=len(runner.frozen_anchor_positions(first_token_count)), + fisher_boundary_count=first_fisher_count, + observed_fisher_step_count=first_fisher_count, + stability={ + "checks": [], + "evaluated": False, + "passed": None, + "scope": "smoke_only", + }, + artifacts={}, + runtime={ + "adapter": {"fisher_step_count": first_fisher_count}, + "authenticated_distribution_count": authenticated_runtime.distribution_count, + "authenticated_file_count": authenticated_runtime.file_count, + "cuda_available": True, + "cuda_runtime": "test", + "elapsed_seconds_hex": (0.0).hex(), + "gpu": {"name": "test-gpu"}, + "packages": dict(authenticated_runtime.distributions), + "platform": "test", + "python": "test", + "runtime_manifest_file_sha256": authenticated_runtime.manifest_file_sha256, + "torch": "test", + }, + fisher_h1_smoke_report_file_sha256=None, + ) + config = replace( + config, + prior_fisher_h1_smoke_report_bytes=smoke_report, + prior_fisher_h1_smoke_complete_bytes=runner.FISHER_SMOKE_COMPLETE_BYTES, + ) return config, adapter, services, events @@ -597,9 +790,12 @@ def test_success_authenticates_every_boundary_and_publishes_complete_set(tmp_pat assert result["status"] == "passed" assert set(path.name for path in config.output_dir.iterdir()) == { runner.SCORE_FILENAME, + runner.COMPARATOR_SCORE_FILENAME, runner.SPLIT_FILENAME, runner.K27030_FILENAME, runner.K29334_FILENAME, + runner.MSE_K29334_FILENAME, + runner.FISHER_K29334_FILENAME, runner.Q48_FILENAME, runner.BINDING_FILENAME, runner.REPORT_FILENAME, @@ -625,7 +821,10 @@ def test_success_authenticates_every_boundary_and_publishes_complete_set(tmp_pat report = json.loads((config.output_dir / runner.REPORT_FILENAME).read_text()) assert report["evidence"]["status"] == "passed" assert report["evidence"]["calibration"] == { - "anchor_count": 3, + "expected_fisher_step_count": 1, + "observed_fisher_step_count": 1, + "fisher_boundary_count": 1, + "post_token_anchor_count": 3, "sequence_count": 1, "token_count": 3, } @@ -638,9 +837,106 @@ def test_success_authenticates_every_boundary_and_publishes_complete_set(tmp_pat ), "repository_source_manifest_file_sha256": digest(config.repository_source_manifest_bytes), } + assert report["evidence"]["prerequisites"] == { + "fisher_h1_smoke_report_file_sha256": digest(config.prior_fisher_h1_smoke_report_bytes) + } -def test_identity_view_consumes_strict_schema_v4_execution_bindings( +def test_fisher_h1_smoke_runs_first_frozen_sequence_and_publishes_only_receipt( + tmp_path: Path, +) -> None: + first = record() + second = {**record(), "canonical_id": "item-2"} + config, adapter, services, events = configured_run( + tmp_path, + records=[first, second], + ) + config = replace( + config, + fisher_h1_smoke=True, + prior_fisher_h1_smoke_report_bytes=None, + prior_fisher_h1_smoke_complete_bytes=None, + ) + + result = runner.run_calibration(config, adapter, services=services) + + assert result["status"] == "fisher_h1_smoke_passed" + assert result["sequence_count"] == 1 + assert result["token_count"] == 3 + assert result["fisher_boundary_count"] == 1 + assert events.count("materialize_sequence") == 2 + assert events.count("reduce_sequence") == 1 + assert "finalize" not in events + assert adapter.closed + assert {path.name for path in config.output_dir.iterdir()} == { + runner.FISHER_SMOKE_REPORT_FILENAME, + runner.FISHER_SMOKE_COMPLETE_FILENAME, + } + report = json.loads((config.output_dir / runner.FISHER_SMOKE_REPORT_FILENAME).read_text()) + assert report["evidence"]["status"] == "fisher_h1_smoke_passed" + assert report["evidence"]["calibration"] == { + "expected_fisher_step_count": 1, + "observed_fisher_step_count": 1, + "fisher_boundary_count": 1, + "post_token_anchor_count": 3, + "sequence_count": 1, + "token_count": 3, + } + assert report["evidence"]["artifacts"] == {} + assert report["evidence"]["prerequisites"] == {"fisher_h1_smoke_report_file_sha256": None} + + +def test_full_calibration_requires_authenticated_prior_fisher_smoke_before_data( + tmp_path: Path, +) -> None: + config, adapter, services, events = configured_run(tmp_path) + config = replace( + config, + prior_fisher_h1_smoke_report_bytes=None, + prior_fisher_h1_smoke_complete_bytes=None, + ) + + with pytest.raises(runner.CalibrationRunError, match="requires the prior Fisher H=1"): + runner.run_calibration(config, adapter, services=services) + + assert events == ["decode_identity"] + assert not config.output_dir.exists() + + +def test_full_calibration_rejects_rehashed_smoke_from_another_identity_before_data( + tmp_path: Path, +) -> None: + config, adapter, services, events = configured_run(tmp_path) + document = json.loads(config.prior_fisher_h1_smoke_report_bytes) + document["evidence"]["identity"]["file_sha256"] = "0" * 64 + document["canonical_evidence_sha256"] = digest( + runner.canonical_json_bytes(document["evidence"]) + ) + config = replace( + config, + prior_fisher_h1_smoke_report_bytes=runner.canonical_json_bytes(document), + ) + + with pytest.raises(runner.CalibrationRunError, match="smoke identity receipt"): + runner.run_calibration(config, adapter, services=services) + + assert "materialize_sequence" not in events + assert "authenticate_model_files" not in events + assert "load_model" not in events + + +def test_authenticated_unchanged_descendant_retains_h0_provenance(tmp_path: Path) -> None: + h0 = "1" * 40 + config, adapter, services, _events = configured_run(tmp_path, source_commit=h0) + + runner.run_calibration(config, adapter, services=services) + + report = json.loads((config.output_dir / runner.REPORT_FILENAME).read_text()) + assert report["evidence"]["repository"]["source_commit"] == h0 + assert report["evidence"]["repository"]["source_commit"] != current_head() + + +def test_identity_view_consumes_schema_v5_bindings_and_preserves_fisher_boundary( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -650,9 +946,17 @@ def test_identity_view_consumes_strict_schema_v4_execution_bindings( "parquet_materialization_manifest_file_sha256": "9" * 64, "repository_source_manifest_file_sha256": "3" * 64, } + boundary = fisher_boundary_contract() + decoded_record = { + "canonical_id": "item-1", + "fisher_boundary": boundary, + "sequence_length": 3, + } evidence = { "execution_bindings": bindings, "model_contracts": {"primary": {"id": "example/model", "revision": "4" * 40}}, + "records": [decoded_record], + "schema_version": runner.FROZEN_IDENTITY_SCHEMA_VERSION, "source_manifest_sha256": "5" * 64, "tokenizer": {"transformers_version": "5.14.1"}, } @@ -661,7 +965,7 @@ def test_identity_view_consumes_strict_schema_v4_execution_bindings( class Decoded: file_sha256 = digest(payload) canonical_evidence_sha256 = "6" * 64 - records = () + records = (decoded_record,) assignment = () assignment_sha256 = "7" * 64 tokenizer_manifest_sha256 = "8" * 64 @@ -680,6 +984,49 @@ def deserialize_frozen_calibration_identity_artifact(data: bytes) -> Decoded: assert decoded.runtime_manifest_file_sha256 == "1" * 64 assert decoded.model_file_manifest_file_sha256 == "2" * 64 assert decoded.parquet_materialization_manifest_file_sha256 == "9" * 64 + assert decoded.records[0]["fisher_boundary"] == boundary + assert decoded.records[0]["fisher_boundary"] is not boundary + + +def test_identity_view_rejects_resolver_record_that_drops_fisher_boundary() -> None: + bindings = { + "calibration_runtime_manifest_file_sha256": "1" * 64, + "model_file_manifest_file_sha256": "2" * 64, + "parquet_materialization_manifest_file_sha256": "9" * 64, + "repository_source_manifest_file_sha256": "3" * 64, + } + evidence_record = { + "canonical_id": "item-1", + "fisher_boundary": fisher_boundary_contract(), + "sequence_length": 3, + } + evidence = { + "execution_bindings": bindings, + "model_contracts": {"primary": {"id": "example/model", "revision": "4" * 40}}, + "records": [evidence_record], + "schema_version": runner.FROZEN_IDENTITY_SCHEMA_VERSION, + "source_manifest_sha256": "5" * 64, + "tokenizer": {"transformers_version": "5.14.1"}, + } + payload = runner.canonical_json_bytes({"evidence": evidence}) + + class Decoded: + file_sha256 = digest(payload) + canonical_evidence_sha256 = "6" * 64 + records = ({"canonical_id": "item-1"},) + assignment = () + assignment_sha256 = "7" * 64 + tokenizer_manifest_sha256 = "8" * 64 + execution_bindings = bindings + + class Resolver: + @staticmethod + def deserialize_frozen_calibration_identity_artifact(data: bytes) -> Decoded: + assert data == payload + return Decoded() + + with pytest.raises(runner.CalibrationRunError, match="fisher_boundary"): + runner._identity_view_from_resolver(payload, Resolver()) def test_failed_stability_publishes_only_report_and_never_binding(tmp_path: Path) -> None: @@ -734,7 +1081,7 @@ def test_source_drift_stops_before_dataset_and_model_access(tmp_path: Path) -> N assert not config.output_dir.exists() -def test_source_manifest_commit_must_equal_reported_head(tmp_path: Path) -> None: +def test_source_manifest_commit_must_equal_requested_frozen_commit(tmp_path: Path) -> None: config, adapter, services, events = configured_run(tmp_path) def wrong_commit(expected: Mapping[str, object], root: Path) -> Any: @@ -772,6 +1119,29 @@ def test_materialized_token_mismatch_stops_before_model_file_open_or_load(tmp_pa assert "load_model" not in events +def test_materialized_fisher_token_hash_mismatch_reaches_exact_boundary_check( + tmp_path: Path, +) -> None: + item = record() + boundary = dict(item["fisher_boundary"]) + boundary["input_token_ids_sha256"] = "0" * 64 + boundary_without_hash = { + name: value for name, value in boundary.items() if name != "fisher_boundary_sha256" + } + boundary["fisher_boundary_sha256"] = digest( + identity_resolver.FISHER_BOUNDARY_NAMESPACE.encode("utf-8") + + runner.canonical_json_bytes(boundary_without_hash) + ) + item["fisher_boundary"] = boundary + config, adapter, services, events = configured_run(tmp_path, records=[item]) + + with pytest.raises(runner.CalibrationRunError, match="Fisher input/target tokens"): + runner.run_calibration(config, adapter, services=services) + + assert "authenticate_model_files" not in events + assert "load_model" not in events + + def test_model_manifest_commitment_is_checked_before_data_access(tmp_path: Path) -> None: config, adapter, services, events = configured_run(tmp_path) config = replace(config, expected_model_file_manifest_sha256="9" * 64) @@ -821,11 +1191,7 @@ def drifting(manifest: Any) -> Any: nonlocal calls calls += 1 authenticated = original(manifest) - return ( - replace(authenticated, machine_name="drifted") - if calls == 2 - else authenticated - ) + return replace(authenticated, machine_name="drifted") if calls == 2 else authenticated services = replace(services, authenticate_runtime=drifting) with pytest.raises(runner.CalibrationRunError, match="before data access"): @@ -846,7 +1212,15 @@ def test_empty_calibration_target_is_hash_checked_and_allowed() -> None: item["target_token_ids_sha256"] = token_digest(()) candidate = materialized(item, token_ids) - assert runner.validate_materialized_sequence(item, candidate, calibration_api=api) == token_ids + assert ( + runner.validate_materialized_sequence( + item, + candidate, + calibration_api=api, + identity_resolver=identity_resolver, + ) + == token_ids + ) def test_exact_local_model_file_mismatch_stops_before_model_load(tmp_path: Path) -> None: @@ -882,7 +1256,11 @@ def test_model_authentication_rejects_reparse_or_symlink_entries( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - files = {"config.json": b"{}", "model.safetensors": b"weights"} + files = { + "config.json": b"{}", + "model.safetensors": b"weights", + "model.safetensors.index.json": b"{}", + } root = tmp_path / "model" write_model_root(root, files) manifest = runner.parse_model_file_manifest(model_manifest_bytes(files)) @@ -948,6 +1326,7 @@ def step_token( ) adapter = QueryAdapter({}, events) + adapter.geometry = geometry captured = runner.capture_sequence_causally( adapter, object(), @@ -957,9 +1336,13 @@ def step_token( calibration_api=api, require_cuda=False, distortion_function=fake_distortions, + fisher_distortion_function=fake_fisher_distortions, ) assert captured.anchor_positions == runner.frozen_anchor_positions(17) + assert captured.fisher_boundary_positions == runner.frozen_anchor_positions(15) + assert len(adapter.capture_flags) == len(tokens) + assert events.count("step_token_with_fisher") == len(captured.fisher_boundary_positions) assert adapter.capture_flags[15] is False assert sum(adapter.capture_flags) == 16 first_energy = torch.tensor( @@ -971,11 +1354,54 @@ def step_token( torch.testing.assert_close(captured.query_energy[0, 0, 0], expected, rtol=1e-6, atol=1e-8) assert captured.query_energy.dtype == torch.float64 assert captured.q4_mse.shape == (16, 1, 1, 4) + assert captured.fisher_q4_risk.shape == (15, 1, 1, 4) + assert captured.fisher_target_nlls.shape == (15,) + assert captured.fisher_target_nlls.dtype == torch.float64 + + +def test_three_token_capture_uses_one_exact_h1_fisher_step_and_three_forwards() -> None: + tokens = (10, 11, 12) + events: list[str] = [] + + class CountingModel: + def __init__(self) -> None: + self.forward_count = 0 + + def forward(self) -> None: + self.forward_count += 1 + + class CountingAdapter(FakeAdapter): + def step_token(self, model: object, **kwargs: object) -> Any: + assert isinstance(model, CountingModel) + model.forward() + return super().step_token(model, **kwargs) + + adapter = CountingAdapter({}, events) + model = CountingModel() + + captured = runner.capture_sequence_causally( + adapter, + model, + record(tokens), + tokens, + geometry=FakeBackend.geometry, + calibration_api=api, + require_cuda=False, + distortion_function=fake_distortions, + fisher_distortion_function=fake_fisher_distortions, + ) + + assert captured.fisher_boundary_positions == (0,) + assert events.count("step_token") == len(tokens) + assert events.count("step_token_with_fisher") == 1 + assert adapter.fisher_calls == [(1, 11, 12)] + assert len(adapter.capture_flags) == len(tokens) + assert model.forward_count == len(tokens) def test_official_capture_rejects_cpu_queries_even_when_cuda_might_exist() -> None: geometry = FakeBackend.geometry - tokens = (1,) + tokens = (1, 2, 3) adapter = FakeAdapter({}, []) with pytest.raises(runner.CalibrationRunError, match="actual CUDA"): @@ -988,6 +1414,7 @@ def test_official_capture_rejects_cpu_queries_even_when_cuda_might_exist() -> No calibration_api=api, require_cuda=True, distortion_function=fake_distortions, + fisher_distortion_function=fake_fisher_distortions, ) @@ -1006,16 +1433,19 @@ def step_token(self, model: object, **kwargs: object) -> Any: successful_kernel_calls_per_layer=(1,), ) + adapter = Bf16Adapter({}, []) + adapter.geometry = geometry with pytest.raises(runner.CalibrationRunError, match="must be FP32"): runner.capture_sequence_causally( - Bf16Adapter({}, []), + adapter, object(), - record((1,)), - (1,), + record((1, 2, 3)), + (1, 2, 3), geometry=geometry, calibration_api=api, require_cuda=False, distortion_function=fake_distortions, + fisher_distortion_function=fake_fisher_distortions, ) @@ -1037,12 +1467,59 @@ def test_compute_anchor_distortions_is_cpu_fp64_and_precision_ordered() -> None: assert torch.all(d8 <= d4) +def test_compute_fisher_distortions_delegates_exact_cpu_fp64_endpoint_math() -> None: + from recurquant.static_q468 import StaticRhtQ468Geometry + from recurquant.static_q468_calibration import ( + compute_rht_diagonal_empirical_fisher_h1_endpoints, + ) + + geometry = runner.Geometry(layer_indices=(3,), heads=1, key_rows=2, value_width=4) + source = torch.tensor( + [[[[1.2, -0.1, 0.4, 2.3], [0.2, 0.7, -1.4, 0.5]]]], + dtype=torch.float32, + ) + gradient = torch.tensor( + [[[[0.3, -0.2, 0.1, 0.4], [0.5, 0.1, -0.3, 0.2]]]], + dtype=torch.float32, + ) + endpoint_geometry = StaticRhtQ468Geometry( + layer_indices=geometry.layer_indices, + heads=geometry.heads, + key_rows=geometry.key_rows, + value_width=geometry.value_width, + target_resident_bytes=1, + ) + + actual = runner.compute_fisher_distortions(source, gradient, geometry) + expected = compute_rht_diagonal_empirical_fisher_h1_endpoints( + source, + gradient, + geometry=endpoint_geometry, + ) + + for actual_tensor, expected_tensor in zip(actual, expected, strict=True): + assert actual_tensor.device.type == "cpu" + assert actual_tensor.dtype == torch.float64 + torch.testing.assert_close(actual_tensor, expected_tensor, rtol=0, atol=0) + + with pytest.raises(runner.CalibrationRunError, match="source state/gradient"): + runner.compute_fisher_distortions(source, gradient[..., :3], geometry) + + def test_model_manifest_rejects_noncanonical_duplicate_and_traversal() -> None: - files = {"config.json": b"{}", "model.safetensors": b"weights"} + files = { + "config.json": b"{}", + "model.safetensors": b"weights", + "model.safetensors.index.json": b"{}", + } valid = model_manifest_bytes(files) parsed = runner.parse_model_file_manifest(valid) assert parsed.file_sha256 == digest(valid) - assert [item.name for item in parsed.files] == ["config.json", "model.safetensors"] + assert [item.name for item in parsed.files] == [ + "config.json", + "model.safetensors", + "model.safetensors.index.json", + ] with pytest.raises(ValueError, match="canonical"): runner.parse_model_file_manifest(valid.rstrip(b"\n") + b" \n") @@ -1057,6 +1534,25 @@ def test_model_manifest_rejects_noncanonical_duplicate_and_traversal() -> None: runner.parse_model_file_manifest(runner.canonical_json_bytes(payload)) +def test_model_manifest_enforces_exact_selection_profile_and_portable_names() -> None: + required = { + "config.json": b"{}", + "model.safetensors": b"weights", + "model.safetensors.index.json": b"{}", + } + + with pytest.raises(ValueError, match="case-insensitive collision"): + runner.parse_model_file_manifest( + model_manifest_bytes({**required, "CONFIG.JSON": b"other"}) + ) + with pytest.raises(ValueError, match="selection profile"): + runner.parse_model_file_manifest(model_manifest_bytes({**required, "unbound.json": b"x"})) + with pytest.raises(ValueError, match="index"): + runner.parse_model_file_manifest( + model_manifest_bytes({"config.json": b"{}", "model.safetensors": b"weights"}) + ) + + def hub_tree_entries() -> list[dict[str, object]]: return [ { @@ -1074,6 +1570,12 @@ def hub_tree_entries() -> list[dict[str, object]]: { "blob_id": "3" * 40, "lfs": None, + "path": "model.safetensors.index.json", + "size": 321, + }, + { + "blob_id": "4" * 40, + "lfs": None, "path": "README.md", "size": 50, }, @@ -1091,7 +1593,11 @@ def test_model_manifest_capture_uses_only_pinned_hub_tree_lfs_metadata() -> None ) parsed = runner.parse_model_file_manifest(payload) - assert [item.name for item in parsed.files] == ["config.json", "model.safetensors"] + assert [item.name for item in parsed.files] == [ + "config.json", + "model.safetensors", + "model.safetensors.index.json", + ] assert parsed.files[0].sha256 is None assert parsed.files[0].git_blob_oid == "1" * 40 assert parsed.files[1].sha256 == "a" * 64 @@ -1145,13 +1651,15 @@ class Info: class Api: @staticmethod def model_info(*args: object, **kwargs: object) -> Info: - del args, kwargs + del args + assert kwargs["token"] is False calls.append("model_info") return Info() @staticmethod def list_repo_tree(*args: object, **kwargs: object) -> list[dict[str, object]]: - del args, kwargs + del args + assert kwargs["token"] is False calls.append("list_repo_tree") return hub_tree_entries() @@ -1221,7 +1729,11 @@ def test_model_manifest_parser_detects_tree_metadata_tamper() -> None: def test_model_authentication_rejects_extra_files(tmp_path: Path) -> None: - files = {"config.json": b"{}", "model.safetensors": b"weights"} + files = { + "config.json": b"{}", + "model.safetensors": b"weights", + "model.safetensors.index.json": b"{}", + } root = tmp_path / "model" write_model_root(root, {**files, "unbound.txt": b"extra"}) manifest = runner.parse_model_file_manifest(model_manifest_bytes(files)) @@ -1230,6 +1742,343 @@ def test_model_authentication_rejects_extra_files(tmp_path: Path) -> None: runner.authenticate_local_model_files(root, manifest, calibration_api=api) +def test_stage_model_authenticates_before_touching_downloader_cache_or_output( + tmp_path: Path, +) -> None: + candidate = runner.canonical_json_bytes( + { + "canonical_evidence_sha256": "0" * 64, + "evidence": { + "identity_only": True, + "phase": "calibration", + "promotion_required": True, + "schema_version": runner.FROZEN_IDENTITY_SCHEMA_VERSION, + "status": "candidate", + }, + } + ) + identity_path = tmp_path / "candidate.json" + identity_path.write_bytes(candidate) + cache = tmp_path / "cache" + output = tmp_path / "model" + calls: list[dict[str, object]] = [] + + with pytest.raises(runner.CalibrationRunError, match="state"): + runner.stage_identity_bound_model( + frozen_identity_path=identity_path, + expected_frozen_identity_sha256=digest(candidate), + identity_commit="3" * 40, + repository_root=SCRIPT.parents[1], + repository_source_manifest_path=tmp_path / "missing-source.json", + source_commit="1" * 40, + model_file_manifest_path=tmp_path / "missing-model.json", + expected_model_file_manifest_sha256="2" * 64, + hub_cache_root=cache, + output_root=output, + downloader=lambda **kwargs: calls.append(kwargs), + ) + + assert calls == [] + assert not cache.exists() + assert not output.exists() + + +def test_committed_frozen_identity_requires_exact_head_index_and_worktree_blob( + tmp_path: Path, +) -> None: + repository = tmp_path / "repository" + repository.mkdir() + subprocess.run(["git", "init", "-b", "main"], cwd=repository, check=True, capture_output=True) + subprocess.run( + ["git", "config", "user.email", "test@example.invalid"], + cwd=repository, + check=True, + ) + subprocess.run(["git", "config", "user.name", "Test"], cwd=repository, check=True) + identity_path = repository / "evidence" / "identity.json" + identity_path.parent.mkdir() + identity_bytes = b'{"status":"frozen"}\n' + identity_path.write_bytes(identity_bytes) + subprocess.run(["git", "add", "--", "evidence/identity.json"], cwd=repository, check=True) + subprocess.run( + ["git", "-c", "commit.gpgsign=false", "commit", "-m", "freeze identity"], + cwd=repository, + check=True, + ) + head = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=repository, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + assert ( + runner._verify_committed_frozen_identity( + runner._authenticate_git_executable(authenticated_git_path()), + repository, + identity_path, + identity_bytes, + identity_commit=head, + ) + == head + ) + identity_path.write_bytes(b'{"status":"changed"}\n') + with pytest.raises(runner.CalibrationRunError, match="changed|differ"): + runner._verify_committed_frozen_identity( + runner._authenticate_git_executable(authenticated_git_path()), + repository, + identity_path, + identity_bytes, + identity_commit=head, + ) + + +def test_stage_model_downloads_only_exact_bound_files_and_publishes_atomically( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + files = staged_model_files() + authorization = model_staging_authorization(files) + authorizations: list[str] = [] + + def authenticate(**kwargs: object) -> Any: + del kwargs + authorizations.append("authenticated") + return authorization + + calls: list[dict[str, object]] = [] + + def download(**kwargs: object) -> str: + calls.append(dict(kwargs)) + name = str(kwargs["filename"]) + cache = Path(kwargs["cache_dir"]) + target = cache / "models--example--model" / "snapshots" / ("2" * 40) / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(files[name]) + return str(target) + + monkeypatch.setattr(runner, "_authenticate_model_staging_authorization", authenticate) + cache = tmp_path / "hub-cache" + output = tmp_path / "published-model" + result = runner.stage_identity_bound_model( + frozen_identity_path=tmp_path / "identity.json", + expected_frozen_identity_sha256="d" * 64, + identity_commit="3" * 40, + repository_root=SCRIPT.parents[1], + repository_source_manifest_path=tmp_path / "source.json", + source_commit="1" * 40, + model_file_manifest_path=tmp_path / "model-manifest.json", + expected_model_file_manifest_sha256=authorization.model_manifest.file_sha256, + hub_cache_root=cache, + output_root=output, + local_files_only=True, + downloader=download, + ) + + assert authorizations == ["authenticated", "authenticated"] + assert [call["filename"] for call in calls] == sorted(files) + assert all( + call + == { + "cache_dir": cache.resolve(), + "endpoint": "https://huggingface.co", + "filename": call["filename"], + "local_files_only": True, + "repo_id": "example/model", + "repo_type": "model", + "revision": "2" * 40, + "token": False, + } + for call in calls + ) + assert {path.name for path in output.iterdir()} == set(files) + assert all((output / name).read_bytes() == content for name, content in files.items()) + assert not list(tmp_path.glob(".published-model.staging-*")) + assert result["status"] == "staged_authenticated_model" + assert result["source_commit"] == "1" * 40 + + +def test_stage_model_failure_cleans_owned_staging_and_never_exposes_final_root( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + files = staged_model_files() + authorization = model_staging_authorization(files) + calls = 0 + + def download(**kwargs: object) -> str: + nonlocal calls + calls += 1 + if calls == 2: + raise OSError("injected download failure") + name = str(kwargs["filename"]) + target = Path(kwargs["cache_dir"]) / "snapshot" / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(files[name]) + return str(target) + + monkeypatch.setattr( + runner, + "_authenticate_model_staging_authorization", + lambda **_kwargs: authorization, + ) + output = tmp_path / "published-model" + with pytest.raises(OSError, match="injected"): + runner.stage_identity_bound_model( + frozen_identity_path=tmp_path / "identity.json", + expected_frozen_identity_sha256="d" * 64, + identity_commit="3" * 40, + repository_root=SCRIPT.parents[1], + repository_source_manifest_path=tmp_path / "source.json", + source_commit="1" * 40, + model_file_manifest_path=tmp_path / "manifest.json", + expected_model_file_manifest_sha256=authorization.model_manifest.file_sha256, + hub_cache_root=tmp_path / "cache", + output_root=output, + downloader=download, + ) + + assert not output.exists() + assert not list(tmp_path.glob(".published-model.staging-*")) + + +@pytest.mark.parametrize("outside_kind", ("outside", "wrong-content")) +def test_stage_model_rejects_untrusted_cache_payload_without_publication( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + outside_kind: str, +) -> None: + files = staged_model_files() + authorization = model_staging_authorization(files) + monkeypatch.setattr( + runner, + "_authenticate_model_staging_authorization", + lambda **_kwargs: authorization, + ) + + def download(**kwargs: object) -> str: + name = str(kwargs["filename"]) + if outside_kind == "outside": + target = tmp_path / "outside" / name + content = files[name] + else: + target = Path(kwargs["cache_dir"]) / "snapshot" / name + content = b"tampered" if name.endswith(".safetensors") else files[name] + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + return str(target) + + output = tmp_path / "published-model" + message = "outside" if outside_kind == "outside" else "size differs|authentication failed" + with pytest.raises(runner.CalibrationRunError, match=message): + runner.stage_identity_bound_model( + frozen_identity_path=tmp_path / "identity.json", + expected_frozen_identity_sha256="d" * 64, + identity_commit="3" * 40, + repository_root=SCRIPT.parents[1], + repository_source_manifest_path=tmp_path / "source.json", + source_commit="1" * 40, + model_file_manifest_path=tmp_path / "manifest.json", + expected_model_file_manifest_sha256=authorization.model_manifest.file_sha256, + hub_cache_root=tmp_path / "cache", + output_root=output, + downloader=download, + ) + assert not output.exists() + + +def test_cache_pointer_is_confined_and_never_published_as_a_link(tmp_path: Path) -> None: + cache = tmp_path / "cache" + blob = cache / "blobs" / "payload" + pointer = cache / "snapshots" / ("2" * 40) / "model.safetensors" + blob.parent.mkdir(parents=True) + pointer.parent.mkdir(parents=True) + blob.write_bytes(b"payload") + try: + pointer.symlink_to(blob) + except OSError as error: + pytest.skip(f"symlink creation is unavailable: {type(error).__name__}") + + assert runner._assert_regular_cache_payload(cache, pointer) == blob.resolve() + pointer.unlink() + outside = tmp_path / "outside-payload" + outside.write_bytes(b"payload") + pointer.symlink_to(outside) + with pytest.raises(runner.CalibrationRunError, match="escapes"): + runner._assert_regular_cache_payload(cache, pointer) + + +def test_atomic_model_directory_publish_never_replaces_racing_destination(tmp_path: Path) -> None: + source = tmp_path / "staging" + destination = tmp_path / "model" + source.mkdir() + destination.mkdir() + (source / "source.txt").write_text("source", encoding="utf-8") + (destination / "owner.txt").write_text("owner", encoding="utf-8") + + with pytest.raises(FileExistsError, match="refusing to overwrite"): + runner._atomic_rename_directory_no_overwrite(source, destination) + + assert (source / "source.txt").read_text(encoding="utf-8") == "source" + assert (destination / "owner.txt").read_text(encoding="utf-8") == "owner" + + +def test_runtime_probe_and_manifest_accept_exact_runtime_root_sys_path_sentinel( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + probe_payload = runner.canonical_json_bytes( + { + "base_sys_path": ["Lib", "."], + "machine": { + "architecture": "64bit", + "byteorder": "little", + "machine": "test-machine", + "pointer_bits": 64, + "system": "TestOS", + }, + "python": { + "abi_flags": "", + "cache_tag": "cpython-311", + "implementation": "CPython", + "version": "3.11.0", + }, + } + ) + monkeypatch.setattr( + runner.subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess( + args=args, + returncode=0, + stdout=probe_payload, + stderr=b"", + ), + ) + + probe = runner._probe_staged_interpreter(tmp_path / "python", tmp_path / "runtime") + assert probe.base_sys_path == ("Lib", ".") + + manifest_payload = json.loads(runtime_manifest_bytes()) + manifest_payload["base_sys_path"] = ["."] + manifest = runner.parse_calibration_runtime_manifest( + runner.canonical_json_bytes(manifest_payload) + ) + assert manifest.base_sys_path == (".",) + + +@pytest.mark.parametrize("value", ["", False, "/absolute", "../escape"]) +def test_runtime_root_sys_path_normalizer_rejects_malformed_entries(value: object) -> None: + with pytest.raises(runner.CalibrationRunError): + runner._canonical_base_sys_path_entry(value, context="test base sys.path") + + +def test_runtime_root_sentinel_remains_invalid_for_ordinary_repository_paths() -> None: + with pytest.raises(runner.CalibrationRunError): + runner._canonical_relative_path(".", context="ordinary repository path") + + def test_runtime_manifest_hashes_complete_record_inventory(tmp_path: Path) -> None: base_root = tmp_path / "base" package_root = tmp_path / "packages" @@ -1404,11 +2253,7 @@ def test_record_only_staging_preserves_scripts_outside_site_packages(tmp_path: P path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(content) record_path.parent.mkdir(parents=True, exist_ok=True) - record_text = ( - "demo/__init__.py,,\n" - "demo-1.0.dist-info/RECORD,,\n" - "../../Scripts/demo.exe,,\n" - ) + record_text = "demo/__init__.py,,\ndemo-1.0.dist-info/RECORD,,\n../../Scripts/demo.exe,,\n" record_path.write_text(record_text, encoding="utf-8") destination.mkdir() @@ -1507,8 +2352,13 @@ def test_capture_manifest_cli_modes_are_no_overwrite( class FakeSourceCapture: @staticmethod - def capture_experiment013_source_manifest(root: Path) -> dict[str, object]: + def capture_experiment013_source_manifest( + root: Path, + *, + git_executable: Path, + ) -> dict[str, object]: assert root == repository_root + assert git_executable == authenticated_git_path() return source_manifest @staticmethod @@ -1529,9 +2379,12 @@ def canonical_experiment013_source_manifest_bytes( def verify_experiment013_source_manifest( manifest: Mapping[str, object], root: Path, + *, + git_executable: Path, ) -> dict[str, object]: assert manifest is source_manifest assert root == repository_root + assert git_executable == authenticated_git_path() verification_calls.append(dict(manifest)) return source_manifest @@ -1548,6 +2401,8 @@ def runtime_capture(**kwargs: object) -> bytes: runner.main( [ "capture-source-manifest", + "--git-executable", + str(authenticated_git_path()), "--repository-root", str(repository_root), "--output", @@ -1562,6 +2417,8 @@ def runtime_capture(**kwargs: object) -> bytes: runner.main( [ "capture-source-manifest", + "--git-executable", + str(authenticated_git_path()), "--repository-root", str(repository_root), "--output", @@ -1571,6 +2428,8 @@ def runtime_capture(**kwargs: object) -> bytes: runtime_args = [ "capture-runtime-manifest", + "--git-executable", + str(authenticated_git_path()), "--output", str(runtime_output), "--base-runtime-root", @@ -1587,6 +2446,7 @@ def runtime_capture(**kwargs: object) -> bytes: assert runtime_calls == [ { "base_runtime_root": tmp_path / "base", + "git_executable_path": authenticated_git_path(), "interpreter_path": tmp_path / "base" / "python.exe", "package_import_paths": {"packages": "Lib/site-packages"}, "package_roots": {"packages": tmp_path / "packages"}, @@ -1600,12 +2460,14 @@ def model_capture( revision: str, *, transformers_version: str, + token: bool, ) -> bytes: assert (model_id, revision, transformers_version) == ( "example/model", "4" * 40, "5.14.1", ) + assert token is False return b"model\n" monkeypatch.setattr(runner, "capture_model_file_manifest_from_hub", model_capture) @@ -1628,6 +2490,63 @@ def model_capture( assert model_output.read_bytes() == b"model\n" +def test_stage_model_cli_forwards_only_explicit_authorization_and_roots( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[dict[str, object]] = [] + + def stage(**kwargs: object) -> dict[str, object]: + calls.append(dict(kwargs)) + return {"status": "staged_authenticated_model"} + + monkeypatch.setattr(runner, "stage_identity_bound_model", stage) + arguments = [ + "stage-model", + "--git-executable", + str(authenticated_git_path()), + "--frozen-identity", + str(tmp_path / "identity.json"), + "--expected-frozen-identity-sha256", + "1" * 64, + "--identity-commit", + "2" * 40, + "--repository-root", + str(tmp_path / "repository"), + "--repository-source-manifest", + str(tmp_path / "source.json"), + "--source-commit", + "3" * 40, + "--model-file-manifest", + str(tmp_path / "model.json"), + "--expected-model-file-manifest-sha256", + "4" * 64, + "--hub-cache-root", + str(tmp_path / "cache"), + "--output-root", + str(tmp_path / "output"), + "--local-files-only", + ] + + assert runner.main(arguments) == 0 + assert calls == [ + { + "expected_frozen_identity_sha256": "1" * 64, + "expected_model_file_manifest_sha256": "4" * 64, + "frozen_identity_path": tmp_path / "identity.json", + "git_executable_path": authenticated_git_path(), + "hub_cache_root": tmp_path / "cache", + "identity_commit": "2" * 40, + "local_files_only": True, + "model_file_manifest_path": tmp_path / "model.json", + "output_root": tmp_path / "output", + "repository_root": tmp_path / "repository", + "repository_source_manifest_path": tmp_path / "source.json", + "source_commit": "3" * 40, + } + ] + + def test_source_manifest_output_location_allows_only_external_or_ignored_paths( tmp_path: Path, ) -> None: @@ -1638,13 +2557,32 @@ def test_source_manifest_output_location_allows_only_external_or_ignored_paths( outside = tmp_path / "outside.json" ignored = repository / "artifacts" / "source.json" - assert runner._assert_source_manifest_output_location(repository, outside) == outside.resolve() - assert runner._assert_source_manifest_output_location(repository, ignored) == ignored.resolve() + git_executable = runner._authenticate_git_executable(authenticated_git_path()) + assert ( + runner._assert_source_manifest_output_location( + repository, outside, git_executable=git_executable + ) + == outside.resolve() + ) + assert ( + runner._assert_source_manifest_output_location( + repository, ignored, git_executable=git_executable + ) + == ignored.resolve() + ) with pytest.raises(runner.CalibrationRunError, match="must be ignored"): - runner._assert_source_manifest_output_location(repository, repository / "source.json") + runner._assert_source_manifest_output_location( + repository, + repository / "source.json", + git_executable=git_executable, + ) with pytest.raises(runner.CalibrationRunError, match="repository metadata"): - runner._assert_source_manifest_output_location(repository, repository / ".git" / "new.json") + runner._assert_source_manifest_output_location( + repository, + repository / ".git" / "new.json", + git_executable=git_executable, + ) def test_public_main_binds_manifest_bytes_before_adapter_import( @@ -1713,6 +2651,7 @@ def test_public_main_binds_manifest_bytes_before_adapter_import( base_runtime_root=tmp_path / "base", package_roots={"packages": tmp_path / "packages"}, package_import_paths={"packages": "Lib/site-packages"}, + git_executable_path=authenticated_git_path(), pycache_prefix=tmp_path / "pycache", ), interpreter_path=tmp_path / "base" / "python.exe", diff --git a/tests/test_screen_static_q468_stage_a.py b/tests/test_screen_static_q468_stage_a.py new file mode 100644 index 0000000..9bc18b0 --- /dev/null +++ b/tests/test_screen_static_q468_stage_a.py @@ -0,0 +1,1860 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import os +import subprocess +import sys +import weakref +from pathlib import Path +from types import MappingProxyType, SimpleNamespace +from typing import Any + +import pytest +import torch + +ROOT = Path(__file__).resolve().parents[1] +MODULE_PATH = ROOT / "scripts" / "screen_static_q468_stage_a.py" +SPEC = importlib.util.spec_from_file_location("screen_static_q468_stage_a_test", MODULE_PATH) +assert SPEC is not None and SPEC.loader is not None +stage_a = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = stage_a +SPEC.loader.exec_module(stage_a) + + +def _digest(label: str) -> str: + return hashlib.sha256(label.encode()).hexdigest() + + +def _identity_bytes(execution_bindings: dict[str, str] | None = None) -> bytes: + execution = ( + {name: _digest(name) for name in sorted(stage_a.EXECUTION_BINDING_FIELDS)} + if execution_bindings is None + else dict(execution_bindings) + ) + calibration = {name: _digest(name) for name in sorted(stage_a.BINDING_FIELDS)} + records = [] + for family in ("pg19", "ruler", "humaneval_plus"): + for rank in range(4): + prompt = 4_096 if family == "pg19" else 7 + records.append( + { + "family": family, + "selection_rank": rank, + "token_span": { + "prefill_start": 0, + "prefill_stop": prompt, + "scored_start": prompt, + "scored_stop": prompt + 2, + "cache_exposed_start": prompt + 1, + "cache_exposed_stop": prompt + 2, + }, + } + ) + evidence = { + "schema_version": 5, + "identity_schema": "recurquant.experiment013.identity-frozen.v5", + "status": "frozen", + "phase": "stage_a", + "identity_only": True, + "promotion_required": False, + "promotion": {"explicit": True}, + "execution_bindings": execution, + "calibration_binding": calibration, + "records": records, + } + return stage_a.canonical_json_bytes( + { + "canonical_evidence_sha256": stage_a.sha256_bytes( + stage_a.canonical_json_bytes(evidence) + ), + "evidence": evidence, + } + ) + + +def _bootstrap() -> Any: + return stage_a.bootstrap_stage_a_identity(_identity_bytes()) + + +def _runtime_record() -> dict[str, object]: + return { + "base_runtime_file_count": 3, + "distribution_count": 1, + "distributions": [["torch", "2.13.0+cu130"]], + "file_count": 5, + "git_executable_absolute_path_sha256": _digest("git-absolute-path"), + "git_executable_sha256": _digest("git-executable"), + "git_executable_size_bytes": 123_456, + "interpreter_sha256": _digest("interpreter"), + "machine_name": "AMD64", + "manifest_file_sha256": _bootstrap().execution_bindings[ + "calibration_runtime_manifest_file_sha256" + ], + "package_root_count": 1, + "python_cache_tag": "cpython-311", + "python_implementation": "CPython", + "python_version": "3.11.15", + } + + +def _runtime_namespace() -> Any: + record = _runtime_record() + return SimpleNamespace( + **{ + **record, + "distributions": tuple(tuple(item) for item in record["distributions"]), + } + ) + + +def test_bootstrap_requires_promoted_v5_and_exact_forward_formula() -> None: + decoded = _bootstrap() + assert decoded.expected_forward_count == 9 * 12 * 2 + assert set(decoded.execution_bindings) == stage_a.EXECUTION_BINDING_FIELDS + assert set(decoded.calibration_binding) == stage_a.BINDING_FIELDS + + root = stage_a._strict_json(_identity_bytes(), context="test identity") + root["evidence"]["phase"] = "calibration" + root["canonical_evidence_sha256"] = stage_a.sha256_bytes( + stage_a.canonical_json_bytes(root["evidence"]) + ) + with pytest.raises(stage_a.StageAError, match="promoted resolver-v5"): + stage_a.bootstrap_stage_a_identity(stage_a.canonical_json_bytes(root)) + + +@pytest.mark.parametrize( + "mutation", + ( + lambda root: root["evidence"]["records"].reverse(), + lambda root: root["evidence"]["records"][0]["token_span"].__setitem__( + "cache_exposed_start", 4_096 + ), + lambda root: root["evidence"]["calibration_binding"].pop( + "static_k29334_policy_file_sha256" + ), + ), +) +def test_bootstrap_rejects_reorder_span_and_binding_inventory(mutation: Any) -> None: + root = stage_a._strict_json(_identity_bytes(), context="test identity") + mutation(root) + root["canonical_evidence_sha256"] = stage_a.sha256_bytes( + stage_a.canonical_json_bytes(root["evidence"]) + ) + with pytest.raises(stage_a.StageAError): + stage_a.bootstrap_stage_a_identity(stage_a.canonical_json_bytes(root)) + + +class _Sequence: + def __init__(self, family: str, rank: int) -> None: + prompt = (10, 11, 12) + target = (1, 2) + self.prompt_token_ids = prompt + self.target_token_ids = target + self.identity_record = { + "family": family, + "canonical_id": f"{family}-{rank}", + "selection_rank": rank, + "identity_record_sha256": _digest(f"{family}-{rank}"), + "token_span": { + "prefill_start": 0, + "prefill_stop": len(prompt), + "scored_start": len(prompt), + "scored_stop": len(prompt) + len(target), + "cache_exposed_start": len(prompt) + 1, + "cache_exposed_stop": len(prompt) + len(target), + }, + } + + +class _Engine: + def __init__(self) -> None: + self.events: list[str] = [] + self.forward_calls = 0 + self.load_count = 0 + self.close_count = 0 + + def load_model(self, authenticated_model_files: object) -> object: + self.events.append("load") + self.load_count += 1 + return authenticated_model_files + + def close_model(self, model: object) -> None: + self.events.append("close") + self.close_count += 1 + + def begin_method(self, model: object, method: Any, sequence: object) -> object: + self.events.append(f"begin:{method.method_id}") + return {"method": method.method_id} + + @staticmethod + def _observation(position: int, target: int, method_id: str) -> Any: + logits = torch.tensor([1.2, 0.4, -0.8], dtype=torch.float32) + logp = torch.log_softmax(logits, dim=-1) + return stage_a.ForwardObservation( + position=position, + target_token_id=target, + comparison_logits=logits, + target_nll=-float(logp[target].item()), + top1_token_id=0, + local_codec_sse=0.01, + trajectory_nmse=0.02, + latency_ns=10, + peak_allocated_bytes=20, + peak_reserved_bytes=30, + resident_bytes=stage_a.EXPECTED_RECURRENT_RESIDENT_BYTES[method_id], + transient_bytes=50, + ) + + def prefill( + self, + session: object, + *, + prompt_token_ids: tuple[int, ...], + first_target_token_id: int, + position: int, + ) -> Any: + del prompt_token_ids + self.forward_calls += 1 + return self._observation(position, first_target_token_id, session["method"]) + + def step( + self, + session: object, + *, + input_token_id: int, + target_token_id: int, + position: int, + ) -> Any: + del input_token_id + self.forward_calls += 1 + return self._observation(position, target_token_id, session["method"]) + + def end_method(self, session: object) -> dict[str, int]: + return { + "resident_bytes": stage_a.EXPECTED_RECURRENT_RESIDENT_BYTES[session["method"]], + "raw_state_workspace_peak_bytes": 20, + "query_workspace_peak_bytes": 30, + } + + def runtime_snapshot(self, model: object) -> dict[str, object]: + del model + return { + "attention_implementation": "eager", + "capability": [12, 0], + "cuda_runtime": "13.0", + "device_index": 0, + "model_class": "Qwen3_5ForCausalLM", + "model_config_class": "Qwen3_5TextConfig", + "model_parameter_dtype": "torch.bfloat16", + "name": "Test GPU", + "torch_version": "2.13.0+cu130", + "total_memory_bytes": 8_000_000_000, + } + + +def _authenticated() -> Any: + bootstrap = _bootstrap() + methods = tuple( + stage_a.StageAMethodSpec( + method, + None, + ( + None + if method in {stage_a.FP32_METHOD, stage_a.DYNAMIC_K27030_METHOD} + else _digest(f"policy-{method}") + ), + "test", + ) + for method in stage_a.METHOD_ORDER + ) + return stage_a.AuthenticatedStageA( + bootstrap_identity=bootstrap, + identity=object(), + binding=SimpleNamespace(file_sha256=_digest("binding")), + dependency_bytes=MappingProxyType({}), + execution_artifact_bytes=MappingProxyType({}), + source_manifest=MappingProxyType({}), + source_manifest_file_sha256=_digest("source"), + source_commit="1" * 40, + input_bundle=object(), + input_bundle_manifest_file_sha256=_digest("input-bundle"), + model_manifest=object(), + authenticated_model_files=object(), + runtime_manifest=object(), + authenticated_runtime=_runtime_namespace(), + resolver=SimpleNamespace(), + capture=SimpleNamespace(), + calibration_runner=SimpleNamespace(), + source_module=SimpleNamespace(), + methods=methods, + ) + + +def _smoke_report() -> dict[str, object]: + return { + "profile": stage_a.PRESEAL_ENGINE_SMOKE_PROFILE, + "passed": True, + "stage_a_content_accessed": False, + "input_profile": "fixed_public_synthetic_4096_plus_128_v3", + "prompt_token_count": stage_a.PRESEAL_ENGINE_SMOKE_PROMPT_TOKEN_COUNT, + "continuation_token_count": stage_a.PRESEAL_ENGINE_SMOKE_TARGET_TOKEN_COUNT, + "model_load_count": 1, + "method_order": list(stage_a.METHOD_ORDER), + "forward_count": len(stage_a.METHOD_ORDER) * len(stage_a.PRESEAL_ENGINE_SMOKE_TARGET), + "method_receipts": [ + { + "method_id": method_id, + "forward_count": len(stage_a.PRESEAL_ENGINE_SMOKE_TARGET), + "logical_recurrent_resident_bytes": ( + stage_a.EXPECTED_RECURRENT_RESIDENT_BYTES[method_id] + ), + "equal_byte_observer_required": method_id != stage_a.FP32_METHOD, + } + for method_id in stage_a.METHOD_ORDER + ], + "device": _Engine().runtime_snapshot(object()), + } + + +def _materialization() -> Any: + sequences = tuple( + _Sequence(family, rank) + for family in ("pg19", "ruler", "humaneval_plus") + for rank in range(4) + ) + return SimpleNamespace( + sequences=sequences, + frozen_identity_file_sha256=_bootstrap().file_sha256, + calibration_binding_file_sha256=_digest("binding"), + capture_input_sha256=_digest("capture"), + token_sequence_manifest_sha256=_digest("tokens"), + tokenizer_manifest_sha256=_digest("tokenizer"), + ) + + +def test_fixed_grid_counts_every_prefill_and_transition_without_token_ids() -> None: + engine = _Engine() + result = stage_a.evaluate_materialized_stage_a( + _authenticated(), _materialization(), engine, object() + ) + assert result.forward_count == 9 * 12 * 2 + assert engine.forward_calls == result.forward_count + assert len(result.raw_rows) == 9 * 12 + assert len(result.gate_rows) == 9 * 12 + assert [row["method_id"] for row in result.raw_rows[:9]] == list(stage_a.METHOD_ORDER) + assert all( + "input_token_id" not in row and "target_token_id" not in row for row in result.raw_rows + ) + assert not hasattr(stage_a, "_token_hash") + assert all( + "input_token_ids_sha256" not in row and "target_token_ids_sha256" not in row + for row in result.raw_rows + ) + assert all(len(row["authenticated_transition_sha256"]) == 64 for row in result.raw_rows) + assert all(all(row["finite_checks"].values()) for row in result.raw_rows) + + +def test_comparison_logits_are_compact_fp32_and_kl_matches_reviewed_equation() -> None: + reference = torch.tensor([2.0, -0.5, 0.25, 1.0], dtype=torch.float32) + candidate = torch.tensor([1.75, -0.1, 0.5, 0.8], dtype=torch.float32) + reference_logp = torch.log_softmax(reference, dim=-1) + candidate_logp = torch.log_softmax(candidate, dim=-1) + expected = float((reference_logp.exp() * (reference_logp - candidate_logp)).sum(dim=-1).item()) + assert stage_a._kl(reference, candidate) == expected + observation = stage_a.ForwardObservation( + position=3, + target_token_id=2, + comparison_logits=reference, + target_nll=-float(reference_logp[2].item()), + top1_token_id=0, + ) + validated = stage_a._finite_observation( + observation, + expected_position=3, + expected_target=2, + ) + assert validated.comparison_logits is reference + assert reference.device.type == "cpu" and reference.dtype == torch.float32 + assert reference.is_contiguous() + assert ".tolist()" not in MODULE_PATH.read_text(encoding="utf-8") + + +def test_comparison_logits_reject_python_tuple_and_non_fp32_tensor() -> None: + base = _Engine._observation(3, 1, stage_a.FP32_METHOD) + for invalid in ((1.0, 0.0), torch.tensor([1.0, 0.0], dtype=torch.float64)): + with pytest.raises(stage_a.StageAError, match="contiguous CPU float32"): + stage_a._finite_observation( + stage_a.dataclasses.replace(base, comparison_logits=invalid), + expected_position=3, + expected_target=1, + ) + + +def test_evaluator_rejects_method_reordering_before_any_forward() -> None: + authenticated = _authenticated() + reordered = (authenticated.methods[1], authenticated.methods[0], *authenticated.methods[2:]) + authenticated = stage_a.dataclasses.replace(authenticated, methods=reordered) + engine = _Engine() + with pytest.raises(stage_a.StageAError, match="reordered"): + stage_a.evaluate_materialized_stage_a(authenticated, _materialization(), engine, object()) + assert engine.forward_calls == 0 + + +def test_evaluator_rejects_materialized_span_tamper() -> None: + materialization = _materialization() + materialization.sequences[0].identity_record["token_span"]["cache_exposed_start"] = 3 + engine = _Engine() + with pytest.raises(stage_a.StageAError, match="token span"): + stage_a.evaluate_materialized_stage_a(_authenticated(), materialization, engine, object()) + + +def test_run_orders_auth_seal_materialization_single_load_and_reauthentication( + monkeypatch: pytest.MonkeyPatch, +) -> None: + events: list[str] = [] + authenticated = _authenticated() + materialization = _materialization() + reservation = stage_a.AttemptReservation(MappingProxyType({}), "1" * 40, "2" * 40, "3" * 40) + + class OrderedEngine(_Engine): + def load_model(self, authenticated_model_files: object) -> object: + events.append("load") + return super().load_model(authenticated_model_files) + + def close_model(self, model: object) -> None: + events.append("close") + super().close_model(model) + + def begin_method(self, model: object, method: Any, sequence: object) -> object: + if "evaluate" not in events: + events.append("evaluate") + return super().begin_method(model, method, sequence) + + engine = OrderedEngine() + + def authenticate(_config: Any) -> Any: + events.append("authenticate") + return authenticated + + def reserve(_config: Any, _authenticated: Any) -> Any: + events.append("reserve") + return reservation + + def smoke(_authenticated: Any) -> Any: + events.append("smoke") + return _smoke_report() + + def reauthenticate( + _config: Any, + _authenticated: Any, + _reservation: Any, + ) -> None: + events.append("reauthenticate") + + def persist(_config: Any, current: Any, updates: Any) -> Any: + events.append(f"receipt:{updates['status']}") + return stage_a.dataclasses.replace( + current, + receipt=MappingProxyType({**dict(current.receipt), **dict(updates)}), + ) + + def materialize(_config: Any, _authenticated: Any) -> Any: + events.append("materialize") + return materialization + + def build( + _authenticated: Any, + _materialization: Any, + _evaluation: Any, + _reservation: Any, + ) -> bytes: + events.append("build") + return stage_a.canonical_json_bytes({"evidence": {"stage_a_passed": True}}) + + def publish(_config: Any, _reservation: Any, _payload: bytes, summary: Any) -> Any: + events.append("publish") + return {"status": "published", **dict(summary)} + + monkeypatch.setattr(stage_a, "build_execution_artifact", build) + services = stage_a.StageAServices( + authenticate=authenticate, + reauthenticate=reauthenticate, + preseal_smoke=smoke, + reserve=reserve, + materialize=materialize, + engine=engine, + persist_receipt=persist, + publish=publish, + record_failure=lambda *_args: events.append("failure"), + ) + result = stage_a.run_stage_a(SimpleNamespace(), services) + assert result == {"status": "published", "stage_a_passed": True} + assert engine.load_count == 1 and engine.close_count == 1 + assert events == [ + "authenticate", + "smoke", + "reauthenticate", + "reserve", + "receipt:preseal_engine_smoke_bound_before_materialization", + "reauthenticate", + "receipt:stage_a_materialization_entered", + "materialize", + "receipt:stage_a_content_materialized_before_model_load", + "reauthenticate", + "load", + "reauthenticate", + "receipt:model_loaded_once_before_evaluation", + "evaluate", + "close", + "receipt:evaluation_returned_before_result_build", + "reauthenticate", + "build", + "reauthenticate", + "publish", + ] + + +def test_loaded_model_is_closed_if_post_load_receipt_persistence_fails() -> None: + authenticated = _authenticated() + materialization = _materialization() + reservation = stage_a.AttemptReservation( + MappingProxyType({}), + "1" * 40, + "2" * 40, + "3" * 40, + ) + engine = _Engine() + failures: list[str] = [] + + def persist(_config: Any, current: Any, updates: Any) -> Any: + if updates["status"] == "model_loaded_once_before_evaluation": + raise OSError("injected receipt failure after model load") + return stage_a.dataclasses.replace( + current, + receipt=MappingProxyType({**dict(current.receipt), **dict(updates)}), + ) + + services = stage_a.StageAServices( + authenticate=lambda _config: authenticated, + reauthenticate=lambda *_args: None, + preseal_smoke=lambda _authenticated: _smoke_report(), + reserve=lambda *_args: reservation, + materialize=lambda *_args: materialization, + engine=engine, + persist_receipt=persist, + publish=lambda *_args: pytest.fail("publication must not run"), + record_failure=lambda _config, _reservation, _error, phase: failures.append(phase), + ) + + with pytest.raises(OSError, match="after model load"): + stage_a.run_stage_a(SimpleNamespace(), services) + + assert engine.load_count == 1 + assert engine.close_count == 1 + assert failures == ["post_model_load_receipt"] + + +def test_run_rejects_device_runtime_drift_after_evaluation() -> None: + authenticated = _authenticated() + materialization = _materialization() + reservation = stage_a.AttemptReservation( + MappingProxyType({}), + "1" * 40, + "2" * 40, + "3" * 40, + ) + + class DriftingEngine(_Engine): + def __init__(self) -> None: + super().__init__() + self.snapshot_count = 0 + + def runtime_snapshot(self, model: object) -> dict[str, object]: + result = super().runtime_snapshot(model) + self.snapshot_count += 1 + if self.snapshot_count == 2: + result["name"] = "Drifted GPU" + return result + + engine = DriftingEngine() + failures: list[str] = [] + + def persist(_config: Any, current: Any, updates: Any) -> Any: + return stage_a.dataclasses.replace( + current, + receipt=MappingProxyType({**dict(current.receipt), **dict(updates)}), + ) + + services = stage_a.StageAServices( + authenticate=lambda _config: authenticated, + reauthenticate=lambda *_args: None, + preseal_smoke=lambda _authenticated: _smoke_report(), + reserve=lambda *_args: reservation, + materialize=lambda *_args: materialization, + engine=engine, + persist_receipt=persist, + publish=lambda *_args: pytest.fail("publication must not run"), + record_failure=lambda _config, _reservation, _error, phase: failures.append(phase), + ) + + with pytest.raises(stage_a.StageAError, match="between model load and evaluation end"): + stage_a.run_stage_a(SimpleNamespace(), services) + + assert engine.snapshot_count == 2 + assert engine.close_count == 1 + assert failures == ["post_evaluation_device_reauthentication"] + + +def test_reconstruction_has_no_q48_or_uniform_supplied_dependency(monkeypatch: Any) -> None: + dependencies = {name: name.encode() for name in stage_a.BINDING_DEPENDENCY_NAMES} + dependencies["extra_q48_policy"] = b"forbidden" + with pytest.raises(stage_a.StageAError, match="exact eight"): + stage_a.reconstruct_stage_a_methods( + dependency_bytes=dependencies, + frozen_stage_a_identity=object(), + source_commit="1" * 40, + ) + + aggregate = SimpleNamespace( + d4=object(), + d6=object(), + d8=object(), + sequence_score_manifest_sha256=_digest("scores"), + ) + scores = SimpleNamespace( + calibration_identity_sha256=_digest("calibration-identity"), + calibration_scores_sha256=_digest("calibration-scores"), + geometry=SimpleNamespace(total_rows=4), + aggregate=aggregate, + ) + + class Policy: + def __init__(self, method_id: str) -> None: + self.method_id = method_id + self.source_commit = "1" * 40 + self.policy_sha256 = _digest(method_id) + + static = SimpleNamespace( + FROZEN_STATIC_Q48_PROMOTIONS=14_739, + static_q48_distortion_sha256=lambda *args, **kwargs: _digest("q48-scores"), + build_static_rht_q468_policy=lambda *args, method_id, **kwargs: Policy(method_id), + build_static_rht_q48_policy=lambda *args, method_id, **kwargs: Policy(method_id), + deserialize_static_rht_q468_policy=lambda payload: Policy(payload.decode()), + serialize_static_rht_q468_policy=lambda policy: policy.method_id.encode(), + serialize_static_rht_q48_policy=lambda policy: policy.method_id.encode(), + ) + calibration = SimpleNamespace(deserialize_calibration_score_artifact=lambda payload: scores) + original_import = stage_a.importlib.import_module + + def fake_import(name: str) -> object: + if name == "recurquant.static_q468": + return static + if name == "recurquant.static_q468_calibration": + return calibration + return original_import(name) + + monkeypatch.setattr(stage_a.importlib, "import_module", fake_import) + clean = {name: name.encode() for name in stage_a.BINDING_DEPENDENCY_NAMES} + clean["static_k27030_policy_artifact"] = stage_a.STATIC_K27030_METHOD.encode() + clean["static_k29334_policy_artifact"] = stage_a.PRIMARY_K29334_METHOD.encode() + clean["static_mse_k29334_policy_artifact"] = stage_a.MSE_K29334_METHOD.encode() + clean["static_fisher_k29334_policy_artifact"] = stage_a.FISHER_K29334_METHOD.encode() + identity = SimpleNamespace( + calibration_binding={"calibration_identity_file_sha256": _digest("calibration-identity")}, + tokenizer_manifest_sha256=_digest("tokenizer"), + ) + methods = stage_a.reconstruct_stage_a_methods( + dependency_bytes=clean, + frozen_stage_a_identity=identity, + source_commit="1" * 40, + ) + origins = {method.method_id: method.origin for method in methods} + assert tuple(method.method_id for method in methods) == stage_a.METHOD_ORDER + assert origins[stage_a.Q48_METHOD] == "reconstructed_candidate_scores_p14739" + assert origins[stage_a.UNIFORM_Q4_METHOD] == "reconstructed_candidate_scores_k0" + assert origins[stage_a.UNIFORM_Q8_METHOD] == "reconstructed_candidate_scores_k8" + + +def _run(root: Path, *args: str, input_bytes: bytes | None = None) -> str: + completed = subprocess.run( + ["git", *args], + cwd=root, + input=input_bytes, + capture_output=True, + check=True, + ) + return completed.stdout.decode().strip() + + +def _git_config(tmp_path: Path) -> tuple[Any, bytes]: + _run(tmp_path, "init") + _run(tmp_path, "config", "user.email", "stage-a@example.invalid") + _run(tmp_path, "config", "user.name", "Stage A Test") + _run(tmp_path, "config", "core.autocrlf", "false") + identity_bytes = _identity_bytes() + identity = tmp_path / "identity.json" + identity.write_bytes(identity_bytes) + (tmp_path / ".gitignore").write_text("out/\n", encoding="utf-8") + _run(tmp_path, "add", "identity.json", ".gitignore") + _run(tmp_path, "commit", "-m", "identity authorization") + commit = _run(tmp_path, "rev-parse", "HEAD") + config = stage_a.StageAConfig( + frozen_identity_path=identity, + calibration_binding_path=tmp_path / "binding.json", + repository_source_manifest_path=tmp_path / "source.json", + runtime_manifest_path=tmp_path / "runtime.json", + model_file_manifest_path=tmp_path / "model.json", + parquet_materialization_manifest_path=tmp_path / "parquet.json", + model_root=tmp_path / "model-root", + cache_root=tmp_path, + ruler_root=tmp_path, + input_bundle_root=tmp_path / "input-bundle", + repository_root=tmp_path, + source_commit=commit, + identity_commit=commit, + output_dir=tmp_path / "out", + expected_runtime_manifest_sha256=_digest("runtime"), + expected_model_file_manifest_sha256=_digest("model"), + expected_parquet_materialization_manifest_sha256=_digest("parquet"), + ) + return config, identity_bytes + + +def test_one_run_receipt_precedes_empty_diff_commit_cas_and_recovery( + tmp_path: Path, + monkeypatch: Any, +) -> None: + config, _identity = _git_config(tmp_path) + authenticated = _authenticated() + authenticated = stage_a.dataclasses.replace( + authenticated, + source_commit=config.source_commit, + ) + reservation = stage_a.reserve_one_run(config, authenticated) + assert _run(tmp_path, "rev-parse", "HEAD") == reservation.seal_commit + assert _run(tmp_path, "show", "-s", "--format=%P", reservation.seal_commit) == ( + reservation.h1_commit + ) + assert _run(tmp_path, "show", "-s", "--format=%T", reservation.seal_commit) == ( + reservation.tree + ) + assert reservation.receipt["status"] == "reserved_before_stage_a_content_access" + assert reservation.receipt["automatic_retry_authorized"] is False + lock_path = stage_a._identity_attempt_lock_path( + tmp_path, + authenticated.bootstrap_identity.file_sha256, + git_executable_path=config.git_executable_path, + ) + assert lock_path.is_file() + assert ( + hashlib.sha256(lock_path.read_bytes()).hexdigest() + == reservation.receipt["identity_scoped_attempt_lock_file_sha256"] + ) + with pytest.raises(stage_a.StageAError, match="already exists"): + stage_a.reserve_one_run(config, authenticated) + + monkeypatch.setattr( + stage_a, + "_authenticate_recovery_boundary", + lambda _config, _receipt, **_kwargs: ( + authenticated.bootstrap_identity, + authenticated.binding.file_sha256, + reservation.seal_commit, + ), + ) + recovered = stage_a.recover_interrupted(config) + assert recovered == { + "status": "consumed_attempt_interrupted_no_result", + "result_available": False, + "automatic_retry_authorized": False, + } + + +def test_one_run_is_not_reopened_by_fresh_output_or_head_reset(tmp_path: Path) -> None: + config, _identity = _git_config(tmp_path) + authenticated = stage_a.dataclasses.replace( + _authenticated(), + source_commit=config.source_commit, + ) + reservation = stage_a.reserve_one_run(config, authenticated) + + seal_as_h1 = stage_a.dataclasses.replace( + config, + identity_commit=reservation.seal_commit, + output_dir=config.output_dir / "retry-from-seal", + ) + with pytest.raises(stage_a.StageAError, match="one-run seal in Git history"): + stage_a.reserve_one_run(seal_as_h1, authenticated) + + _run(tmp_path, "update-ref", "HEAD", reservation.h1_commit) + reset_to_original_h1 = stage_a.dataclasses.replace( + config, + output_dir=config.output_dir / "retry-after-reset", + ) + with pytest.raises(stage_a.StageAError, match="one-run seal in Git history"): + stage_a.reserve_one_run(reset_to_original_h1, authenticated) + + +def test_post_reservation_reauthentication_rejects_same_tree_seal_substitution( + tmp_path: Path, +) -> None: + config, identity_bytes = _git_config(tmp_path) + authenticated = stage_a.dataclasses.replace( + _authenticated(), + source_commit=config.source_commit, + ) + reservation = stage_a.reserve_one_run(config, authenticated) + alternate = _run( + tmp_path, + "commit-tree", + reservation.tree, + "-p", + reservation.h1_commit, + input_bytes=b"unrelated empty child\n", + ) + _run(tmp_path, "update-ref", "HEAD", alternate, reservation.seal_commit) + + with pytest.raises(stage_a.StageAError, match="reserved one-run seal"): + stage_a._assert_tracked_identity_bytes_after_seal( + config, + identity_bytes, + authenticated, + reservation, + ) + + +def test_reservation_rejects_linked_or_reparse_output_parent(tmp_path: Path) -> None: + config, _identity = _git_config(tmp_path) + authenticated = stage_a.dataclasses.replace( + _authenticated(), + source_commit=config.source_commit, + ) + actual_output = tmp_path / "actual-output" + actual_output.mkdir() + linked_output = tmp_path / "linked-output" + try: + os.symlink(actual_output, linked_output, target_is_directory=True) + except OSError as error: + pytest.skip(f"directory symlink creation is unavailable: {error}") + linked_config = stage_a.dataclasses.replace(config, output_dir=linked_output) + + with pytest.raises(stage_a.StageAError, match="link or reparse"): + stage_a.reserve_one_run(linked_config, authenticated) + + assert not (actual_output / stage_a.ATTEMPT_FILENAME).exists() + + +def test_pre_cas_recovery_branch_remains_reachable( + tmp_path: Path, + monkeypatch: Any, +) -> None: + config, _identity = _git_config(tmp_path) + authenticated = stage_a.dataclasses.replace( + _authenticated(), + source_commit=config.source_commit, + ) + original = stage_a._git_process + + def fail_cas( + git_executable_path: Path | None, + root: Path, + *arguments: str, + input_bytes: bytes | None = None, + ) -> Any: + if arguments and arguments[0] == "update-ref": + return subprocess.CompletedProcess( + ["git", *arguments], + 1, + b"", + b"injected CAS failure", + ) + return original( + git_executable_path, + root, + *arguments, + input_bytes=input_bytes, + ) + + monkeypatch.setattr(stage_a, "_git_process", fail_cas) + with pytest.raises(stage_a.StageAError, match="compare-and-swap failed"): + stage_a.reserve_one_run(config, authenticated) + monkeypatch.setattr(stage_a, "_git_process", original) + receipt = stage_a._strict_json(config.attempt_path.read_bytes(), context="test receipt") + seal = receipt["one_run_seal_commit"] + observed: dict[str, object] = {} + + def authenticate_boundary(_config: Any, _receipt: Any, **kwargs: Any) -> Any: + observed.update(kwargs) + return authenticated.bootstrap_identity, authenticated.binding.file_sha256, seal + + monkeypatch.setattr(stage_a, "_authenticate_recovery_boundary", authenticate_boundary) + recovered = stage_a.recover_interrupted(config) + assert observed == {"allow_pre_cas_head": True} + assert recovered["status"] == "pre_cas_attempt_receipt_present_no_automatic_retry" + assert recovered["result_available"] is False + receipt_before = config.attempt_path.read_bytes() + assert stage_a.recover_interrupted(config) == recovered + assert config.attempt_path.read_bytes() == receipt_before + + +def test_lock_only_crash_is_administratively_recovered_without_retry( + tmp_path: Path, + monkeypatch: Any, +) -> None: + config, _identity = _git_config(tmp_path) + authenticated = stage_a.dataclasses.replace( + _authenticated(), + source_commit=config.source_commit, + ) + original_write = stage_a._exclusive_write + injected = False + + def fail_attempt_write(path: Path, payload: bytes) -> None: + nonlocal injected + if Path(path) == config.attempt_path and not injected: + injected = True + raise OSError("injected attempt-receipt write failure") + original_write(path, payload) + + monkeypatch.setattr(stage_a, "_exclusive_write", fail_attempt_write) + with pytest.raises(OSError, match="attempt-receipt"): + stage_a.reserve_one_run(config, authenticated) + monkeypatch.setattr(stage_a, "_exclusive_write", original_write) + assert not config.attempt_path.exists() + assert _run(tmp_path, "rev-parse", "HEAD") == config.identity_commit + lock_path = stage_a._identity_attempt_lock_path( + tmp_path, + authenticated.bootstrap_identity.file_sha256, + git_executable_path=config.git_executable_path, + ) + lock = stage_a._strict_json(lock_path.read_bytes(), context="test attempt lock") + observed: list[bool] = [] + + def authenticate_boundary(_config: Any, _receipt: Any, **kwargs: Any) -> Any: + observed.append(kwargs.get("allow_pre_cas_head") is True) + return ( + authenticated.bootstrap_identity, + authenticated.binding.file_sha256, + lock["one_run_seal_commit"], + ) + + monkeypatch.setattr(stage_a, "_authenticate_recovery_boundary", authenticate_boundary) + recovered = stage_a.recover_interrupted(config) + assert recovered["status"] == "pre_cas_attempt_receipt_present_no_automatic_retry" + assert recovered["automatic_retry_authorized"] is False + assert observed == [True, True] + receipt = stage_a._strict_json(config.attempt_path.read_bytes(), context="recovered receipt") + assert "lock_only_recovered_at_utc" in receipt + + +def test_recovery_rejects_missing_completed_result_and_preserves_failure( + tmp_path: Path, + monkeypatch: Any, +) -> None: + config, _identity = _git_config(tmp_path) + authenticated = stage_a.dataclasses.replace( + _authenticated(), + source_commit=config.source_commit, + ) + reservation = stage_a.reserve_one_run(config, authenticated) + monkeypatch.setattr( + stage_a, + "_authenticate_recovery_boundary", + lambda _config, _receipt, **_kwargs: ( + authenticated.bootstrap_identity, + authenticated.binding.file_sha256, + reservation.seal_commit, + ), + ) + completed = stage_a.persist_receipt( + config, + reservation, + { + "status": "completed_with_authenticated_stage_a_result", + "result_available": True, + }, + ) + with pytest.raises(stage_a.StageAError, match="missing its published result"): + stage_a.recover_interrupted(config) + + stage_a.persist_receipt( + config, + completed, + { + "status": "consumed_attempt_failed_no_automatic_retry", + "result_available": False, + }, + ) + before = config.attempt_path.read_bytes() + recovered = stage_a.recover_interrupted(config) + assert recovered["status"] == "consumed_attempt_failed_no_automatic_retry" + assert config.attempt_path.read_bytes() == before + + +def test_real_recovery_boundary_authenticates_final_lock_and_is_idempotent( + tmp_path: Path, +) -> None: + source = stage_a.importlib.import_module("recurquant.experiment013_source") + git_executable = stage_a._authenticated_git_executable(None) + _run(tmp_path, "init") + _run(tmp_path, "config", "user.email", "stage-a@example.invalid") + _run(tmp_path, "config", "user.name", "Stage A Test") + _run(tmp_path, "config", "core.autocrlf", "false") + for relative in source.EXPERIMENT013_SOURCE_PATHS: + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(f"authenticated fixture for {relative}\n".encode()) + (tmp_path / ".gitignore").write_text("evidence/\nout/\n", encoding="utf-8") + _run(tmp_path, "add", ".") + _run(tmp_path, "commit", "-m", "frozen Experiment 013 source") + h0 = _run(tmp_path, "rev-parse", "HEAD") + + source_manifest = source.capture_experiment013_source_manifest( + tmp_path, + git_executable=git_executable, + ) + source_bytes = source.canonical_experiment013_source_manifest_bytes(source_manifest) + evidence_dir = tmp_path / "evidence" + evidence_dir.mkdir() + artifact_bytes = { + "repository_source_manifest_file_sha256": source_bytes, + "calibration_runtime_manifest_file_sha256": b"runtime manifest fixture\n", + "model_file_manifest_file_sha256": b"model manifest fixture\n", + "parquet_materialization_manifest_file_sha256": b"parquet manifest fixture\n", + } + artifact_paths = { + "repository_source_manifest_file_sha256": evidence_dir / "source.json", + "calibration_runtime_manifest_file_sha256": evidence_dir / "runtime.json", + "model_file_manifest_file_sha256": evidence_dir / "model.json", + "parquet_materialization_manifest_file_sha256": evidence_dir / "parquet.json", + } + for name, payload in artifact_bytes.items(): + artifact_paths[name].write_bytes(payload) + binding_bytes = b"calibration binding fixture\n" + binding_path = evidence_dir / "binding.json" + binding_path.write_bytes(binding_bytes) + execution_bindings = { + name: stage_a.sha256_bytes(payload) for name, payload in artifact_bytes.items() + } + identity_bytes = _identity_bytes(execution_bindings) + identity_path = tmp_path / "identity.json" + identity_path.write_bytes(identity_bytes) + _run(tmp_path, "add", "identity.json") + _run(tmp_path, "commit", "-m", "authorize Stage-A identity") + h1 = _run(tmp_path, "rev-parse", "HEAD") + + config = stage_a.StageAConfig( + frozen_identity_path=identity_path, + calibration_binding_path=binding_path, + repository_source_manifest_path=artifact_paths["repository_source_manifest_file_sha256"], + runtime_manifest_path=artifact_paths["calibration_runtime_manifest_file_sha256"], + model_file_manifest_path=artifact_paths["model_file_manifest_file_sha256"], + parquet_materialization_manifest_path=artifact_paths[ + "parquet_materialization_manifest_file_sha256" + ], + model_root=tmp_path / "model-root", + cache_root=tmp_path / "cache-root", + ruler_root=tmp_path / "ruler-root", + input_bundle_root=tmp_path / "input-bundle", + repository_root=tmp_path, + source_commit=h0, + identity_commit=h1, + output_dir=tmp_path / "out", + expected_runtime_manifest_sha256=execution_bindings[ + "calibration_runtime_manifest_file_sha256" + ], + expected_model_file_manifest_sha256=execution_bindings["model_file_manifest_file_sha256"], + expected_parquet_materialization_manifest_sha256=execution_bindings[ + "parquet_materialization_manifest_file_sha256" + ], + git_executable_path=git_executable, + ) + bootstrap = stage_a.bootstrap_stage_a_identity(identity_bytes) + authenticated = stage_a.dataclasses.replace( + _authenticated(), + bootstrap_identity=bootstrap, + binding=SimpleNamespace(file_sha256=stage_a.sha256_bytes(binding_bytes)), + execution_artifact_bytes=MappingProxyType(artifact_bytes), + source_manifest=MappingProxyType(source_manifest), + source_manifest_file_sha256=stage_a.sha256_bytes(source_bytes), + source_commit=h0, + input_bundle_manifest_file_sha256=_digest("real-input-bundle"), + ) + reservation = stage_a.reserve_one_run(config, authenticated) + lock_path = stage_a._identity_attempt_lock_path( + tmp_path, + bootstrap.file_sha256, + git_executable_path=git_executable, + ) + lock = stage_a._strict_json(lock_path.read_bytes(), context="real recovery lock") + assert lock["schema"] == stage_a.IDENTITY_ATTEMPT_LOCK_SCHEMA + assert reservation.receipt["schema"] == stage_a.ATTEMPT_SCHEMA + assert reservation.receipt["stage_a_input_bundle_manifest_file_sha256"] == _digest( + "real-input-bundle" + ) + + recovered = stage_a.recover_interrupted(config) + receipt_after_first_recovery = config.attempt_path.read_bytes() + recovered_again = stage_a.recover_interrupted(config) + + assert recovered == { + "status": "consumed_attempt_interrupted_no_result", + "result_available": False, + "automatic_retry_authorized": False, + } + assert recovered_again == recovered + assert config.attempt_path.read_bytes() == receipt_after_first_recovery + assert _run(tmp_path, "rev-parse", "HEAD") == reservation.seal_commit + + +def _test_execution_artifact( + reservation: Any, + authenticated: Any, +) -> bytes: + materialization = _materialization() + evaluation = stage_a.evaluate_materialized_stage_a( + authenticated, + materialization, + _Engine(), + object(), + ) + gate = stage_a.importlib.import_module("recurquant.experiment013_stage_a") + gate_bytes = gate.build_stage_a_evidence_artifact( + evaluation.examples, + evaluation.gate_rows, + stage_a_identity_file_sha256=authenticated.bootstrap_identity.file_sha256, + stage_a_calibration_binding_file_sha256=authenticated.binding.file_sha256, + ) + gate_artifact = stage_a._strict_json(gate_bytes, context="test gate") + verified_gate = gate.deserialize_stage_a_evidence_artifact(gate_bytes) + smoke = _smoke_report() + smoke_sha256 = stage_a.sha256_bytes(stage_a.canonical_json_bytes(smoke)) + runtime_record = stage_a._authenticated_runtime_record( + authenticated.authenticated_runtime, + expected_manifest_file_sha256=authenticated.bootstrap_identity.execution_bindings[ + "calibration_runtime_manifest_file_sha256" + ], + ) + evidence = { + "artifact_revision": stage_a.RUNNER_REVISION, + "claim_boundary": stage_a.CLAIM_BOUNDARY, + "dependencies": { + "stage_a_identity_file_sha256": authenticated.bootstrap_identity.file_sha256, + "stage_a_calibration_binding_file_sha256": authenticated.binding.file_sha256, + "repository_source_manifest_file_sha256": authenticated.source_manifest_file_sha256, + "stage_a_input_bundle_manifest_file_sha256": ( + authenticated.input_bundle_manifest_file_sha256 + ), + "execution_bindings": dict(authenticated.bootstrap_identity.execution_bindings), + "method_specs": stage_a._method_spec_receipts(authenticated.methods), + }, + "execution_contract": stage_a._expected_execution_contract(evaluation.forward_count), + "materialization": stage_a._materialization_receipt(materialization), + "method_runtime": [dict(row) for row in evaluation.method_runtime], + "one_run": { + "attempt_schema": stage_a.ATTEMPT_SCHEMA, + "automatic_retry_authorized": False, + "h0_source_commit": authenticated.source_commit, + "h1_identity_commit": reservation.h1_commit, + "identity_scoped_attempt_lock_file_sha256": reservation.receipt[ + "identity_scoped_attempt_lock_file_sha256" + ], + "one_run_marker": stage_a.ONE_RUN_MARKER, + "one_run_seal_commit": reservation.seal_commit, + "one_run_seal_message_sha256": reservation.receipt["one_run_seal_message_sha256"], + "one_run_seal_tree": reservation.tree, + "preseal_engine_smoke_sha256": smoke_sha256, + "stage_a_input_bundle_manifest_file_sha256": ( + authenticated.input_bundle_manifest_file_sha256 + ), + }, + "preseal_engine_smoke": smoke, + "raw_token_evidence": [dict(row) for row in evaluation.raw_rows], + "runtime": { + "authenticated_runtime": dict(runtime_record), + "device": _Engine().runtime_snapshot(object()), + }, + "stage_a_gate_artifact": gate_artifact, + "stage_a_gate_file_sha256": verified_gate.file_sha256, + "stage_a_passed": verified_gate.passed, + } + return stage_a.canonical_json_bytes( + { + "artifact_kind": stage_a.EXECUTION_ARTIFACT_KIND, + "schema_version": stage_a.EXECUTION_ARTIFACT_SCHEMA, + "canonical_evidence_sha256": stage_a.sha256_bytes( + stage_a.canonical_json_bytes(evidence) + ), + "evidence": evidence, + } + ) + + +def _verification_kwargs(reservation: Any, authenticated: Any) -> dict[str, Any]: + return { + "expected_identity_file_sha256": authenticated.bootstrap_identity.file_sha256, + "expected_calibration_binding_file_sha256": authenticated.binding.file_sha256, + "expected_h1_commit": reservation.h1_commit, + "expected_seal_commit": reservation.seal_commit, + "expected_source_commit": authenticated.source_commit, + "expected_source_manifest_file_sha256": authenticated.source_manifest_file_sha256, + "expected_input_bundle_manifest_file_sha256": ( + authenticated.input_bundle_manifest_file_sha256 + ), + "expected_execution_bindings": authenticated.bootstrap_identity.execution_bindings, + "expected_method_specs": stage_a._method_spec_receipts(authenticated.methods), + "expected_materialization": stage_a._materialization_receipt(_materialization()), + "expected_seal_tree": reservation.tree, + "expected_seal_message_sha256": reservation.receipt["one_run_seal_message_sha256"], + "expected_attempt_lock_file_sha256": reservation.receipt[ + "identity_scoped_attempt_lock_file_sha256" + ], + "expected_authenticated_runtime": stage_a._authenticated_runtime_record( + authenticated.authenticated_runtime, + expected_manifest_file_sha256=authenticated.bootstrap_identity.execution_bindings[ + "calibration_runtime_manifest_file_sha256" + ], + ), + "expected_device_runtime": _Engine().runtime_snapshot(object()), + "expected_forward_count": authenticated.bootstrap_identity.expected_forward_count, + } + + +def test_execution_artifact_verifier_binds_seal_runtime_and_redaction(tmp_path: Path) -> None: + config, _identity = _git_config(tmp_path) + authenticated = stage_a.dataclasses.replace( + _authenticated(), + source_commit=config.source_commit, + ) + reservation = stage_a.reserve_one_run(config, authenticated) + payload = _test_execution_artifact(reservation, authenticated) + verified = stage_a.verify_execution_artifact( + payload, + **_verification_kwargs(reservation, authenticated), + ) + assert verified["schema_version"] == 3 + + tampered = stage_a._strict_json(payload, context="test result") + tampered["evidence"]["raw_token_evidence"][0]["target_token_ids_sha256"] = _digest( + "reversible-token" + ) + tampered["canonical_evidence_sha256"] = stage_a.sha256_bytes( + stage_a.canonical_json_bytes(tampered["evidence"]) + ) + with pytest.raises(stage_a.StageAError, match="low-entropy"): + stage_a.verify_execution_artifact( + stage_a.canonical_json_bytes(tampered), + **_verification_kwargs(reservation, authenticated), + ) + + +def test_execution_artifact_verifier_rejects_forged_outer_diagnostics( + tmp_path: Path, +) -> None: + config, _identity = _git_config(tmp_path) + authenticated = stage_a.dataclasses.replace( + _authenticated(), + source_commit=config.source_commit, + ) + reservation = stage_a.reserve_one_run(config, authenticated) + payload = _test_execution_artifact(reservation, authenticated) + mutations = ( + ( + lambda root: root["evidence"]["raw_token_evidence"][0].__setitem__( + "decode_model_forward_latency_ns", -1 + ), + "nonnegative", + ), + ( + lambda root: root["evidence"]["method_runtime"][0].__setitem__( + "policy_origin", "forged" + ), + "policy identity", + ), + ( + lambda root: root["evidence"]["method_runtime"][0]["storage"].__setitem__( + "forged", "accepted" + ), + "frozen schema", + ), + ( + lambda root: root["evidence"]["materialization"].__setitem__( + "capture_input_sha256", _digest("forged-capture") + ), + "materialization receipt drifted", + ), + ( + lambda root: root["evidence"]["runtime"]["authenticated_runtime"].__setitem__( + "machine_name", "FORGED_MACHINE" + ), + "authenticated runtime evidence drifted", + ), + ( + lambda root: root["evidence"]["runtime"]["device"].__setitem__("name", "FORGED_GPU"), + "device runtime evidence drifted", + ), + ) + for mutate, message in mutations: + forged = stage_a._strict_json(payload, context="forged result") + mutate(forged) + forged["canonical_evidence_sha256"] = stage_a.sha256_bytes( + stage_a.canonical_json_bytes(forged["evidence"]) + ) + with pytest.raises(stage_a.StageAError, match=message): + stage_a.verify_execution_artifact( + stage_a.canonical_json_bytes(forged), + **_verification_kwargs(reservation, authenticated), + ) + + +def test_execution_artifact_builder_emits_self_verifying_bundle(tmp_path: Path) -> None: + config, _identity = _git_config(tmp_path) + base = _authenticated() + authenticated = stage_a.dataclasses.replace( + base, + source_commit=config.source_commit, + authenticated_runtime=_runtime_namespace(), + ) + reservation = stage_a.reserve_one_run(config, authenticated) + materialization = _materialization() + evaluation = stage_a.evaluate_materialized_stage_a( + authenticated, + materialization, + _Engine(), + object(), + ) + smoke = _smoke_report() + reservation = stage_a.persist_receipt( + config, + reservation, + { + "status": "preseal_engine_smoke_bound_before_materialization", + "preseal_engine_smoke": smoke, + "preseal_engine_smoke_sha256": stage_a.sha256_bytes( + stage_a.canonical_json_bytes(smoke) + ), + "post_load_authenticated_runtime": _runtime_record(), + "post_load_device_runtime": _Engine().runtime_snapshot(object()), + }, + ) + + payload = stage_a.build_execution_artifact( + authenticated, + materialization, + evaluation, + reservation, + ) + result = stage_a._strict_json(payload, context="built result") + assert result["evidence"]["one_run"]["one_run_seal_commit"] == reservation.seal_commit + assert result["evidence"]["runtime"]["device"]["name"] == "Test GPU" + + +@pytest.mark.parametrize("completed_receipt_before_crash", (False, True)) +def test_recovery_finishes_missing_completion_marker_without_reevaluation( + tmp_path: Path, + completed_receipt_before_crash: bool, + monkeypatch: Any, +) -> None: + config, _identity = _git_config(tmp_path) + authenticated = stage_a.dataclasses.replace( + _authenticated(), + source_commit=config.source_commit, + ) + reservation = stage_a.reserve_one_run(config, authenticated) + payload = _test_execution_artifact(reservation, authenticated) + canonical_hash = stage_a._strict_json(payload, context="test result")[ + "canonical_evidence_sha256" + ] + smoke = _smoke_report() + updates: dict[str, Any] = { + "status": "result_prepared_before_atomic_publication", + "result_available": False, + "capture_input_sha256": _materialization().capture_input_sha256, + "token_sequence_manifest_sha256": _materialization().token_sequence_manifest_sha256, + "tokenizer_manifest_sha256": _materialization().tokenizer_manifest_sha256, + "preseal_engine_smoke": smoke, + "preseal_engine_smoke_sha256": stage_a.sha256_bytes(stage_a.canonical_json_bytes(smoke)), + "post_load_authenticated_runtime": _runtime_record(), + "post_load_device_runtime": _Engine().runtime_snapshot(object()), + "result_file_sha256": stage_a.sha256_bytes(payload), + "result_canonical_evidence_sha256": canonical_hash, + } + if completed_receipt_before_crash: + updates.update( + { + "status": "completed_with_authenticated_stage_a_result", + "result_available": True, + } + ) + stage_a.persist_receipt(config, reservation, updates) + stage_a._atomic_publish_new(config.output_path, payload) + assert not config.complete_path.exists() + monkeypatch.setattr( + stage_a, + "_authenticate_recovery_boundary", + lambda _config, _receipt, **_kwargs: ( + authenticated.bootstrap_identity, + authenticated.binding.file_sha256, + reservation.seal_commit, + ), + ) + + recovered = stage_a.recover_interrupted(config) + + assert recovered["result_available"] is True + assert recovered["completion_marker_available"] is True + marker = stage_a._strict_json(config.complete_path.read_bytes(), context="test marker") + assert marker["result_file_sha256"] == stage_a.sha256_bytes(payload) + assert marker["attempt_file_sha256"] == stage_a.sha256_bytes(config.attempt_path.read_bytes()) + attempt_before = config.attempt_path.read_bytes() + marker_before = config.complete_path.read_bytes() + assert stage_a.recover_interrupted(config) == recovered + assert config.attempt_path.read_bytes() == attempt_before + assert config.complete_path.read_bytes() == marker_before + + +def test_failure_recording_rebases_stale_in_memory_receipt(tmp_path: Path) -> None: + config, _identity = _git_config(tmp_path) + authenticated = stage_a.dataclasses.replace( + _authenticated(), + source_commit=config.source_commit, + ) + reservation = stage_a.reserve_one_run(config, authenticated) + stage_a.persist_receipt( + config, + reservation, + {"status": "result_prepared_before_atomic_publication"}, + ) + + stage_a.record_failure(config, reservation, RuntimeError("injected"), "publication") + + receipt = stage_a._strict_json(config.attempt_path.read_bytes(), context="test receipt") + assert receipt["status"] == "consumed_attempt_failed_no_automatic_retry" + assert receipt["failure_phase"] == "publication" + + +def test_atomic_publication_refuses_overwrite(tmp_path: Path) -> None: + destination = tmp_path / "result.json" + stage_a._atomic_publish_new(destination, b"first") + with pytest.raises(stage_a.StageAError, match="overwrite"): + stage_a._atomic_publish_new(destination, b"second") + assert destination.read_bytes() == b"first" + + +class _CausalCache: + def __init__(self, length: int = 0) -> None: + self.length = length + + def get_seq_length(self) -> int: + return self.length + + def storage_summary(self) -> dict[str, int]: + return {"resident_bytes": 0} + + +class _CausalModel: + def __init__( + self, + cache: _CausalCache, + *, + advance: bool = True, + replace_cache: bool = False, + ) -> None: + self.parameter = torch.nn.Parameter(torch.zeros(()), requires_grad=False) + self.cache = cache + self.advance = advance + self.replace_cache = replace_cache + self.call: dict[str, Any] | None = None + + def parameters(self) -> Any: + yield self.parameter + + def __call__(self, **kwargs: Any) -> Any: + self.call = kwargs + if self.advance: + self.cache.length += int(kwargs["input_ids"].shape[1]) + returned = _CausalCache(self.cache.length) if self.replace_cache else self.cache + return SimpleNamespace( + logits=torch.zeros((1, 1, 8), dtype=torch.float32), + past_key_values=returned, + ) + + +def _disable_cuda_measurements(monkeypatch: Any) -> None: + monkeypatch.setattr(torch.cuda, "synchronize", lambda _device: None) + monkeypatch.setattr(torch.cuda, "reset_peak_memory_stats", lambda _device: None) + monkeypatch.setattr(torch.cuda, "max_memory_allocated", lambda _device: 0) + monkeypatch.setattr(torch.cuda, "max_memory_reserved", lambda _device: 0) + + +def test_device_runtime_rejects_fallback_model_dtype() -> None: + runtime = _Engine().runtime_snapshot(object()) + runtime["model_parameter_dtype"] = "torch.float32" + with pytest.raises(stage_a.StageAError, match="BF16 dtype contract"): + stage_a._validated_device_runtime(runtime) + + +def test_loaded_model_contract_rejects_fallback_parameter_dtype() -> None: + class Config: + def __init__(self) -> None: + self._attn_implementation = "eager" + self._attn_implementation_internal = "eager" + + class Model: + def __init__(self, dtype: torch.dtype) -> None: + self.config = Config() + self.parameter = torch.nn.Parameter(torch.zeros((), dtype=dtype)) + + def parameters(self) -> Any: + yield self.parameter + + transformers = SimpleNamespace( + Qwen3_5ForCausalLM=Model, + Qwen3_5TextConfig=Config, + ) + device = torch.device("cpu") + stage_a.TorchStageAEngine._assert_loaded_model_contract( + Model(torch.bfloat16), + torch=torch, + transformers=transformers, + device=device, + ) + with pytest.raises(stage_a.StageAError, match="is not BF16"): + stage_a.TorchStageAEngine._assert_loaded_model_contract( + Model(torch.float32), + torch=torch, + transformers=transformers, + device=device, + ) + sdpa_model = Model(torch.bfloat16) + sdpa_model.config._attn_implementation = "sdpa" + sdpa_model.config._attn_implementation_internal = "sdpa" + with pytest.raises(stage_a.StageAError, match="retain eager attention"): + stage_a.TorchStageAEngine._assert_loaded_model_contract( + sdpa_model, + torch=torch, + transformers=transformers, + device=device, + ) + + +def _dynamic_recurrent_cache(dtype: torch.dtype, *, complete: bool) -> object: + static = stage_a.importlib.import_module("recurquant.static_q468") + geometry = static.FROZEN_QWEN35_STATIC_Q468_GEOMETRY + shape = (1, geometry.heads, geometry.key_rows, geometry.value_width) + expected = set(static.FROZEN_RECURRENT_LAYER_INDICES) + layer_count = max(expected) + 1 + layers: list[object] = [] + for layer_index in range(layer_count): + if layer_index in expected and (complete or layer_index == min(expected)): + layers.append( + SimpleNamespace( + recurrent_states=[torch.zeros(shape, dtype=dtype)], + is_recurrent_states_initialized=[True], + ) + ) + else: + layers.append(SimpleNamespace(recurrent_states=None)) + return SimpleNamespace(layers=layers) + + +def _packed_recurrent_cache(dtype: torch.dtype, *, device: torch.device) -> object: + static = stage_a.importlib.import_module("recurquant.static_q468") + geometry = static.FROZEN_QWEN35_STATIC_Q468_GEOMETRY + shape = (1, geometry.heads, geometry.key_rows, geometry.value_width) + states = { + layer_index: torch.zeros(shape, dtype=dtype, device=device) + for layer_index in static.FROZEN_RECURRENT_LAYER_INDICES + } + return SimpleNamespace( + checkpoint=SimpleNamespace(materialize=lambda: states), + ) + + +def test_fp32_reference_rejects_bf16_live_cache_state() -> None: + engine = stage_a.TorchStageAEngine() + engine._torch = torch + parameter = torch.nn.Parameter(torch.zeros((), dtype=torch.bfloat16)) + engine._model = SimpleNamespace(parameters=lambda: iter([parameter])) + with pytest.raises(stage_a.StageAError, match="non-FP32 recurrent state"): + engine._recurrent_states( + _dynamic_recurrent_cache(torch.bfloat16, complete=False), + packed=False, + ) + + +def test_fp32_reference_derives_exact_live_byte_ledger() -> None: + engine = stage_a.TorchStageAEngine() + engine._torch = torch + parameter = torch.nn.Parameter(torch.zeros((), dtype=torch.bfloat16)) + engine._model = SimpleNamespace(parameters=lambda: iter([parameter])) + states = engine._recurrent_states( + _dynamic_recurrent_cache(torch.float32, complete=True), + packed=False, + ) + observed_bytes = sum(tensor.numel() * tensor.element_size() for tensor in states.values()) + assert observed_bytes == stage_a.EXPECTED_RECURRENT_RESIDENT_BYTES[stage_a.FP32_METHOD] + + +def test_packed_reference_rejects_bf16_live_materialization() -> None: + engine = stage_a.TorchStageAEngine() + engine._torch = torch + parameter = torch.nn.Parameter(torch.zeros((), dtype=torch.bfloat16)) + engine._model = SimpleNamespace(parameters=lambda: iter([parameter])) + with pytest.raises(stage_a.StageAError, match="non-FP32 recurrent state"): + engine._recurrent_states( + _packed_recurrent_cache(torch.bfloat16, device=torch.device("cpu")), + packed=True, + ) + + +def test_packed_reference_rejects_wrong_live_device() -> None: + engine = stage_a.TorchStageAEngine() + engine._torch = torch + parameter = torch.nn.Parameter( + torch.empty((), dtype=torch.bfloat16, device=torch.device("meta")) + ) + engine._model = SimpleNamespace(parameters=lambda: iter([parameter])) + with pytest.raises(stage_a.StageAError, match="model CUDA device"): + engine._recurrent_states( + _packed_recurrent_cache(torch.float32, device=torch.device("cpu")), + packed=True, + ) + + +def test_production_engine_binds_explicit_positions_and_cache_advancement( + monkeypatch: Any, +) -> None: + _disable_cuda_measurements(monkeypatch) + cache = _CausalCache() + model = _CausalModel(cache) + engine = stage_a.TorchStageAEngine() + engine._torch = torch + + observation = engine._forward( + { + "model": model, + "cache": cache, + "method": stage_a.StageAMethodSpec( + stage_a.FP32_METHOD, + None, + None, + "test", + ), + }, + (4, 5, 6), + target_token_id=2, + position=2, + scored=False, + ) + + assert observation.position == 2 + assert cache.length == 3 + assert model.call is not None + assert model.call["past_key_values"] is cache + assert model.call["position_ids"].tolist() == [[0, 1, 2]] + assert model.call["cache_position"].tolist() == [0, 1, 2] + + +@pytest.mark.parametrize( + ("initial_length", "advance", "replace_cache", "message"), + ( + (1, True, False, "length drifted before"), + (0, True, True, "different Stage-A cache"), + (0, False, False, "did not advance"), + ), +) +def test_production_engine_rejects_noncausal_cache_behavior( + monkeypatch: Any, + initial_length: int, + advance: bool, + replace_cache: bool, + message: str, +) -> None: + _disable_cuda_measurements(monkeypatch) + cache = _CausalCache(initial_length) + model = _CausalModel(cache, advance=advance, replace_cache=replace_cache) + engine = stage_a.TorchStageAEngine() + engine._torch = torch + + with pytest.raises(stage_a.StageAError, match=message): + engine._forward( + { + "model": model, + "cache": cache, + "method": stage_a.StageAMethodSpec( + stage_a.FP32_METHOD, + None, + None, + "test", + ), + }, + (4, 5, 6), + target_token_id=2, + position=2, + scored=False, + ) + + +def test_production_engine_installs_and_removes_equal_byte_observer( + monkeypatch: Any, +) -> None: + identity = _digest("observer-lifecycle") + model = SimpleNamespace( + parameters=lambda: iter([SimpleNamespace(device="cuda:0")]), + ) + cache = SimpleNamespace(storage_summary=lambda: {"resident_bytes": 123}) + events: list[str] = [] + + class Observer: + def __init__(self, observed_model: object, *, caches: list[object]) -> None: + assert observed_model is model + assert caches == [cache] + + def __enter__(self) -> Any: + events.append("install") + return self + + def remove(self) -> None: + events.append("remove") + + cache_module = SimpleNamespace( + create_qwen35_static_rht_cache=lambda *args, **kwargs: cache, + ) + observer_module = SimpleNamespace(Qwen35EqualByteObserver=Observer) + original_import = stage_a.importlib.import_module + + def fake_import(name: str) -> object: + if name == "recurquant.static_q468_cache": + return cache_module + if name == "recurquant.statelease_equal_byte_cache": + return observer_module + return original_import(name) + + monkeypatch.setattr(stage_a.importlib, "import_module", fake_import) + engine = stage_a.TorchStageAEngine() + engine._model = model + engine._torch = SimpleNamespace( + cuda=SimpleNamespace( + synchronize=lambda _device: None, + empty_cache=lambda: None, + ) + ) + engine._reference_states[identity] = [{}, {}] + policy = SimpleNamespace(policy_sha256=_digest("policy")) + method = stage_a.StageAMethodSpec( + stage_a.MSE_K29334_METHOD, + policy, + policy.policy_sha256, + "test", + ) + sequence = SimpleNamespace( + identity_record_sha256=identity, + target_token_ids=(1, 2, 3), + ) + + session = engine.begin_method(model, method, sequence) + session["step_index"] = 2 + assert engine.end_method(session) == {"resident_bytes": 123} + assert events == ["install", "remove"] + + events.clear() + session = engine.begin_method(model, method, sequence) + with pytest.raises(stage_a.StageAError, match="did not complete"): + engine.end_method(session) + assert events == ["install", "remove"] + + +def test_primary_method_completion_releases_fp32_trajectory() -> None: + engine = stage_a.TorchStageAEngine() + identity = _digest("release-reference") + engine._reference_states[identity] = [{}] + session = { + "cache": SimpleNamespace(storage_summary=lambda: {"resident_bytes": 1}), + "method": stage_a.StageAMethodSpec( + stage_a.PRIMARY_K29334_METHOD, + None, + _digest("primary-policy"), + "test", + ), + "identity_record_sha256": identity, + "expected_steps": 1, + "step_index": 1, + } + assert engine.end_method(session) == {"resident_bytes": 1} + assert identity not in engine._reference_states + + +def test_end_method_releases_completed_cache_before_next_method_boundary() -> None: + class Cache: + @staticmethod + def storage_summary() -> dict[str, int]: + return {"resident_bytes": 1} + + engine = stage_a.TorchStageAEngine() + cache = Cache() + cache_reference = weakref.ref(cache) + session = { + "model": object(), + "cache": cache, + "observer": None, + "method": stage_a.StageAMethodSpec( + stage_a.MSE_K29334_METHOD, + None, + _digest("release-cache-policy"), + "test", + ), + "identity_record_sha256": _digest("release-cache-identity"), + "expected_steps": 1, + "step_index": 1, + } + assert engine.end_method(session) == {"resident_bytes": 1} + assert session == {} + del cache + assert cache_reference() is None + + +@pytest.mark.parametrize("protected_kind", ("bundle", "model", "base", "package")) +@pytest.mark.parametrize("protected_inside_output", (False, True)) +def test_authenticated_root_output_overlap_is_rejected_without_consuming_attempt( + tmp_path: Path, + protected_kind: str, + protected_inside_output: bool, +) -> None: + config, _identity = _git_config(tmp_path) + protected_root = tmp_path / f"{protected_kind}-protected-root" + if protected_inside_output: + protected_root = config.output_dir / f"{protected_kind}-protected-root" + else: + config = stage_a.dataclasses.replace(config, output_dir=protected_root / "output") + if protected_kind == "bundle": + config = stage_a.dataclasses.replace(config, input_bundle_root=protected_root) + elif protected_kind == "model": + config = stage_a.dataclasses.replace(config, model_root=protected_root) + elif protected_kind == "base": + config = stage_a.dataclasses.replace(config, base_runtime_root=protected_root) + else: + config = stage_a.dataclasses.replace( + config, + package_roots=MappingProxyType({"fixture": protected_root}), + ) + authenticated = stage_a.dataclasses.replace( + _authenticated(), + source_commit=config.source_commit, + ) + head_before = _run(tmp_path, "rev-parse", "HEAD") + lock_path = stage_a._identity_attempt_lock_path( + config.repository_root, + authenticated.bootstrap_identity.file_sha256, + git_executable_path=config.git_executable_path, + ) + + with pytest.raises(stage_a.StageAError, match="must not overlap"): + stage_a.reserve_one_run(config, authenticated) + + assert _run(tmp_path, "rev-parse", "HEAD") == head_before + assert not lock_path.exists() + assert not config.attempt_path.exists() + assert not config.output_path.exists() + assert not config.complete_path.exists() diff --git a/tests/test_static_q468.py b/tests/test_static_q468.py index fd049be..b252735 100644 --- a/tests/test_static_q468.py +++ b/tests/test_static_q468.py @@ -14,9 +14,15 @@ FROZEN_STATIC_Q48_PROMOTIONS, FROZEN_STATIC_Q468_ABLATION_STEPS, FROZEN_STATIC_Q468_PRIMARY_STEPS, + FROZEN_STATIC_Q468_UNIFORM_Q4_STEPS, + FROZEN_STATIC_Q468_UNIFORM_Q8_STEPS, STATIC_Q48_COMPARATOR_METHOD, STATIC_Q468_ABLATION_METHOD, + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + STATIC_Q468_MSE_METHOD, STATIC_Q468_PRIMARY_METHOD, + STATIC_Q468_UNIFORM_Q4_METHOD, + STATIC_Q468_UNIFORM_Q8_METHOD, StaticPackedRhtQ48State, StaticPackedRhtQ468State, StaticRhtQ468Geometry, @@ -226,6 +232,44 @@ def test_frozen_static_ledgers_distinguish_data_alignment_and_budget_eligibility "selected_units": FROZEN_STATIC_Q468_ABLATION_STEPS, "target_resident_bytes": 3_454_664, } + for method_id in ( + STATIC_Q468_MSE_METHOD, + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + ): + assert accounting[method_id] == { + **accounting[STATIC_Q468_PRIMARY_METHOD], + "method_id": method_id, + } + assert accounting[STATIC_Q468_UNIFORM_Q4_METHOD] == { + "alignment_bytes": 0, + "budget_delta_bytes": 938_696, + "codec": "q468", + "data_bytes": 2_515_968, + "exact_budget_eligible": False, + "method_id": STATIC_Q468_UNIFORM_Q4_METHOD, + "payload_bytes": 2_359_296, + "pool_offset_bytes": 73_728, + "precision_code_bytes": 9_216, + "resident_bytes": 2_515_968, + "scale_bytes": 73_728, + "selected_units": FROZEN_STATIC_Q468_UNIFORM_Q4_STEPS, + "target_resident_bytes": 3_454_664, + } + assert accounting[STATIC_Q468_UNIFORM_Q8_METHOD] == { + "alignment_bytes": 0, + "budget_delta_bytes": -1_420_600, + "codec": "q468", + "data_bytes": 4_875_264, + "exact_budget_eligible": False, + "method_id": STATIC_Q468_UNIFORM_Q8_METHOD, + "payload_bytes": 4_718_592, + "pool_offset_bytes": 73_728, + "precision_code_bytes": 9_216, + "resident_bytes": 4_875_264, + "scale_bytes": 73_728, + "selected_units": FROZEN_STATIC_Q468_UNIFORM_Q8_STEPS, + "target_resident_bytes": 3_454_664, + } assert accounting[STATIC_Q48_COMPARATOR_METHOD] == { "alignment_bytes": 8, "budget_delta_bytes": 0, @@ -325,10 +369,13 @@ def test_policy_codes_offsets_hashes_and_serialization_are_deterministic(tmp_pat assert loaded.evidence_dict() == first.evidence_dict() assert torch.equal(loaded.precision_codes(), first.precision_codes()) assert torch.equal(loaded.pool_offsets, first.pool_offsets) - assert verify_static_rht_q468_policy( - loaded, - expected_policy_sha256=first.policy_sha256, - )["policy_sha256"] == first.policy_sha256 + assert ( + verify_static_rht_q468_policy( + loaded, + expected_policy_sha256=first.policy_sha256, + )["policy_sha256"] + == first.policy_sha256 + ) def test_q48_policy_artifact_is_deterministic_strict_and_atomically_published( @@ -358,10 +405,13 @@ def test_q48_policy_artifact_is_deterministic_strict_and_atomically_published( assert serialized == serialize_static_rht_q48_policy(second) loaded = deserialize_static_rht_q48_policy(serialized) assert loaded.evidence_dict() == first.evidence_dict() - assert verify_static_rht_q48_policy( - loaded, - expected_policy_sha256=first.policy_sha256, - )["policy_sha256"] == first.policy_sha256 + assert ( + verify_static_rht_q48_policy( + loaded, + expected_policy_sha256=first.policy_sha256, + )["policy_sha256"] + == first.policy_sha256 + ) path = tmp_path / "nested" / "q48-policy.json" save_static_rht_q48_policy(first, path) @@ -464,6 +514,22 @@ def test_q468_policy_rejects_unsupported_runtime_contracts( replace(_tiny_policy(), method_id=STATIC_Q48_COMPARATOR_METHOD) +@pytest.mark.parametrize( + "method_id", + [ + STATIC_Q468_MSE_METHOD, + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + STATIC_Q468_UNIFORM_Q4_METHOD, + STATIC_Q468_UNIFORM_Q8_METHOD, + ], +) +def test_reserved_q468_comparator_methods_cannot_identify_q48_policy( + method_id: str, +) -> None: + with pytest.raises(ValueError, match="Q468 method cannot identify a Q48 policy"): + replace(_tiny_q48_policy(), method_id=method_id) + + def test_pool_offsets_are_canonical_prefix_indices_within_each_pool() -> None: policy = _tiny_policy() codes = policy.precision_codes().reshape(-1) @@ -523,8 +589,7 @@ def test_tiny_static_pack_owns_exact_bytes_without_persistent_fp32() -> None: assert packed.resident_bytes == 47 assert packed.ledger.exact_budget_eligible is True assert ( - packed.policy.packed_precision_codes.data_ptr() - != policy.packed_precision_codes.data_ptr() + packed.policy.packed_precision_codes.data_ptr() != policy.packed_precision_codes.data_ptr() ) assert packed.policy.pool_offsets.data_ptr() != policy.pool_offsets.data_ptr() assert all( @@ -656,9 +721,9 @@ def test_pool_offsets_independently_reconstruct_q468_and_q48_payloads() -> None: (q468, observed_q468_codes, expected_q468_scales), (q48, observed_q48_codes, expected_q48_scales), ): - manually_dequantized = integer_codes.to(torch.float32) * scales.to( - torch.float32 - ).unsqueeze(1) + manually_dequantized = integer_codes.to(torch.float32) * scales.to(torch.float32).unsqueeze( + 1 + ) manually_restored = right_rht_decode( manually_dequantized.reshape(1, 1, 4, 8), layer_index=0, @@ -689,9 +754,7 @@ def test_q48_packed_state_rejects_alias_hidden_fp32_and_bad_payload() -> None: @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA device is unavailable") def test_static_q468_and_q48_reference_codecs_round_trip_on_cuda() -> None: generator = torch.Generator(device="cuda").manual_seed(2339) - source = { - 0: torch.randn((1, 1, 4, 8), generator=generator, device="cuda") - } + source = {0: torch.randn((1, 1, 4, 8), generator=generator, device="cuda")} q468 = pack_static_rht_q468(source, _tiny_policy()) q48 = pack_static_rht_q48(source, _tiny_q48_policy()) @@ -753,6 +816,101 @@ def test_real_geometry_primary_policy_has_exact_k_counts_and_uint16_offsets() -> replace(policy, model_id="Qwen/other-model") +@pytest.mark.parametrize( + "method_id", + [ + STATIC_Q468_MSE_METHOD, + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + ], +) +def test_official_k29334_comparator_policy_round_trip_and_strict_budget( + method_id: str, +) -> None: + geometry = FROZEN_QWEN35_STATIC_Q468_GEOMETRY + row = torch.arange(geometry.total_rows, dtype=torch.float64) + policy = build_static_rht_q468_policy( + (((17 * row + 13) % 1009) / 1009).reshape(geometry.layers, -1), + (((29 * row + 7) % 1013) / 1013).reshape(geometry.layers, -1), + (((43 * row + 3) % 1019) / 1019).reshape(geometry.layers, -1), + geometry=geometry, + marginal_steps=FROZEN_STATIC_Q468_PRIMARY_STEPS, + method_id=method_id, + calibration_manifest_sha256=MANIFEST_SHA256, + **BINDINGS, + ) + + serialized = serialize_static_rht_q468_policy(policy) + restored = deserialize_static_rht_q468_policy(serialized) + ledger = restored.evidence_dict()["ledger"] + + assert restored.method_id == method_id + assert restored.marginal_steps == FROZEN_STATIC_Q468_PRIMARY_STEPS + assert restored.policy_sha256 == policy.policy_sha256 + assert serialize_static_rht_q468_policy(restored) == serialized + assert ledger["resident_bytes"] == 3_454_664 + assert ledger["target_resident_bytes"] == 3_454_664 + assert ledger["budget_delta_bytes"] == 0 + assert ledger["exact_budget_eligible"] is True + + with pytest.raises(ValueError, match="wrong exact-K budget"): + replace(policy, marginal_steps=FROZEN_STATIC_Q468_PRIMARY_STEPS - 1) + + +@pytest.mark.parametrize( + ("method_id", "marginal_steps", "expected_code", "expected_resident"), + [ + ( + STATIC_Q468_UNIFORM_Q4_METHOD, + FROZEN_STATIC_Q468_UNIFORM_Q4_STEPS, + 0, + 2_515_968, + ), + ( + STATIC_Q468_UNIFORM_Q8_METHOD, + FROZEN_STATIC_Q468_UNIFORM_Q8_STEPS, + 2, + 4_875_264, + ), + ], +) +def test_official_uniform_rht_anchor_policy_uses_same_q468_physical_codec( + method_id: str, + marginal_steps: int, + expected_code: int, + expected_resident: int, +) -> None: + geometry = FROZEN_QWEN35_STATIC_Q468_GEOMETRY + row = torch.arange(geometry.total_rows, dtype=torch.float64).reshape(1, -1) + policy = build_static_rht_q468_policy( + row + 3.0, + row + 2.0, + row + 1.0, + geometry=geometry, + marginal_steps=marginal_steps, + method_id=method_id, + calibration_manifest_sha256=MANIFEST_SHA256, + **BINDINGS, + ) + + assert torch.all(policy.precision_codes() == expected_code).item() + assert policy.evidence_dict()["ledger"]["resident_bytes"] == expected_resident + assert policy.codec_revision == "rht-q468-pools-u16-offsets-v1" + restored = deserialize_static_rht_q468_policy(serialize_static_rht_q468_policy(policy)) + assert restored.policy_sha256 == policy.policy_sha256 + + wrong_steps = 1 if marginal_steps == 0 else marginal_steps - 1 + with pytest.raises(ValueError, match="wrong exact-K budget"): + replace(policy, marginal_steps=wrong_steps) + with pytest.raises(ValueError, match="frozen geometry"): + replace( + policy, + geometry=replace( + geometry, + target_resident_bytes=geometry.target_resident_bytes + 8, + ), + ) + + def test_real_q468_policy_has_a_physical_exact_3454664_byte_state() -> None: """Construct every real Q4/Q6/Q8 pool without allocating dense model state.""" @@ -774,12 +932,8 @@ def test_real_q468_policy_has_a_physical_exact_3454664_byte_state() -> None: q4_count, q6_count, q8_count = policy.pool_counts packed = StaticPackedRhtQ468State( policy=policy, - int4_payload=torch.zeros( - (q4_count, geometry.value_width * 4 // 8), dtype=torch.uint8 - ), - int6_payload=torch.zeros( - (q6_count, geometry.value_width * 6 // 8), dtype=torch.uint8 - ), + int4_payload=torch.zeros((q4_count, geometry.value_width * 4 // 8), dtype=torch.uint8), + int6_payload=torch.zeros((q6_count, geometry.value_width * 6 // 8), dtype=torch.uint8), int8_payload=torch.zeros((q8_count, geometry.value_width), dtype=torch.int8), scales=torch.ones(geometry.total_rows, dtype=torch.float16), padding=torch.zeros(8, dtype=torch.uint8), @@ -788,9 +942,7 @@ def test_real_q468_policy_has_a_physical_exact_3454664_byte_state() -> None: evidence = verify_static_packed_rht_q468(packed) assert q4_count + q6_count + q8_count == 36_864 assert ( - packed.int4_payload.numel() - + packed.int6_payload.numel() - + packed.int8_payload.numel() + packed.int4_payload.numel() + packed.int6_payload.numel() + packed.int8_payload.numel() == 3_297_984 ) assert policy.packed_precision_codes.numel() == 9_216 diff --git a/tests/test_static_q468_cache.py b/tests/test_static_q468_cache.py index f8f0f05..fd62425 100644 --- a/tests/test_static_q468_cache.py +++ b/tests/test_static_q468_cache.py @@ -2,11 +2,22 @@ import copy from collections.abc import Iterator +from dataclasses import replace import pytest import torch from transformers import Qwen3_5ForCausalLM +from recurquant import ( + DYNAMIC_Q468_BASELINE_METHOD, + DYNAMIC_Q468_ORACLE_METHOD, + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + STATIC_Q468_MSE_METHOD, + STATIC_Q468_UNIFORM_Q4_METHOD, + STATIC_Q468_UNIFORM_Q8_METHOD, + create_qwen35_dynamic_q468_baseline_cache, + create_qwen35_dynamic_q468_oracle_cache, +) from recurquant.statelease_equal_byte_baselines import ( FROZEN_QWEN35_EQUAL_BYTE_LAYOUT, RHT_Q4_Q6_Q8, @@ -21,6 +32,8 @@ FROZEN_STATIC_Q48_PROMOTIONS, FROZEN_STATIC_Q468_ABLATION_STEPS, FROZEN_STATIC_Q468_PRIMARY_STEPS, + FROZEN_STATIC_Q468_UNIFORM_Q4_STEPS, + FROZEN_STATIC_Q468_UNIFORM_Q8_STEPS, STATIC_Q48_COMPARATOR_METHOD, STATIC_Q468_ABLATION_METHOD, STATIC_Q468_PRIMARY_METHOD, @@ -31,9 +44,7 @@ build_static_rht_q468_policy, ) from recurquant.static_q468_cache import ( - DYNAMIC_Q468_ORACLE_METHOD, StaticRhtQwen35Cache, - create_qwen35_dynamic_q468_oracle_cache, create_qwen35_static_rht_cache, ) from tests.test_transformers_cache import tiny_config @@ -454,7 +465,7 @@ def test_static_cache_rejects_policy_hash_geometry_and_public_method_drift() -> layout=mismatched_layout, ) - with pytest.raises(ValueError, match="three frozen methods"): + with pytest.raises(ValueError, match="seven frozen methods"): create_qwen35_static_rht_cache( tiny_config(), policy=policy, @@ -479,7 +490,7 @@ def _frozen_config(): @pytest.fixture(scope="module") -def frozen_policies() -> tuple[StaticRhtQ468Policy, StaticRhtQ468Policy, StaticRhtQ48Policy]: +def frozen_policies() -> tuple[StaticRhtQ468Policy | StaticRhtQ48Policy, ...]: rows = FROZEN_QWEN35_STATIC_Q468_GEOMETRY.total_rows order = torch.arange(rows, dtype=torch.float64).reshape(1, rows) d4 = 5.0 + order / (rows + 1) @@ -503,6 +514,42 @@ def frozen_policies() -> tuple[StaticRhtQ468Policy, StaticRhtQ468Policy, StaticR method_id=STATIC_Q468_ABLATION_METHOD, **BINDINGS, ) + mse = build_static_rht_q468_policy( + d4, + d6, + d8, + geometry=FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + marginal_steps=FROZEN_STATIC_Q468_PRIMARY_STEPS, + method_id=STATIC_Q468_MSE_METHOD, + **BINDINGS, + ) + fisher = build_static_rht_q468_policy( + d4, + d6, + d8, + geometry=FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + marginal_steps=FROZEN_STATIC_Q468_PRIMARY_STEPS, + method_id=STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD, + **BINDINGS, + ) + uniform_q4 = build_static_rht_q468_policy( + d4, + d6, + d8, + geometry=FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + marginal_steps=FROZEN_STATIC_Q468_UNIFORM_Q4_STEPS, + method_id=STATIC_Q468_UNIFORM_Q4_METHOD, + **BINDINGS, + ) + uniform_q8 = build_static_rht_q468_policy( + d4, + d6, + d8, + geometry=FROZEN_QWEN35_STATIC_Q468_GEOMETRY, + marginal_steps=FROZEN_STATIC_Q468_UNIFORM_Q8_STEPS, + method_id=STATIC_Q468_UNIFORM_Q8_METHOD, + **BINDINGS, + ) q48 = build_static_rht_q48_policy( d4, d8, @@ -511,19 +558,23 @@ def frozen_policies() -> tuple[StaticRhtQ468Policy, StaticRhtQ468Policy, StaticR method_id=STATIC_Q48_COMPARATOR_METHOD, **BINDINGS, ) - return primary, ablation, q48 + return primary, ablation, mse, fisher, uniform_q4, uniform_q8, q48 def test_public_factory_supports_all_reserved_methods_without_loading_weights( frozen_policies, ) -> None: - primary, ablation, q48 = frozen_policies + primary, ablation, mse, fisher, uniform_q4, uniform_q8, q48 = frozen_policies expected = { STATIC_Q468_PRIMARY_METHOD: (3_454_664, 0, True), STATIC_Q468_ABLATION_METHOD: (3_380_928, 73_736, False), + STATIC_Q468_MSE_METHOD: (3_454_664, 0, True), + STATIC_Q468_DIAG_EMPIRICAL_FISHER_H1_METHOD: (3_454_664, 0, True), + STATIC_Q468_UNIFORM_Q4_METHOD: (2_515_968, 938_696, False), + STATIC_Q468_UNIFORM_Q8_METHOD: (4_875_264, -1_420_600, False), STATIC_Q48_COMPARATOR_METHOD: (3_454_664, 0, True), } - for policy in (primary, ablation, q48): + for policy in (primary, ablation, mse, fisher, uniform_q4, uniform_q8, q48): cache = create_qwen35_static_rht_cache( _frozen_config(), policy=policy, @@ -544,16 +595,47 @@ def test_public_factory_supports_all_reserved_methods_without_loading_weights( if "policy" in name.lower() ) + unregistered = replace( + primary, + method_id="rht_q468_static_unregistered_k29334", + ) + with pytest.raises(ValueError, match="seven frozen methods"): + create_qwen35_static_rht_cache( + _frozen_config(), + policy=unregistered, + expected_policy_sha256=unregistered.policy_sha256, + ) + -def test_named_dynamic_q468_oracle_is_existing_exact_k27030_path() -> None: - cache = create_qwen35_dynamic_q468_oracle_cache(_frozen_config(), record_evidence=True) +def test_named_dynamic_q468_baseline_is_existing_exact_k27030_path() -> None: + cache = create_qwen35_dynamic_q468_baseline_cache( + _frozen_config(), + record_evidence=True, + ) assert cache.codec == RHT_Q4_Q6_Q8 - assert cache.method_id == DYNAMIC_Q468_ORACLE_METHOD + assert cache.method_id == DYNAMIC_Q468_BASELINE_METHOD assert cache.layout is FROZEN_QWEN35_EQUAL_BYTE_LAYOUT assert cache.layout.multibit_marginal_steps == 27_030 assert cache.storage_summary()["expected_resident_bytes"] == 3_454_664 +def test_dynamic_q468_oracle_names_are_compatibility_aliases() -> None: + canonical = create_qwen35_dynamic_q468_baseline_cache( + _frozen_config(), + record_evidence=True, + ) + compatibility = create_qwen35_dynamic_q468_oracle_cache( + _frozen_config(), + record_evidence=True, + ) + assert DYNAMIC_Q468_ORACLE_METHOD == DYNAMIC_Q468_BASELINE_METHOD + assert compatibility.method_id == canonical.method_id + assert compatibility.codec == canonical.codec + assert compatibility.layout is canonical.layout + assert compatibility.record_evidence is canonical.record_evidence + assert compatibility.storage_summary() == canonical.storage_summary() + + def test_public_factories_validate_model_runtime_before_cache_construction( frozen_policies, ) -> None: @@ -569,7 +651,7 @@ def test_public_factories_validate_model_runtime_before_cache_construction( expected_policy_sha256=primary.policy_sha256, ) with pytest.raises(ValueError, match="inference-only"): - create_qwen35_dynamic_q468_oracle_cache(training_model) + create_qwen35_dynamic_q468_baseline_cache(training_model) @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is unavailable") diff --git a/tests/test_static_q468_calibration.py b/tests/test_static_q468_calibration.py index fa22ae3..472f88d 100644 --- a/tests/test_static_q468_calibration.py +++ b/tests/test_static_q468_calibration.py @@ -7,13 +7,17 @@ import math import sys import unicodedata +from dataclasses import replace from pathlib import Path import pytest import torch +import recurquant.static_q468_calibration as calibration from recurquant.evidence import canonical_json_bytes from recurquant.multibit_policy import allocate_exact_multibit_codes +from recurquant.quantization import QuantizationSpec, quantize_dequantize +from recurquant.rht import RHT_SEED, right_rht_encode from recurquant.static_q468 import ( FROZEN_QWEN35_STATIC_Q468_GEOMETRY, FROZEN_STATIC_Q468_ABLATION_STEPS, @@ -25,32 +29,52 @@ CALIBRATION_SCORE_ARTIFACT_KIND, CALIBRATION_SCORE_ARTIFACT_PROFILE, CALIBRATION_SCORE_ARTIFACT_REVISION, + COMPARATOR_SCORE_ARTIFACT_KIND, + COMPARATOR_SCORE_ARTIFACT_PROFILE, + COMPARATOR_SCORE_ARTIFACT_REVISION, + FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE, FROZEN_SOURCE_TENSOR_CONTRACT, + FROZEN_UNWEIGHTED_MSE_PROFILE, GENERIC_CALIBRATION_SCORE_ARTIFACT_KIND, GENERIC_CALIBRATION_SCORE_ARTIFACT_PROFILE, GENERIC_CALIBRATION_SCORE_ARTIFACT_REVISION, AnchorDistortionBatch, CalibrationAggregate, CalibrationSequenceScores, + ComparatorAggregate, + ComparatorSequenceScores, + FrozenComparatorEndpointBatch, + UnweightedEndpointBatch, aggregate_calibration_scores, + aggregate_comparator_scores, allocate_frozen_static_q468_code_maps, allocate_static_q468_code_map, + allocate_unweighted_endpoint_policy, balanced_sha_rank_halves, build_calibration_score_artifact, build_frozen_calibration_score_artifact, + build_frozen_comparator_score_artifact, build_frozen_split_half_stability_artifact, calibration_sequence_rank_sha256, + compute_rht_diagonal_empirical_fisher_h1_endpoints, + compute_rht_unweighted_mse_endpoints, deserialize_calibration_score_artifact, + deserialize_comparator_score_artifact, deserialize_frozen_split_half_stability_artifact, evaluate_policy_stability, + fisher_h1_boundary_positions, fit_split_half_policy, frozen_anchor_positions, + identity_record_sha256, per_layer_mean_bitwidth_shifts, q8_set_jaccard, reduce_anchor_distortions, reduce_frozen_anchor_distortions, + reduce_frozen_comparator_endpoints, + reduce_unweighted_endpoint_anchors, static_q468_code_map_sha256, verify_calibration_score_artifact, + verify_comparator_score_artifact, verify_frozen_split_half_stability_artifact, ) @@ -238,6 +262,164 @@ def _relabel_generic_document_as_official(raw: bytes) -> dict[str, object]: return document +def _comparator_sequence( + selector_profile: str, + family: str, + config: str, + canonical_id: str, + scalar: float, + *, + ruler_category: str | None = None, + seed: int | None = None, + configured_length: int | None = None, +) -> ComparatorSequenceScores: + token_count = 3 + endpoint_positions = ( + frozen_anchor_positions(token_count) + if selector_profile == FROZEN_UNWEIGHTED_MSE_PROFILE + else fisher_h1_boundary_positions(token_count) + ) + token_hash = calibration.sequence_token_ids_sha256((11, 12, 13)) + token_span = ( + ("prefill_start", 0), + ("prefill_stop", 1), + ("scored_start", 1), + ("scored_stop", 3), + ("cache_exposed_start", 3), + ("cache_exposed_stop", 3), + ) + identity_anchor_hash = calibration.identity_anchor_manifest_sha256( + canonical_id=canonical_id, + sequence_length=token_count, + sequence_token_ids_sha256_value=token_hash, + token_span=token_span, + ) + identity_record_hash = hashlib.sha256(f"record:{canonical_id}".encode()).hexdigest() + fisher_boundary_hash = hashlib.sha256(f"fisher:{canonical_id}".encode()).hexdigest() + position_payload = calibration._comparator_position_payload( + selector_profile=selector_profile, + token_count=token_count, + endpoint_positions=endpoint_positions, + sequence_token_ids_sha256_value=token_hash, + identity_anchor_manifest_sha256_value=identity_anchor_hash, + identity_record_sha256_value=identity_record_hash, + fisher_boundary_sha256=fisher_boundary_hash, + ) + position_hash = calibration._domain_json_sha256( + calibration._COMPARATOR_POSITION_HASH_DOMAIN, + position_payload, + ) + endpoint_hash = hashlib.sha256( + f"endpoint:{selector_profile}:{canonical_id}".encode() + ).hexdigest() + rows = FROZEN_QWEN35_STATIC_Q468_GEOMETRY.total_rows + d4 = torch.full((rows,), scalar, dtype=torch.float64) + d6 = torch.full((rows,), scalar / 2, dtype=torch.float64) + d8 = torch.full((rows,), scalar / 4, dtype=torch.float64) + sequence_hash = calibration._comparator_sequence_score_sha256( + selector_profile=selector_profile, + position_manifest_sha256=position_hash, + endpoint_inputs_sha256=endpoint_hash, + identity_record_sha256_value=identity_record_hash, + d4=d4, + d6=d6, + d8=d8, + ) + return ComparatorSequenceScores( + selector_profile=selector_profile, + family=family, + config=config, + ruler_category=ruler_category, + canonical_id=canonical_id, + seed=seed, + configured_length=configured_length, + token_count=token_count, + endpoint_positions=endpoint_positions, + position_manifest_sha256=position_hash, + endpoint_inputs_sha256=endpoint_hash, + sequence_scores_sha256=sequence_hash, + d4=d4, + d6=d6, + d8=d8, + source_shape=( + len(endpoint_positions), + *FROZEN_SOURCE_TENSOR_CONTRACT.trailing_shape, + ), + sequence_token_ids_sha256=token_hash, + token_span=token_span, + identity_anchor_manifest_sha256=identity_anchor_hash, + identity_record_sha256=identity_record_hash, + fisher_boundary_sha256=fisher_boundary_hash, + target_nlls_sha256=( + None + if selector_profile == FROZEN_UNWEIGHTED_MSE_PROFILE + else hashlib.sha256(f"nll:{canonical_id}".encode()).hexdigest() + ), + ) + + +def _comparator_sequences(selector_profile: str) -> list[ComparatorSequenceScores]: + specifications = [ + ("mbpp", None, "default", 3.0), + ("mbpp", None, "default", 9.0), + ("pg19", None, "default", 12.0), + ("pg19", None, "default", 18.0), + ("ruler", "retrieval", "niah", 3.0), + ("ruler", "retrieval", "niah", 9.0), + ("ruler", "multi_hop_tracing", "vt", 6.0), + ("ruler", "multi_hop_tracing", "vt", 12.0), + ("ruler", "aggregation", "cwe", 9.0), + ("ruler", "aggregation", "fwe", 15.0), + ("ruler", "question_answering", "qa", 12.0), + ("ruler", "question_answering", "qa", 18.0), + ] + return [ + _comparator_sequence( + selector_profile, + family, + config, + f"{family}-{category}-{index}", + scalar, + ruler_category=category, + seed=index if family == "ruler" else None, + configured_length=2_048 if family == "ruler" else None, + ) + for index, (family, category, config, scalar) in enumerate(specifications) + ] + + +def _synthetic_comparator_aggregate( + selector_profile: str, + *, + identity_manifest_sha256: str = "e" * 64, +) -> ComparatorAggregate: + rows = FROZEN_QWEN35_STATIC_Q468_GEOMETRY.total_rows + row_axis = torch.arange(rows, dtype=torch.float64) / rows + profile_offset = 0.0 if selector_profile == FROZEN_UNWEIGHTED_MSE_PROFILE else 0.125 + provisional = ComparatorAggregate( + selector_profile=selector_profile, + d4=4.0 + profile_offset + row_axis, + d6=2.0 + profile_offset + row_axis / 2, + d8=1.0 + profile_offset + row_axis / 4, + family_sequence_counts=(("mbpp", 128), ("pg19", 16), ("ruler", 16)), + ruler_category_sequence_counts=tuple( + (category, 4) for category in calibration.RULER_CATEGORY_ORDER + ), + position_manifest_sha256=hashlib.sha256( + f"positions:{selector_profile}".encode() + ).hexdigest(), + sequence_score_manifest_sha256=hashlib.sha256( + f"sequences:{selector_profile}".encode() + ).hexdigest(), + identity_record_manifest_sha256=identity_manifest_sha256, + aggregate_scores_sha256="0" * 64, + ) + return replace( + provisional, + aggregate_scores_sha256=calibration._comparator_aggregate_score_sha256(provisional), + ) + + @pytest.mark.parametrize( ("token_count", "expected"), [ @@ -272,6 +454,376 @@ def test_frozen_anchor_equation(token_count: int, expected: tuple[int, ...]) -> assert frozen_anchor_positions(token_count) == expected +def test_fisher_h1_boundary_positions_reserve_causal_input_and_target() -> None: + assert fisher_h1_boundary_positions(3) == (0,) + assert fisher_h1_boundary_positions(7) == (0, 1, 2, 3, 4) + assert fisher_h1_boundary_positions(18) == tuple(range(16)) + assert fisher_h1_boundary_positions(19) == frozen_anchor_positions(17) + with pytest.raises(ValueError, match="at least three tokens"): + fisher_h1_boundary_positions(2) + + +def _tiny_endpoint_state() -> torch.Tensor: + return torch.tensor( + [ + -1.17, + -0.83, + -0.21, + 0.47, + 0.18, + 0.71, + 1.09, + 1.63, + -0.94, + -0.37, + 0.26, + 0.88, + 0.33, + 0.69, + 1.31, + 1.91, + ], + dtype=torch.float32, + ).reshape(2, 1, 2, 4) + + +def _tiny_endpoint_specs() -> tuple[QuantizationSpec, ...]: + return tuple( + QuantizationSpec( + bits=bits, + group_size=TINY_GEOMETRY.value_width, + scale_bits=16, + flatten_last_dims=1, + rounding="nearest", + seed=RHT_SEED, + ) + for bits in (4, 6, 8) + ) + + +def test_unweighted_mse_endpoint_math_matches_exact_rht_q468_codec() -> None: + state = _tiny_endpoint_state() + actual = compute_rht_unweighted_mse_endpoints(state, geometry=TINY_GEOMETRY) + expected: list[list[torch.Tensor]] = [[], [], []] + fp32_reductions: list[list[torch.Tensor]] = [[], [], []] + for local_index, layer_index in enumerate(TINY_GEOMETRY.layer_indices): + encoded = right_rht_encode( + state[local_index].unsqueeze(0), + layer_index=layer_index, + expected_heads=TINY_GEOMETRY.heads, + ) + for destination, fp32_destination, specification in zip( + expected, + fp32_reductions, + _tiny_endpoint_specs(), + strict=True, + ): + restored = quantize_dequantize(encoded, specification).tensor + error = restored - encoded + destination.append(error.to(torch.float64).square().mean(dim=-1).squeeze(0)) + fp32_destination.append(error.square().mean(dim=-1).squeeze(0).to(torch.float64)) + for observed, rows in zip(actual, expected, strict=True): + reference = torch.stack(rows) + assert observed.device.type == "cpu" + assert observed.dtype == torch.float64 + assert torch.all(observed >= 0).item() + torch.testing.assert_close(observed, reference, rtol=0.0, atol=0.0) + assert any( + not torch.equal(observed, torch.stack(legacy_rows)) + for observed, legacy_rows in zip(actual, fp32_reductions, strict=True) + ) + + +def test_diagonal_empirical_fisher_h1_math_transforms_state_and_gradient() -> None: + state = _tiny_endpoint_state() + gradient = torch.tensor( + [ + 0.11, + -0.29, + 0.53, + -0.71, + 0.07, + 0.19, + -0.41, + 0.89, + -0.13, + 0.31, + -0.61, + 0.79, + 0.17, + -0.23, + 0.47, + -0.97, + ], + dtype=torch.float32, + ).reshape_as(state) + actual = compute_rht_diagonal_empirical_fisher_h1_endpoints( + state, + gradient, + geometry=TINY_GEOMETRY, + ) + expected: list[list[torch.Tensor]] = [[], [], []] + fp32_reductions: list[list[torch.Tensor]] = [[], [], []] + for local_index, layer_index in enumerate(TINY_GEOMETRY.layer_indices): + encoded_state = right_rht_encode( + state[local_index].unsqueeze(0), + layer_index=layer_index, + expected_heads=TINY_GEOMETRY.heads, + ) + encoded_gradient = right_rht_encode( + gradient[local_index].unsqueeze(0), + layer_index=layer_index, + expected_heads=TINY_GEOMETRY.heads, + ) + for destination, fp32_destination, specification in zip( + expected, + fp32_reductions, + _tiny_endpoint_specs(), + strict=True, + ): + restored = quantize_dequantize(encoded_state, specification).tensor + error = restored - encoded_state + destination.append( + ( + 0.5 + * ( + encoded_gradient.to(torch.float64).square() + * error.to(torch.float64).square() + ).sum(dim=-1) + ).squeeze(0) + ) + fp32_destination.append( + (0.5 * (encoded_gradient.square() * error.square()).sum(dim=-1)) + .squeeze(0) + .to(torch.float64) + ) + for observed, rows in zip(actual, expected, strict=True): + reference = torch.stack(rows) + assert observed.device.type == "cpu" + assert observed.dtype == torch.float64 + assert torch.all(observed >= 0).item() + torch.testing.assert_close(observed, reference, rtol=0.0, atol=0.0) + assert any( + not torch.equal(observed, torch.stack(legacy_rows)) + for observed, legacy_rows in zip(actual, fp32_reductions, strict=True) + ) + + +def test_unweighted_endpoint_reduction_and_exact_policy_ignore_query_proxies() -> None: + positions = frozen_anchor_positions(3) + q4 = torch.tensor( + [ + [[4.0, 3.0, 2.0, 1.0]], + [[8.0, 6.0, 4.0, 2.0]], + [[12.0, 9.0, 6.0, 3.0]], + ], + dtype=torch.float64, + ) + batch = UnweightedEndpointBatch( + selector_profile=FROZEN_UNWEIGHTED_MSE_PROFILE, + token_count=3, + anchor_positions=positions, + q4_scores=q4, + q6_scores=q4 / 2, + q8_scores=q4 / 4, + ) + + reduced = reduce_unweighted_endpoint_anchors(batch) + torch.testing.assert_close(reduced[0], q4.mean(dim=0).reshape(-1)) + torch.testing.assert_close(reduced[1], (q4 / 2).mean(dim=0).reshape(-1)) + torch.testing.assert_close(reduced[2], (q4 / 4).mean(dim=0).reshape(-1)) + first = allocate_unweighted_endpoint_policy(batch, marginal_steps=3) + second = allocate_unweighted_endpoint_policy(batch, marginal_steps=3) + assert torch.equal(first, second) + assert first.dtype == torch.uint8 + assert int(first.to(torch.int64).sum().item()) == 3 + + fisher_batch = UnweightedEndpointBatch( + selector_profile=FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE, + token_count=5, + anchor_positions=fisher_h1_boundary_positions(5), + q4_scores=q4, + q6_scores=q4 / 2, + q8_scores=q4 / 4, + ) + assert reduce_unweighted_endpoint_anchors(fisher_batch)[0].shape == (4,) + + +def _identity_v5_record( + *, + canonical_id: str, + sequence_token_ids: tuple[int, ...], +) -> dict[str, object]: + tokenizer_manifest_sha256 = "a" * 64 + captured_record = capture._base_record( + phase="calibration", + family="mbpp", + canonical_id=canonical_id, + config="full", + seed=None, + configured_length=None, + ruler_category=None, + generator_receipt_sha256=None, + source_payload={"task_id": canonical_id}, + formatted_payload={"prompt": "fixture"}, + prompt_ids=sequence_token_ids, + target_ids=(), + tokenizer_manifest_sha256=tokenizer_manifest_sha256, + ) + captured_record = capture._assign_sha_ranks([captured_record])[0] + return capture.resolver._normalize_record( + captured_record, + index=0, + phase="calibration", + tokenizer_hash=tokenizer_manifest_sha256, + ) + + +def test_frozen_comparator_reduction_binds_v5_positions_inputs_scores_and_nll_hash() -> None: + token_ids = (17, 18, 19) + record = _identity_v5_record(canonical_id="comparator-1", sequence_token_ids=token_ids) + trailing = FROZEN_SOURCE_TENSOR_CONTRACT.trailing_shape + mse_positions = frozen_anchor_positions(len(token_ids)) + mse_values = torch.arange( + 1, + len(mse_positions) * math.prod(trailing) + 1, + dtype=torch.float64, + ).reshape(len(mse_positions), *trailing) + mse = reduce_frozen_comparator_endpoints( + FrozenComparatorEndpointBatch( + selector_profile=FROZEN_UNWEIGHTED_MSE_PROFILE, + family="mbpp", + config="full", + ruler_category=None, + canonical_id="comparator-1", + seed=None, + configured_length=None, + token_count=len(token_ids), + endpoint_positions=mse_positions, + q4_scores=mse_values, + q6_scores=mse_values / 2, + q8_scores=mse_values / 4, + sequence_token_ids=token_ids, + identity_record=record, + ) + ) + permuted_mse = reduce_frozen_comparator_endpoints( + FrozenComparatorEndpointBatch( + selector_profile=FROZEN_UNWEIGHTED_MSE_PROFILE, + family="mbpp", + config="full", + ruler_category=None, + canonical_id="comparator-1", + seed=None, + configured_length=None, + token_count=len(token_ids), + endpoint_positions=mse_positions, + q4_scores=mse_values.flip(0), + q6_scores=(mse_values / 2).flip(0), + q8_scores=(mse_values / 4).flip(0), + sequence_token_ids=token_ids, + identity_record=record, + ) + ) + fisher_positions = fisher_h1_boundary_positions(len(token_ids)) + fisher_values = mse_values[:1] / 10 + fisher = reduce_frozen_comparator_endpoints( + FrozenComparatorEndpointBatch( + selector_profile=FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE, + family="mbpp", + config="full", + ruler_category=None, + canonical_id="comparator-1", + seed=None, + configured_length=None, + token_count=len(token_ids), + endpoint_positions=fisher_positions, + q4_scores=fisher_values, + q6_scores=fisher_values / 2, + q8_scores=fisher_values / 4, + sequence_token_ids=token_ids, + identity_record=record, + target_nlls=torch.tensor([1.25], dtype=torch.float64), + ) + ) + + torch.testing.assert_close(mse.d4, mse_values.mean(dim=0).reshape(-1)) + torch.testing.assert_close(mse.d4, permuted_mse.d4) + torch.testing.assert_close(fisher.d4, fisher_values.reshape(-1)) + assert mse.endpoint_positions == (0, 1, 2) + assert fisher.endpoint_positions == (0,) + assert mse.position_manifest_sha256 != fisher.position_manifest_sha256 + assert mse.endpoint_inputs_sha256 != fisher.endpoint_inputs_sha256 + assert mse.sequence_scores_sha256 != fisher.sequence_scores_sha256 + assert mse.position_manifest_sha256 == permuted_mse.position_manifest_sha256 + assert mse.endpoint_inputs_sha256 != permuted_mse.endpoint_inputs_sha256 + assert mse.sequence_scores_sha256 != permuted_mse.sequence_scores_sha256 + assert mse.target_nlls_sha256 is None + assert ( + fisher.target_nlls_sha256 + == hashlib.sha256( + calibration._COMPARATOR_TARGET_NLL_HASH_DOMAIN + + calibration._tensor_bytes(torch.tensor([1.25], dtype=torch.float64)) + ).hexdigest() + ) + assert "target_nlls" not in fisher.manifest_record() + assert fisher.manifest_record()["target_nlls_sha256"] == fisher.target_nlls_sha256 + + +def test_frozen_comparator_reduction_rejects_v4_substitution_offbyone_and_nonfinite() -> None: + token_ids = (31, 32, 33) + record = _identity_v5_record(canonical_id="comparator-2", sequence_token_ids=token_ids) + positions = fisher_h1_boundary_positions(len(token_ids)) + shape = (len(positions), *FROZEN_SOURCE_TENSOR_CONTRACT.trailing_shape) + values = torch.ones(shape, dtype=torch.float64) + baseline = FrozenComparatorEndpointBatch( + selector_profile=FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE, + family="mbpp", + config="full", + ruler_category=None, + canonical_id="comparator-2", + seed=None, + configured_length=None, + token_count=len(token_ids), + endpoint_positions=positions, + q4_scores=values, + q6_scores=values / 2, + q8_scores=values / 4, + sequence_token_ids=token_ids, + identity_record=record, + target_nlls=torch.tensor([1.25], dtype=torch.float64), + ) + + with pytest.raises(ValueError, match="requires target_nlls"): + reduce_frozen_comparator_endpoints(replace(baseline, target_nlls=None)) + with pytest.raises(ValueError, match=r"A\(T\)/B\(T\)"): + reduce_frozen_comparator_endpoints(replace(baseline, endpoint_positions=(1,))) + with pytest.raises(ValueError, match="token-ID SHA-256"): + reduce_frozen_comparator_endpoints(replace(baseline, sequence_token_ids=(31, 99, 33))) + legacy = dict(record) + legacy.pop("fisher_boundary") + with pytest.raises(ValueError, match="missing=.*fisher_boundary"): + reduce_frozen_comparator_endpoints(replace(baseline, identity_record=legacy)) + for invalid in (math.nan, math.inf): + nonfinite = values.clone() + nonfinite[0, 0, 0, 0] = invalid + with pytest.raises(ValueError, match="finite"): + reduce_frozen_comparator_endpoints(replace(baseline, q4_scores=nonfinite)) + + +def test_fisher_sequence_artifact_requires_exact_target_nll_receipt() -> None: + fisher = _comparator_sequence( + FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE, + "pg19", + "default", + "fisher-without-nll-receipt", + 1.0, + ) + + with pytest.raises(ValueError, match="requires a target NLL receipt"): + calibration.aggregate_comparator_scores([replace(fisher, target_nlls_sha256=None)]) + + @pytest.mark.parametrize("bad", [0, -1, True, 1.5]) def test_frozen_anchor_equation_rejects_empty_or_non_integer_counts(bad: object) -> None: with pytest.raises((TypeError, ValueError)): @@ -477,9 +1029,9 @@ def make(dtype: torch.dtype, shape: tuple[int, ...]) -> AnchorDistortionBatch: def test_frozen_reduction_requires_exact_layer_head_key_row_cpu_fp64_shape() -> None: - shape = (1, *FROZEN_SOURCE_TENSOR_CONTRACT.trailing_shape) + shape = (3, *FROZEN_SOURCE_TENSOR_CONTRACT.trailing_shape) values = torch.ones(shape, dtype=torch.float64) - sequence_token_ids = (17,) + sequence_token_ids = (17, 18, 19) tokenizer_manifest_sha256 = "a" * 64 captured_record = capture._base_record( phase="calibration", @@ -510,8 +1062,8 @@ def test_frozen_reduction_requires_exact_layer_head_key_row_cpu_fp64_shape() -> canonical_id="601", seed=None, configured_length=None, - token_count=1, - anchor_positions=(0,), + token_count=3, + anchor_positions=(0, 1, 2), query_energy=values, q4_mse=values, q6_mse=values / 2, @@ -528,6 +1080,29 @@ def test_frozen_reduction_requires_exact_layer_head_key_row_cpu_fp64_shape() -> assert result.identity_anchor_manifest_sha256 == captured_record["anchor_manifest_sha256"] assert result.identity_record_sha256 == captured_record["identity_record_sha256"] + def with_identity_record(record: dict[str, object]) -> AnchorDistortionBatch: + fields = {name: getattr(batch, name) for name in AnchorDistortionBatch.__dataclass_fields__} + fields["identity_record"] = record + return AnchorDistortionBatch(**fields) + + legacy_record = dict(identity_record) + legacy_record.pop("fisher_boundary") + with pytest.raises(ValueError, match="missing=.*fisher_boundary"): + reduce_frozen_anchor_distortions(with_identity_record(legacy_record)) + + tampered_record = json.loads(json.dumps(identity_record)) + tampered_boundary = tampered_record["fisher_boundary"] + assert isinstance(tampered_boundary, dict) + tampered_boundary["input_token_ids_sha256"] = "0" * 64 + tampered_boundary["fisher_boundary_sha256"] = capture.resolver.fisher_boundary_sha256( + tampered_boundary + ) + original_record_hash = tampered_record["identity_record_sha256"] + tampered_record["identity_record_sha256"] = identity_record_sha256(tampered_record) + assert tampered_record["identity_record_sha256"] != original_record_hash + with pytest.raises(ValueError, match="input token-ID hash"): + reduce_frozen_anchor_distortions(with_identity_record(tampered_record)) + with pytest.raises(ValueError, match="frozen source shape"): reduce_frozen_anchor_distortions( _batch( @@ -579,6 +1154,54 @@ def test_family_aggregation_uses_equal_broad_and_ruler_category_weights() -> Non } == {"niah_single_1", "niah_multikey_1"} +def test_comparator_aggregation_reuses_exact_family_macro_and_domain_hashes() -> None: + sequences = _comparator_sequences(FROZEN_UNWEIGHTED_MSE_PROFILE) + aggregate = aggregate_comparator_scores(sequences) + expected = torch.full( + (FROZEN_QWEN35_STATIC_Q468_GEOMETRY.total_rows,), + 10.5, + dtype=torch.float64, + ) + + torch.testing.assert_close(aggregate.d4, expected) + torch.testing.assert_close(aggregate.d6, expected / 2) + torch.testing.assert_close(aggregate.d8, expected / 4) + assert aggregate.family_sequence_counts == (("mbpp", 2), ("pg19", 2), ("ruler", 8)) + assert aggregate.ruler_category_sequence_counts == tuple( + (category, 2) for category in calibration.RULER_CATEGORY_ORDER + ) + reverse = aggregate_comparator_scores(list(reversed(sequences))) + assert torch.equal(aggregate.d4, reverse.d4) + assert aggregate.position_manifest_sha256 == reverse.position_manifest_sha256 + assert aggregate.sequence_score_manifest_sha256 == reverse.sequence_score_manifest_sha256 + assert aggregate.aggregate_scores_sha256 == reverse.aggregate_scores_sha256 + + substituted = list(sequences) + original = substituted[0] + replacement = replace(original, endpoint_inputs_sha256="9" * 64) + replacement = replace( + replacement, + sequence_scores_sha256=calibration._comparator_sequence_score_sha256( + selector_profile=replacement.selector_profile, + position_manifest_sha256=replacement.position_manifest_sha256, + endpoint_inputs_sha256=replacement.endpoint_inputs_sha256, + identity_record_sha256_value=replacement.identity_record_sha256, + d4=replacement.d4, + d6=replacement.d6, + d8=replacement.d8, + ), + ) + substituted[0] = replacement + changed = aggregate_comparator_scores(substituted) + assert changed.sequence_score_manifest_sha256 != aggregate.sequence_score_manifest_sha256 + assert changed.aggregate_scores_sha256 != aggregate.aggregate_scores_sha256 + assert changed.position_manifest_sha256 == aggregate.position_manifest_sha256 + + drifted_position = replace(original, endpoint_positions=(1, 2, 3)) + with pytest.raises(ValueError, match="positions drifted"): + aggregate_comparator_scores([drifted_position, *sequences[1:]]) + + def test_aggregation_and_split_are_invariant_to_input_order() -> None: sequences = _calibration_sequences() forward = aggregate_calibration_scores(sequences) @@ -940,6 +1563,109 @@ def test_official_frozen_artifact_round_trips_only_with_exact_profile() -> None: assert verify_calibration_score_artifact(raw)["valid"] is True +def test_comparator_score_artifact_round_trips_both_exact_k29334_profiles() -> None: + identity_sha256 = "1" * 64 + raw = build_frozen_comparator_score_artifact( + _synthetic_comparator_aggregate(FROZEN_UNWEIGHTED_MSE_PROFILE), + _synthetic_comparator_aggregate(FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE), + calibration_identity_sha256=identity_sha256, + ) + decoded = deserialize_comparator_score_artifact( + raw, + expected_calibration_identity_sha256=identity_sha256, + ) + document = json.loads(raw) + + assert document["artifact_kind"] == COMPARATOR_SCORE_ARTIFACT_KIND + assert document["schema_version"] == 1 + assert document["evidence"]["artifact_profile"] == COMPARATOR_SCORE_ARTIFACT_PROFILE + assert document["evidence"]["artifact_revision"] == COMPARATOR_SCORE_ARTIFACT_REVISION + assert "target_nll" not in raw.decode("utf-8") + assert "token_ids" not in raw.decode("utf-8") + assert list(decoded.selectors) == [ + FROZEN_UNWEIGHTED_MSE_PROFILE, + FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE, + ] + assert decoded.calibration_identity_sha256 == identity_sha256 + assert decoded.file_sha256 == hashlib.sha256(raw).hexdigest() + for method, selector in decoded.selectors.items(): + assert selector.method_id == method + assert selector.marginal_steps == FROZEN_STATIC_Q468_PRIMARY_STEPS + assert selector.precision_codes.dtype == torch.uint8 + assert selector.precision_codes.device.type == "cpu" + assert int(selector.precision_codes.to(torch.int64).sum().item()) == ( + FROZEN_STATIC_Q468_PRIMARY_STEPS + ) + assert selector.calibration_scores_sha256 == (selector.aggregate.aggregate_scores_sha256) + assert selector.position_manifest_sha256 == (selector.aggregate.position_manifest_sha256) + assert ( + verify_comparator_score_artifact( + raw, + expected_calibration_identity_sha256=identity_sha256, + )["valid"] + is True + ) + + +def test_comparator_score_artifact_rejects_profile_hash_array_and_allocation_tampering() -> None: + raw = build_frozen_comparator_score_artifact( + _synthetic_comparator_aggregate(FROZEN_UNWEIGHTED_MSE_PROFILE), + _synthetic_comparator_aggregate(FROZEN_DIAGONAL_EMPIRICAL_FISHER_H1_PROFILE), + calibration_identity_sha256="1" * 64, + ) + + missing = json.loads(raw) + missing["evidence"]["selectors"].pop() + with pytest.raises(ValueError, match="exactly two selectors"): + deserialize_comparator_score_artifact(_rehashed_document(missing)) + + extra = json.loads(raw) + extra["evidence"]["selectors"].append(extra["evidence"]["selectors"][0]) + with pytest.raises(ValueError, match="exactly two selectors"): + deserialize_comparator_score_artifact(_rehashed_document(extra)) + + reordered = json.loads(raw) + reordered["evidence"]["selectors"].reverse() + with pytest.raises(ValueError, match="exactly MSE then"): + deserialize_comparator_score_artifact(_rehashed_document(reordered)) + + score_tamper = json.loads(raw) + encoded = score_tamper["evidence"]["selectors"][0]["scores"]["data_base64"] + score_bytes = bytearray(base64.b64decode(encoded)) + score_bytes[0] ^= 1 + score_tamper["evidence"]["selectors"][0]["scores"]["data_base64"] = base64.b64encode( + score_bytes + ).decode("ascii") + with pytest.raises(ValueError, match="aggregate-score SHA-256 drifted"): + deserialize_comparator_score_artifact(_rehashed_document(score_tamper)) + + nonfinite = json.loads(raw) + encoded = nonfinite["evidence"]["selectors"][0]["scores"]["data_base64"] + score_bytes = bytearray(base64.b64decode(encoded)) + score_bytes[:8] = torch.tensor(float("nan"), dtype=torch.float64).numpy().tobytes() + nonfinite["evidence"]["selectors"][0]["scores"]["data_base64"] = base64.b64encode( + score_bytes + ).decode("ascii") + with pytest.raises(ValueError, match="finite and non-negative"): + deserialize_comparator_score_artifact(_rehashed_document(nonfinite)) + + position_tamper = json.loads(raw) + position_tamper["evidence"]["selectors"][0]["position_manifest_sha256"] = "9" * 64 + with pytest.raises(ValueError, match="aggregate-score SHA-256 drifted"): + deserialize_comparator_score_artifact(_rehashed_document(position_tamper)) + + allocation_tamper = json.loads(raw) + allocation_tamper["evidence"]["selectors"][0]["allocation"]["code_map_sha256"] = "9" * 64 + with pytest.raises(ValueError, match="code-map SHA-256 drifted"): + deserialize_comparator_score_artifact(_rehashed_document(allocation_tamper)) + + with pytest.raises(ValueError, match="differs from expected identity"): + deserialize_comparator_score_artifact( + raw, + expected_calibration_identity_sha256="2" * 64, + ) + + def test_generic_artifact_cannot_be_relabelled_as_official_geometry_counts_or_budget() -> None: raw = build_calibration_score_artifact( _tiny_aggregate(), From ff3f86ee7cd457ab32a3e57de3d0191a50c3de7d Mon Sep 17 00:00:00 2001 From: HiddenTrojan Date: Fri, 14 Aug 2026 22:28:01 +0800 Subject: [PATCH 14/24] Stabilize CI and package metadata --- .github/workflows/ci.yml | 12 ++++++------ pyproject.toml | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89fd288..80ba141 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,24 +31,24 @@ jobs: python-version: "3.11" version: "0.11.31" - name: Install - run: uv pip install --system -e ".[dev,eval]" + run: uv sync --frozen --all-extras - name: Verify frozen MBPP confirmation run: | - recurquant verify-confirmation \ + uv run --frozen recurquant verify-confirmation \ evidence/mbpp-v02-confirmation.json \ evidence/mbpp-v02-confirmation-manifest.json \ --expect-artifact-sha256 70394c419298fc872cdd08e8aec12d17d5a56aa20f7d3c9f09fe8fdbf26c6ba9 \ --expect-artifact-evidence-sha256 2a652df92f99fa81f785244d966829e909d31f200e5a1520b76e6b46fb45d3e0 - name: Verify frozen StateLease Stage-A record run: >- - recurquant verify-statelease-stage-a + uv run --frozen recurquant verify-statelease-stage-a evidence/experiment012-statelease-stage-a-666.json - name: Lint - run: ruff check . + run: uv run --frozen ruff check . - name: Test - run: pytest + run: uv run --frozen pytest - name: Check generated README assets - run: python scripts/generate_readme_assets.py --check + run: uv run --frozen python scripts/generate_readme_assets.py --check package: name: Build wheel and sdist diff --git a/pyproject.toml b/pyproject.toml index c28efd3..d445ad5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["hatchling>=1.27"] +requires = ["hatchling==1.31.0"] build-backend = "hatchling.build" [project] @@ -63,7 +63,7 @@ recurquant = "recurquant.cli:main" packages = ["src/recurquant"] [tool.pytest.ini_options] -addopts = "-q --basetemp ./.pytest-ci" +addopts = "-q --basetemp ../.pytest-recurquant" testpaths = ["tests"] [tool.ruff] From 19ef835a8ec2341c36657b7e010ad1ae6135de9a Mon Sep 17 00:00:00 2001 From: HiddenTrojan Date: Fri, 14 Aug 2026 22:38:09 +0800 Subject: [PATCH 15/24] Track the frozen CI environment --- uv.lock | 2318 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2318 insertions(+) create mode 100644 uv.lock diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..9ef3168 --- /dev/null +++ b/uv.lock @@ -0,0 +1,2318 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.12' and sys_platform == 'win32'", + "python_full_version < '3.12' and sys_platform == 'emscripten'", + "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] + +[[package]] +name = "aiohappyeyeballs" +version = "2.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/f4/eec0465c2f67b2664688d0240b3212d5196fd89e741df67ddb81f8d35658/aiohappyeyeballs-2.7.1.tar.gz", hash = "sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d", size = 24757, upload-time = "2026-07-01T17:11:55.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/43/1947f06babed6b3f1d7f38b0c767f52df66bfb2bc10b468c4a7de9eceff2/aiohappyeyeballs-2.7.1-py3-none-any.whl", hash = "sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472", size = 15038, upload-time = "2026-07-01T17:11:54.055Z" }, +] + +[[package]] +name = "aiohttp" +version = "3.14.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohappyeyeballs" }, + { name = "aiosignal" }, + { name = "attrs" }, + { name = "frozenlist" }, + { name = "multidict" }, + { name = "propcache" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/d9/22ce5786ac0c1653ae8b6c23bded02c1686d11f0dbb45b31ce128e0df985/aiohttp-3.14.3.tar.gz", hash = "sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc", size = 7971213, upload-time = "2026-07-23T01:57:27.037Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/5c/b3e4ff8ad43a8afef9602c5e90285936da1beaea8b029016b793891f03c3/aiohttp-3.14.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e568e14940c09955aa51f4e645b6daa18a581c5dcfcd73744dcc86a856e3ced3", size = 764250, upload-time = "2026-07-23T01:52:48.525Z" }, + { url = "https://files.pythonhosted.org/packages/0e/da/f1b384465e51449d844056b75070461da03a9a23e6c1747003695bf4172a/aiohttp-3.14.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:54cfcdee2770dac994417cbb0ee1f3eb0e7cb6b30c79bf44f2c02ff79ec5124a", size = 516281, upload-time = "2026-07-23T01:52:51.047Z" }, + { url = "https://files.pythonhosted.org/packages/b9/3f/01264f820ee2e3712a827892b1cd6ff80f3300c1fcbffbb45714a915d47a/aiohttp-3.14.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:21c016079415ed3fd676963e9793700a566d85dbbd6bfc564b9b2d209147dcc8", size = 514742, upload-time = "2026-07-23T01:52:53.779Z" }, + { url = "https://files.pythonhosted.org/packages/9e/8d/a71c6f2db52ac1ed142b133f7feddaa6b70539c3f4de24d7e226c95b794c/aiohttp-3.14.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d6088ec9894113802bddb3c09e974929aed2c7b3a8c456219b8aab4481f1a239", size = 1780613, upload-time = "2026-07-23T01:52:56.948Z" }, + { url = "https://files.pythonhosted.org/packages/a5/11/3dd9b3fb3a170f6ec9011b5291d876a6fab4086714c9e158600edf01b4fd/aiohttp-3.14.3-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:16ea7e24c309fb7c0bbd505d149abe4fe4dccfb8db911db7dbec0921bc889a6f", size = 1737688, upload-time = "2026-07-23T01:52:59.294Z" }, + { url = "https://files.pythonhosted.org/packages/6d/3e/834c26918be7d88068822b40e0db30fca50b5f4fe79104aa16a93f1d74e6/aiohttp-3.14.3-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56f355e79f71aef2a85c80305cc915f894b170dba76de5fe84f6351939b83c06", size = 1845742, upload-time = "2026-07-23T01:53:01.641Z" }, + { url = "https://files.pythonhosted.org/packages/cc/c9/49ab8572df7d66bc13d11e31f781292badb04180dd87ba98733066c6aed7/aiohttp-3.14.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:18c441d0a8fca6de8d1f546849b9f0ab20d435993e2c5b59562b2fae6be2f929", size = 1928412, upload-time = "2026-07-23T01:53:04.018Z" }, + { url = "https://files.pythonhosted.org/packages/a5/b9/2b8f0c0ce09c87a1daf80fd483431b56b1435d3f62789bc86f572e1245de/aiohttp-3.14.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:53e7b4ce82b54a8bcc71b3b67a5cbd177ca1d7f592cbc92cd38b7349f73482db", size = 1786220, upload-time = "2026-07-23T01:53:06.481Z" }, + { url = "https://files.pythonhosted.org/packages/85/00/9c45f81de11710460edfa1dc81317b6e882703b160926c879a9d20da9fcc/aiohttp-3.14.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f55119f7bf25f49ed210f6096090715da24f2943c62102448915fde3c62877ce", size = 1637231, upload-time = "2026-07-23T01:53:10.258Z" }, + { url = "https://files.pythonhosted.org/packages/19/ce/967d628e910756f3539c6107cb7844a1b69440dcb3029a5ee7871b09ab63/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9aa6e61fdf20105c4144e755bd586008ff450791d67b1c8146fdc15959c4d51c", size = 1753161, upload-time = "2026-07-23T01:53:13.817Z" }, + { url = "https://files.pythonhosted.org/packages/11/b2/0c3d4114f0aee4f580f5b3b4eb71b24d7a23b834ea506a4dfebe76513f35/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ccd4893707b3e2a13e39c90d43cf80edf2e4d0457935bcc103bf2346214c3f15", size = 1756356, upload-time = "2026-07-23T01:53:16.211Z" }, + { url = "https://files.pythonhosted.org/packages/63/5d/99e7d91c82f1399d1ae2a854e080bd1493fbc31e5e959dbc4ec33dac3bec/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b2466434105a4e03113c36ec775cc2ebe6676b62eae326fa670bb607ef788c1c", size = 1819846, upload-time = "2026-07-23T01:53:18.289Z" }, + { url = "https://files.pythonhosted.org/packages/ad/05/d5e1cb6480eeffd3f901d40a2c5e2d1e7effdc797837da3b490272699f13/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ba59d59aba08ac02fc03b0c8983ccd5ee39a199d0552ce9e6d2b4845b34d59ae", size = 1628531, upload-time = "2026-07-23T01:53:23.86Z" }, + { url = "https://files.pythonhosted.org/packages/c9/90/b934682bcaefae18a9e04f3dff5b68522ba810906358ae5029b68110ea3b/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ed099d105449c4f9e84f24af203cd131349d4761d8813fa7e02c32e7128cd910", size = 1832712, upload-time = "2026-07-23T01:53:27.551Z" }, + { url = "https://files.pythonhosted.org/packages/21/df/6061679faaf81fac746e7307c7adb71e858071a5d34c27583afefc64f543/aiohttp-3.14.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:152516815ef926786a0b6ae2b8f1fd2e0c71582dee0b435636865316fd4891b7", size = 1775014, upload-time = "2026-07-23T01:53:30.223Z" }, + { url = "https://files.pythonhosted.org/packages/8a/1d/f854878bbc69b88faefe924b619a34a6f59ec05fd387c77690667eaa75eb/aiohttp-3.14.3-cp311-cp311-win32.whl", hash = "sha256:a4af35c443e0b1a1bd6a8af3f3485d7fda15c142751a00f3ff8090f0b93346fa", size = 456006, upload-time = "2026-07-23T01:53:34.97Z" }, + { url = "https://files.pythonhosted.org/packages/73/0c/2af9d1674baccd1dbd47282a93d660a22e57ef6167c856deb24b4214fbab/aiohttp-3.14.3-cp311-cp311-win_amd64.whl", hash = "sha256:e1e74298bab6ee0d6e749ed4fd1901c7e604bdda32c03d787a2cc71c46d0433d", size = 481069, upload-time = "2026-07-23T01:53:39.673Z" }, + { url = "https://files.pythonhosted.org/packages/8e/76/88401ff3fc95e85c5fc38d588f36f55e61ecb64343b2bc8d69326f453cc0/aiohttp-3.14.3-cp311-cp311-win_arm64.whl", hash = "sha256:03cd2bde3d7f085b64e549c985f4bb928cad7e8ecf5323bfca320db548d81b39", size = 453021, upload-time = "2026-07-23T01:53:43.749Z" }, + { url = "https://files.pythonhosted.org/packages/18/d4/eb96299230e20acf2efae207cb8d69051f1f68e357e5ea5e479bf6fb097a/aiohttp-3.14.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5", size = 754690, upload-time = "2026-07-23T01:53:47.332Z" }, + { url = "https://files.pythonhosted.org/packages/88/11/e7a70a209eb9a067c0d3212b518a0134e3484f5178c7533878b6b514d469/aiohttp-3.14.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228", size = 509484, upload-time = "2026-07-23T01:53:51.159Z" }, + { url = "https://files.pythonhosted.org/packages/30/07/4bbc222cc8dbe31d4c3e8a5baad2286e4d42026ac0c570027b89afce6344/aiohttp-3.14.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee", size = 511949, upload-time = "2026-07-23T01:53:55.083Z" }, + { url = "https://files.pythonhosted.org/packages/54/b9/42e74c46b7b7c794b995bbc1f573fb48950c38b19d8600c62a6804ee2d67/aiohttp-3.14.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a", size = 1765282, upload-time = "2026-07-23T01:53:59.662Z" }, + { url = "https://files.pythonhosted.org/packages/6b/ed/62bc4d74363ad346d518e0720363a949f63e2e23439a79eb5813d4d29bb3/aiohttp-3.14.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b", size = 1741511, upload-time = "2026-07-23T01:54:04.063Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9f/181e8a8bc79e47d13c7fc4540bd7a3b729d9505609c61f392a8dd2fbfe55/aiohttp-3.14.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529", size = 1810680, upload-time = "2026-07-23T01:54:09.882Z" }, + { url = "https://files.pythonhosted.org/packages/5c/9a/dec94d6ad694552fe3424e3f1928d7a606a5d9d9433a04e7ecdd9d38ae7f/aiohttp-3.14.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787", size = 1905646, upload-time = "2026-07-23T01:54:13.475Z" }, + { url = "https://files.pythonhosted.org/packages/52/b7/7cd31f29d6055bd711ae6e669367fba6f5ae9de463910a793e30556a8db7/aiohttp-3.14.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42", size = 1792122, upload-time = "2026-07-23T01:54:15.752Z" }, + { url = "https://files.pythonhosted.org/packages/66/73/10b1ef93afa61f4963c746257b70ced619cf31a4798671de5fdb2608501d/aiohttp-3.14.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b", size = 1591127, upload-time = "2026-07-23T01:54:19.489Z" }, + { url = "https://files.pythonhosted.org/packages/49/ed/3b203fa6de1b338c14acdc06bf6ca9b043b7944f005966958c2ced932cde/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043", size = 1725210, upload-time = "2026-07-23T01:54:24.129Z" }, + { url = "https://files.pythonhosted.org/packages/28/b7/1c2aab8c706436dcc28598452488ac9cd7c409da815237c28c27d58993e6/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427", size = 1764848, upload-time = "2026-07-23T01:54:27.973Z" }, + { url = "https://files.pythonhosted.org/packages/54/50/94c28f08b131c4bf10984ea2c7a536c9920608bb2d6e7f95642c30cc87b7/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d", size = 1777102, upload-time = "2026-07-23T01:54:31.775Z" }, + { url = "https://files.pythonhosted.org/packages/13/d4/e7d09ba7d345fb2d74440fd2fa033c5e079fac05552927705986f41a364f/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0", size = 1580205, upload-time = "2026-07-23T01:54:34.518Z" }, + { url = "https://files.pythonhosted.org/packages/a3/84/072a91d68e1e1eb587985b54baab94221277f877e8ef274fc213a0ceae28/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d", size = 1797219, upload-time = "2026-07-23T01:54:36.995Z" }, + { url = "https://files.pythonhosted.org/packages/e0/eb/aad34e897e668424d6e995da5dff8a4a09af93363d3392488772957a63aa/aiohttp-3.14.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19", size = 1768629, upload-time = "2026-07-23T01:54:40.103Z" }, + { url = "https://files.pythonhosted.org/packages/b6/2b/6bb88ddba0fecd9122aa3ebcad25996cf6c083a4a7040dbb3a4f97972af6/aiohttp-3.14.3-cp312-cp312-win32.whl", hash = "sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559", size = 451481, upload-time = "2026-07-23T01:54:42.547Z" }, + { url = "https://files.pythonhosted.org/packages/76/9b/f2f8f108da17ecef2cc3efc424e8b7ad3782b1a8360f7b8eae8ced84f6ea/aiohttp-3.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a", size = 476845, upload-time = "2026-07-23T01:54:44.853Z" }, + { url = "https://files.pythonhosted.org/packages/3e/44/28dac80a8941b604f4da10ce21097614ca1bf905ce93dca28d8d7de9c1e7/aiohttp-3.14.3-cp312-cp312-win_arm64.whl", hash = "sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c", size = 448050, upload-time = "2026-07-23T01:54:47.087Z" }, + { url = "https://files.pythonhosted.org/packages/57/be/5afd201cc0ab139029aadb75392efe85a293403d9dd3a3226161c21ce00c/aiohttp-3.14.3-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86", size = 506269, upload-time = "2026-07-23T01:54:49.075Z" }, + { url = "https://files.pythonhosted.org/packages/22/09/dec8189d62b45ade009f6792a2264b942a90cb88aeaf181239933cd72c3c/aiohttp-3.14.3-cp313-cp313-android_21_x86_64.whl", hash = "sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627", size = 515166, upload-time = "2026-07-23T01:54:51.894Z" }, + { url = "https://files.pythonhosted.org/packages/28/24/2854869d29ed8a8b19d74f9ec6629515f7e04d02dd329d9d179201e58e47/aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82", size = 486263, upload-time = "2026-07-23T01:54:54.223Z" }, + { url = "https://files.pythonhosted.org/packages/d4/dd/57187c8be2a35aea65eaee3bd2c3dcbbcf0204f5106c89637e3610380cd1/aiohttp-3.14.3-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c", size = 492299, upload-time = "2026-07-23T01:54:56.236Z" }, + { url = "https://files.pythonhosted.org/packages/b9/11/06ae6ed8f0d414edf4068861e233d8fe23ee699bfd4b3ceb8663db948a62/aiohttp-3.14.3-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f", size = 502235, upload-time = "2026-07-23T01:54:58.377Z" }, + { url = "https://files.pythonhosted.org/packages/7e/a3/559639c34a345d2cf7c52dff6838119f2eaf29eb508227b5b83f573af813/aiohttp-3.14.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80", size = 750883, upload-time = "2026-07-23T01:55:00.65Z" }, + { url = "https://files.pythonhosted.org/packages/91/cd/41e131f13afd1e7b0172a9d9eda085ef90eb8439f41f0d279db81ed3ae60/aiohttp-3.14.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0", size = 508473, upload-time = "2026-07-23T01:55:02.945Z" }, + { url = "https://files.pythonhosted.org/packages/bc/6b/e7f13410d391c6e55b4c007a8de024355389d7d459e3d64c42b2d33617e5/aiohttp-3.14.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf", size = 509190, upload-time = "2026-07-23T01:55:05.173Z" }, + { url = "https://files.pythonhosted.org/packages/97/21/6464573e53d69672cc1eada3e5c5cb2d2efa82701e8305a0f2047a576967/aiohttp-3.14.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd", size = 1761478, upload-time = "2026-07-23T01:55:07.383Z" }, + { url = "https://files.pythonhosted.org/packages/1a/81/d217043a4c17fbce360905e3b2bdd20139ebc9a2de836d035d179c4da006/aiohttp-3.14.3-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807", size = 1735092, upload-time = "2026-07-23T01:55:09.803Z" }, + { url = "https://files.pythonhosted.org/packages/a1/66/e13a02d0eeb1a9a502402a977abb4e4abff9fe4051c26f80558c57a7c975/aiohttp-3.14.3-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8", size = 1800546, upload-time = "2026-07-23T01:55:12.012Z" }, + { url = "https://files.pythonhosted.org/packages/26/5e/57d42fca1d18cb5acc1cad945d017fabc5d6ae71d8a08ad66be8dc3ee544/aiohttp-3.14.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24", size = 1895250, upload-time = "2026-07-23T01:55:14.357Z" }, + { url = "https://files.pythonhosted.org/packages/ca/1c/7da8d08e74d56f00070822f9638ff3f1c563f8ad87d1efa996c87bfc8644/aiohttp-3.14.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5", size = 1789289, upload-time = "2026-07-23T01:55:16.668Z" }, + { url = "https://files.pythonhosted.org/packages/cd/0f/cf16bcf56896981c1a0319f5d5db9337994b5165730c48a8fa07e9b34be6/aiohttp-3.14.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4", size = 1586706, upload-time = "2026-07-23T01:55:18.913Z" }, + { url = "https://files.pythonhosted.org/packages/fe/6f/76eac12a7f2480e1e304f842efdb07db33256b0d9165b866b6ef0806c202/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9", size = 1724652, upload-time = "2026-07-23T01:55:21.296Z" }, + { url = "https://files.pythonhosted.org/packages/39/b6/19c8c592baeeb94b75f966547d40c02ac7590902306ec5863d5c027cf506/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1", size = 1756239, upload-time = "2026-07-23T01:55:23.705Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c9/4e9383150296f97f873b680c4de8fb2cd88608fb9f48c79edcb111611abc/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371", size = 1769161, upload-time = "2026-07-23T01:55:26.082Z" }, + { url = "https://files.pythonhosted.org/packages/aa/1e/147bdc6cc5de5f3ab011be8bf5d6e786633249f22c20bae06f85e45f5387/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde", size = 1578759, upload-time = "2026-07-23T01:55:28.846Z" }, + { url = "https://files.pythonhosted.org/packages/fd/31/78388a9d6040ece2e11df62ea229a822cf5e52d238374b220ae9975b2623/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e", size = 1792025, upload-time = "2026-07-23T01:55:31.457Z" }, + { url = "https://files.pythonhosted.org/packages/03/51/a3d29fdf2c25d796746af8ad6fe56a45d6256c38b0a8a2ed752e1160b3a2/aiohttp-3.14.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71", size = 1768477, upload-time = "2026-07-23T01:55:33.87Z" }, + { url = "https://files.pythonhosted.org/packages/29/a6/442e18b5afeade534d877a2dc3c3e392aff8d49787890b0cf84790410267/aiohttp-3.14.3-cp313-cp313-win32.whl", hash = "sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0", size = 451069, upload-time = "2026-07-23T01:55:36.121Z" }, + { url = "https://files.pythonhosted.org/packages/9d/69/3d876ac02659f271cf7f6769f14a8e3de5b6e888ed8b5a7e998086a4cec8/aiohttp-3.14.3-cp313-cp313-win_amd64.whl", hash = "sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883", size = 476518, upload-time = "2026-07-23T01:55:38.303Z" }, + { url = "https://files.pythonhosted.org/packages/b2/0e/50d6e6471cd31edce8b282bdec59375a3a69124d8a989a0b1313355cae52/aiohttp-3.14.3-cp313-cp313-win_arm64.whl", hash = "sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2", size = 447676, upload-time = "2026-07-23T01:55:40.451Z" }, + { url = "https://files.pythonhosted.org/packages/c8/20/887fdcf832326571b370ffc347b3e70abe101096f3720126aac161b1d872/aiohttp-3.14.3-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062", size = 509067, upload-time = "2026-07-23T01:55:42.618Z" }, + { url = "https://files.pythonhosted.org/packages/ad/a3/92cec936f78cc4bf0fa5554ebe593b73459d94e3c62303e1902a4cccb6f7/aiohttp-3.14.3-cp314-cp314-android_24_x86_64.whl", hash = "sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6", size = 514774, upload-time = "2026-07-23T01:55:44.937Z" }, + { url = "https://files.pythonhosted.org/packages/29/ba/2a0c38df3fc557620b6a5acd98364af050053b6285b4dc7ee74100c63c18/aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919", size = 488134, upload-time = "2026-07-23T01:55:47.135Z" }, + { url = "https://files.pythonhosted.org/packages/48/d6/d51b7d4bf309af3693940d8ffd2b9ed0b682434ef85959b7c9c137f60cf8/aiohttp-3.14.3-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7", size = 494201, upload-time = "2026-07-23T01:55:49.451Z" }, + { url = "https://files.pythonhosted.org/packages/3f/5a/8f624384e5f1efabb5229b94157eb966b021e97bdb188c62860c2ae243c2/aiohttp-3.14.3-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0", size = 502766, upload-time = "2026-07-23T01:55:51.656Z" }, + { url = "https://files.pythonhosted.org/packages/a6/26/4ff0164370deec18fb19254ee4ab10b7a73304ac0c860b13f5f84663759b/aiohttp-3.14.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924", size = 756557, upload-time = "2026-07-23T01:55:53.964Z" }, + { url = "https://files.pythonhosted.org/packages/97/a3/7056b86dc0d9ec709ea9777eae3b0161428f943372f8b98c01c11593b682/aiohttp-3.14.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646", size = 510168, upload-time = "2026-07-23T01:55:56.22Z" }, + { url = "https://files.pythonhosted.org/packages/85/ed/0357a015892fd68058bf2d39d3fd1958e459b997a7db30aaa6aaa434ae96/aiohttp-3.14.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b", size = 512957, upload-time = "2026-07-23T01:55:58.437Z" }, + { url = "https://files.pythonhosted.org/packages/47/d1/8aba53f15ccb2238405f5e9d30e2a8ca44f93878c26e7165ade00d374b1c/aiohttp-3.14.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30", size = 1750149, upload-time = "2026-07-23T01:56:00.856Z" }, + { url = "https://files.pythonhosted.org/packages/49/bd/40c3fee327529284375c6701cbb0fa4600cc2e8432af1378f897e2ef7d3a/aiohttp-3.14.3-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9", size = 1707685, upload-time = "2026-07-23T01:56:03.371Z" }, + { url = "https://files.pythonhosted.org/packages/2a/a3/ca0cc6724cca8114b05694abd916060758c79894c3aa5b012cdadc1bc28e/aiohttp-3.14.3-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f", size = 1803911, upload-time = "2026-07-23T01:56:05.817Z" }, + { url = "https://files.pythonhosted.org/packages/95/b5/85b099c299c3ffd38ad9b3e43694c8a346934e4a30c88c4fd5a841234f77/aiohttp-3.14.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d", size = 1876929, upload-time = "2026-07-23T01:56:08.413Z" }, + { url = "https://files.pythonhosted.org/packages/d5/b7/1da684a04175473fa4cddbf9a2f572e79514c3fd27a74597f43057d4f3da/aiohttp-3.14.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147", size = 1761112, upload-time = "2026-07-23T01:56:10.918Z" }, + { url = "https://files.pythonhosted.org/packages/d1/16/bc4b55e3e5cb175fd69c53c90d60d2f47797cb343da5106e23863dc4dba4/aiohttp-3.14.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c", size = 1583500, upload-time = "2026-07-23T01:56:13.613Z" }, + { url = "https://files.pythonhosted.org/packages/2a/e8/13a9d957a1ee40837f46aa30f0f4c657e673ad86a2e6362a9f9be20d26d9/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a", size = 1713940, upload-time = "2026-07-23T01:56:15.969Z" }, + { url = "https://files.pythonhosted.org/packages/38/05/d33c680c1bcf1c7e130f9cbfc1fc02fe8bb0c4af2a94a53dd5fb56131e5c/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0", size = 1724413, upload-time = "2026-07-23T01:56:18.591Z" }, + { url = "https://files.pythonhosted.org/packages/85/1d/af798d306f7a74b6a632dbcabcf62a4c91391b7582d2a8c6d7712e2cc54e/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661", size = 1770748, upload-time = "2026-07-23T01:56:21.074Z" }, + { url = "https://files.pythonhosted.org/packages/a8/92/ad720d472556a995049206867765e9410969684f86ee09423ff9969044c1/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22", size = 1577564, upload-time = "2026-07-23T01:56:23.475Z" }, + { url = "https://files.pythonhosted.org/packages/60/ad/0ed7586cbef7a884e23a752fa2bb987a122e6a5dd50dab109258d0a95193/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41", size = 1782080, upload-time = "2026-07-23T01:56:25.994Z" }, + { url = "https://files.pythonhosted.org/packages/97/ea/dbaed0d73e8a69aad653b045dab451c67c2454bb731a37b45a86593e9422/aiohttp-3.14.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf", size = 1745813, upload-time = "2026-07-23T01:56:28.604Z" }, + { url = "https://files.pythonhosted.org/packages/81/1b/6893d4bc57e434fc93a6c9217c637d967a0b651d989f6e3265179375754a/aiohttp-3.14.3-cp314-cp314-win32.whl", hash = "sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da", size = 455872, upload-time = "2026-07-23T01:56:31.031Z" }, + { url = "https://files.pythonhosted.org/packages/f5/8b/c7baa1ba1eda4db6989baefe5de6d99834921b84ebd7918624febcb9f290/aiohttp-3.14.3-cp314-cp314-win_amd64.whl", hash = "sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100", size = 481030, upload-time = "2026-07-23T01:56:33.365Z" }, + { url = "https://files.pythonhosted.org/packages/22/8c/c29d067df825a2df88ca432db848aa2fe8199598359cc06c12b09320cac9/aiohttp-3.14.3-cp314-cp314-win_arm64.whl", hash = "sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc", size = 453669, upload-time = "2026-07-23T01:56:35.731Z" }, + { url = "https://files.pythonhosted.org/packages/6a/a4/9c033beb355d39b6147980597ec9645e4729243f686ee4dc73945de72030/aiohttp-3.14.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b", size = 791403, upload-time = "2026-07-23T01:56:37.972Z" }, + { url = "https://files.pythonhosted.org/packages/80/ca/87c32a0a7704583cfc49660bd817889bae5b830bf53b5dcb4e92145ac2da/aiohttp-3.14.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0", size = 526413, upload-time = "2026-07-23T01:56:40.523Z" }, + { url = "https://files.pythonhosted.org/packages/9e/d8/8ec0e471248c500acdce2be3f46db8fb62b5eb60efef072529cc85ee1d26/aiohttp-3.14.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e", size = 532135, upload-time = "2026-07-23T01:56:42.876Z" }, + { url = "https://files.pythonhosted.org/packages/fe/45/f8919fd936e8b79fcd9bda7b6d8e62613462a713f4f17987fd7c34399142/aiohttp-3.14.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716", size = 1922742, upload-time = "2026-07-23T01:56:45.528Z" }, + { url = "https://files.pythonhosted.org/packages/f6/ec/9ca76b28a27525b0cc53e20842e0228b022f301ce1f436b7d814b4aaf2df/aiohttp-3.14.3-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f", size = 1787371, upload-time = "2026-07-23T01:56:48.045Z" }, + { url = "https://files.pythonhosted.org/packages/b1/04/6acdbf17315f7b55f1937e3387acb89a3cddeb4995689553d064af8e92ab/aiohttp-3.14.3-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553", size = 1912623, upload-time = "2026-07-23T01:56:50.605Z" }, + { url = "https://files.pythonhosted.org/packages/86/e6/438b0c79ca6f45eb9fd9817dd4c01a91919a38c0de5ee9e05e2b4dc0ece7/aiohttp-3.14.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100", size = 2005515, upload-time = "2026-07-23T01:56:53.153Z" }, + { url = "https://files.pythonhosted.org/packages/bb/6b/62cbd6577758699525f5c712d1ddef57d9875fbab0ae8d5f5a202fd598f8/aiohttp-3.14.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85", size = 1879906, upload-time = "2026-07-23T01:56:55.818Z" }, + { url = "https://files.pythonhosted.org/packages/00/95/18bcbf830a21dc3aae24d8f6b6feaf3db1d2090242d00a7868db2ffb0b67/aiohttp-3.14.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33", size = 1675849, upload-time = "2026-07-23T01:56:58.861Z" }, + { url = "https://files.pythonhosted.org/packages/a9/19/47f4968659c5e23606c3790c80fc624e691c153d036148449ee84d31b287/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f", size = 1843496, upload-time = "2026-07-23T01:57:01.591Z" }, + { url = "https://files.pythonhosted.org/packages/64/af/38c33c4dd82fddcb4e56c4653b6f1072a8edbc6b7fa15809f14932c41e2d/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0", size = 1827746, upload-time = "2026-07-23T01:57:05.131Z" }, + { url = "https://files.pythonhosted.org/packages/a1/9d/0537cda4885ac8f5b7053d164dd06312f4c483a4edcb8ee5b8aaf2a989bf/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098", size = 1853810, upload-time = "2026-07-23T01:57:08.043Z" }, + { url = "https://files.pythonhosted.org/packages/19/fe/26f9c5e6458385aa86497836b0dea6fb2f027827d63f37c7856cce9286ee/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25", size = 1668895, upload-time = "2026-07-23T01:57:10.837Z" }, + { url = "https://files.pythonhosted.org/packages/ec/4c/618b1db9b9ba079b8875d2cdf78e7c4a3bf72903bd5850fee7dd9544600a/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9", size = 1883833, upload-time = "2026-07-23T01:57:13.672Z" }, + { url = "https://files.pythonhosted.org/packages/94/c6/bd959bd1e4771f9fd944e9e436224c48c77b018b73b519b5aad346335bcc/aiohttp-3.14.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb", size = 1844251, upload-time = "2026-07-23T01:57:16.593Z" }, + { url = "https://files.pythonhosted.org/packages/5e/19/08d41839658bdd44a0ed2480f3891705ecb487ce28c0dde62c9040c997e0/aiohttp-3.14.3-cp314-cp314t-win32.whl", hash = "sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963", size = 474180, upload-time = "2026-07-23T01:57:19.306Z" }, + { url = "https://files.pythonhosted.org/packages/99/5d/3cd6ef0a2b2851f7ab913b5b079334781bd50ff56a323e4454063377a080/aiohttp-3.14.3-cp314-cp314t-win_amd64.whl", hash = "sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b", size = 500528, upload-time = "2026-07-23T01:57:21.762Z" }, + { url = "https://files.pythonhosted.org/packages/a4/37/cfd1ed540a4d318da025590d96b728e63713c09e9377950fc655dadeb856/aiohttp-3.14.3-cp314-cp314t-win_arm64.whl", hash = "sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7", size = 469280, upload-time = "2026-07-23T01:57:24.241Z" }, +] + +[[package]] +name = "aiosignal" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "frozenlist" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, +] + +[[package]] +name = "annotated-doc" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/8e/38aa427ed5402449e226975b649c5dc73ccadfefeb95e6aecb8f8ea4b6b6/annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb", size = 10758, upload-time = "2026-07-28T13:50:58.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, +] + +[[package]] +name = "anyio" +version = "4.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/2a/23f34ec9d04624958e137efdc394888716353190e75f25dd22c7a2c7a8aa/charset_normalizer-3.4.9.tar.gz", hash = "sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b", size = 152439, upload-time = "2026-07-07T14:34:58.454Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/e3/85ec501f206fb049259288c1f3506e53876937fb00edb47009348e66756b/charset_normalizer-3.4.9-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5", size = 317075, upload-time = "2026-07-07T14:32:56.021Z" }, + { url = "https://files.pythonhosted.org/packages/c3/69/2a5385192e67175f7d8bd5ce4f57c24bc956439adeae5c13a99aa28a53d1/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2", size = 213837, upload-time = "2026-07-07T14:32:57.78Z" }, + { url = "https://files.pythonhosted.org/packages/b3/46/03ddc7da576d814fe0a36dd1f0fd3258e95404b4b2e3c026b7923d7e133f/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a", size = 235503, upload-time = "2026-07-07T14:32:59.205Z" }, + { url = "https://files.pythonhosted.org/packages/4e/6e/de0229a7ef40f6f9d28a837eebf4ec47bdca5dab4e900c84f22919af636a/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29", size = 229944, upload-time = "2026-07-07T14:33:00.803Z" }, + { url = "https://files.pythonhosted.org/packages/a5/34/49b9060e8418b14fb5cba9cf6bfb383111e2538a03a1fb18e66a95aeb3d5/charset_normalizer-3.4.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c", size = 221276, upload-time = "2026-07-07T14:33:02.199Z" }, + { url = "https://files.pythonhosted.org/packages/44/95/80282cce0fae9c3061203d723ee87da996aed79679e65d8935050ee7ca1f/charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b", size = 205260, upload-time = "2026-07-07T14:33:03.698Z" }, + { url = "https://files.pythonhosted.org/packages/0c/74/2f62c8821b969ea3bd67cc2e6976834f48ca5d12664d2559ebcd9bcfbed7/charset_normalizer-3.4.9-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db", size = 217786, upload-time = "2026-07-07T14:33:05.12Z" }, + { url = "https://files.pythonhosted.org/packages/d9/8d/feabb82cb49fcad14515b1d7d1ca4787b0da7fc723a212bf89bc9e0fac52/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993", size = 216798, upload-time = "2026-07-07T14:33:06.629Z" }, + { url = "https://files.pythonhosted.org/packages/a5/ff/c946d63bc3786d5b84d960b0f7ab7e25b828486a946b5aa997625bcaf6a6/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da", size = 206429, upload-time = "2026-07-07T14:33:08.006Z" }, + { url = "https://files.pythonhosted.org/packages/af/ba/5e5007c370702f85d2ef75791fac7943ed41e080364a673b20142e430e3e/charset_normalizer-3.4.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3", size = 223066, upload-time = "2026-07-07T14:33:09.783Z" }, + { url = "https://files.pythonhosted.org/packages/83/d5/9096aa3cf532dfad237861544eb47a0f20d5adbf1039760fed8eaae935d9/charset_normalizer-3.4.9-cp311-cp311-win32.whl", hash = "sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d", size = 150456, upload-time = "2026-07-07T14:33:11.217Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a1/e29995109e455dc8eff8d0fac6ae509be39561318a7cfeac5d33ad029213/charset_normalizer-3.4.9-cp311-cp311-win_amd64.whl", hash = "sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1", size = 161410, upload-time = "2026-07-07T14:33:12.743Z" }, + { url = "https://files.pythonhosted.org/packages/4f/8d/1569f4d0032d6ba2a4fe4591c35bf87868c600c41a71eb5c2e1ffa8464c2/charset_normalizer-3.4.9-cp311-cp311-win_arm64.whl", hash = "sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec", size = 152649, upload-time = "2026-07-07T14:33:14.173Z" }, + { url = "https://files.pythonhosted.org/packages/70/4a/ecbd131485c07fcdfad54e28946d513e3da22ef3b4bd854dcafae54ec739/charset_normalizer-3.4.9-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0", size = 319300, upload-time = "2026-07-07T14:33:15.666Z" }, + { url = "https://files.pythonhosted.org/packages/ec/96/5d9364e3342d69f3a045e1777bc47c85c383e6e9466d561b33fdb419d1f9/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9", size = 215802, upload-time = "2026-07-07T14:33:17.031Z" }, + { url = "https://files.pythonhosted.org/packages/4b/4c/5361f9aa7f2cb58d94f2ab831b3d493f69efb1d239654b4744e3c09527cb/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44", size = 237171, upload-time = "2026-07-07T14:33:18.576Z" }, + { url = "https://files.pythonhosted.org/packages/50/78/ce342ca4ff30b2eb49fe6d9578df85974f90c67d294113e94efdd9664cbd/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9", size = 233075, upload-time = "2026-07-07T14:33:20.084Z" }, + { url = "https://files.pythonhosted.org/packages/01/c4/4fa4c8b3097a11f3c5f09a35b72ed6855fb1d332469504962ab7bafcc702/charset_normalizer-3.4.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd", size = 224256, upload-time = "2026-07-07T14:33:21.747Z" }, + { url = "https://files.pythonhosted.org/packages/87/3a/ad914516df7e358a81aae018caa5e0470ba827fa6d763b1d2e87d920a5f6/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84", size = 208784, upload-time = "2026-07-07T14:33:23.313Z" }, + { url = "https://files.pythonhosted.org/packages/d7/74/3c12f9755717dfe5c5c87da63f35d765fa0c00382ec26bf23f7fae34f2ba/charset_normalizer-3.4.9-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b", size = 219928, upload-time = "2026-07-07T14:33:24.814Z" }, + { url = "https://files.pythonhosted.org/packages/33/9a/895095b83e7907abd6d3d99aad3a38ad0d9686cc186cb0c94c24320fe63e/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde", size = 218489, upload-time = "2026-07-07T14:33:26.42Z" }, + { url = "https://files.pythonhosted.org/packages/a1/34/ef5c05f412f42520d7709b7d3784d19640839eb7366ded1755511585429f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39", size = 210267, upload-time = "2026-07-07T14:33:27.952Z" }, + { url = "https://files.pythonhosted.org/packages/83/dc/9b29fa4412b318bf3bfea985c35d67eb55e04b59a7c3f2237168b0e0be6f/charset_normalizer-3.4.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62", size = 226030, upload-time = "2026-07-07T14:33:29.397Z" }, + { url = "https://files.pythonhosted.org/packages/0e/42/6dbc00b8cd16011691203e33570fa42ed5746599a2e878112d16eab403a3/charset_normalizer-3.4.9-cp312-cp312-win32.whl", hash = "sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642", size = 151185, upload-time = "2026-07-07T14:33:30.781Z" }, + { url = "https://files.pythonhosted.org/packages/80/cc/f920afd1a23c58ccd53c1d36085a71893a4737ff5e66e0371efab6809850/charset_normalizer-3.4.9-cp312-cp312-win_amd64.whl", hash = "sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0", size = 162557, upload-time = "2026-07-07T14:33:32.176Z" }, + { url = "https://files.pythonhosted.org/packages/f0/e6/0386d43a261ff4e4b30c5857af7df877254b46bec7b9d1b74b6bf969a90b/charset_normalizer-3.4.9-cp312-cp312-win_arm64.whl", hash = "sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2", size = 152665, upload-time = "2026-07-07T14:33:33.711Z" }, + { url = "https://files.pythonhosted.org/packages/b2/06/97ec2aeae780b31d742b6352218b43841a6871e2564578ca522dce4a45c3/charset_normalizer-3.4.9-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614", size = 317688, upload-time = "2026-07-07T14:33:35.408Z" }, + { url = "https://files.pythonhosted.org/packages/d0/39/8ff066c672434225f8d25f8b739f992af250944392173dcc88362681c9bf/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698", size = 214982, upload-time = "2026-07-07T14:33:36.996Z" }, + { url = "https://files.pythonhosted.org/packages/92/8f/3a47a3667c83c2df9483d91644c6c107de3bf8874aa1793da9d3012eb986/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b", size = 236460, upload-time = "2026-07-07T14:33:38.536Z" }, + { url = "https://files.pythonhosted.org/packages/f1/60/b22cdbee7e4013dab8b0d7647fc6181120fbbbc8f7025c226d15bd5a47fc/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9", size = 232003, upload-time = "2026-07-07T14:33:40.059Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f8/72eb13dcabe7257035cea8aefd922caad2f110d252bf9f67c4c2ca763aee/charset_normalizer-3.4.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33", size = 223149, upload-time = "2026-07-07T14:33:41.631Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3e/faee8f9de92b14ee1198e9163252bb15efee7301b31256a3b6d9ebfdd0dd/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63", size = 207901, upload-time = "2026-07-07T14:33:43.209Z" }, + { url = "https://files.pythonhosted.org/packages/3a/25/45f30093ae27dd7b92a793b61882a38685f993700113ca36e0c9c14965e1/charset_normalizer-3.4.9-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0", size = 219176, upload-time = "2026-07-07T14:33:44.725Z" }, + { url = "https://files.pythonhosted.org/packages/48/18/c8f397329c35e32f6a837e488986f4ae03bd2abebc453b48714991630c2f/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe", size = 217356, upload-time = "2026-07-07T14:33:46.192Z" }, + { url = "https://files.pythonhosted.org/packages/86/7e/5ce0bba863470fd1902d5e5843968951bddf38abe4742fc97116ef4598b3/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35", size = 209614, upload-time = "2026-07-07T14:33:47.705Z" }, + { url = "https://files.pythonhosted.org/packages/6c/ef/2473d3c4d869155be4af1191111d59c4d5c4e0173026f7e85b176e23bf65/charset_normalizer-3.4.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8", size = 224991, upload-time = "2026-07-07T14:33:49.238Z" }, + { url = "https://files.pythonhosted.org/packages/d0/a3/53ddae3db108a088156aa8ddfafd411ebbc1340f48c5573f697b27f69a39/charset_normalizer-3.4.9-cp313-cp313-win32.whl", hash = "sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9", size = 150622, upload-time = "2026-07-07T14:33:50.711Z" }, + { url = "https://files.pythonhosted.org/packages/e8/ef/6953a77c7cf2c2ff9998e6f575ab3e380119f100223381565a4f94c1f836/charset_normalizer-3.4.9-cp313-cp313-win_amd64.whl", hash = "sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115", size = 161947, upload-time = "2026-07-07T14:33:52.197Z" }, + { url = "https://files.pythonhosted.org/packages/6e/fb/d560d1d1555debbfe7849d9cac6145c1b537709d79576bf22557ed803b82/charset_normalizer-3.4.9-cp313-cp313-win_arm64.whl", hash = "sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012", size = 152594, upload-time = "2026-07-07T14:33:53.486Z" }, + { url = "https://files.pythonhosted.org/packages/7e/8d/496817fa0944239ecae662dd57ea765cfeaec6a735f9f025d4b7b72e7143/charset_normalizer-3.4.9-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380", size = 317253, upload-time = "2026-07-07T14:33:54.994Z" }, + { url = "https://files.pythonhosted.org/packages/2b/f9/ef4a69ea338ad3c0deceea0f5f7d2380ae8b52132b06d652cb0d2cd86706/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9", size = 215898, upload-time = "2026-07-07T14:33:56.334Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e7/5ddfd76fc061eb52de219658a4aa431cbacadf0a0219c8854f00da50d289/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4", size = 236718, upload-time = "2026-07-07T14:33:57.9Z" }, + { url = "https://files.pythonhosted.org/packages/49/ba/768fa3f36048d81c477a0ce61f813bc1454d80917ccfe550abd9f44f5e24/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a", size = 232519, upload-time = "2026-07-07T14:33:59.811Z" }, + { url = "https://files.pythonhosted.org/packages/f4/c4/b3e049d2aa3766180c78507110543d9d50894cc97f57de543f1be521dcdc/charset_normalizer-3.4.9-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046", size = 223143, upload-time = "2026-07-07T14:34:01.517Z" }, + { url = "https://files.pythonhosted.org/packages/19/79/55c32d06d76ae4feafe053f061f3e3ab70bcf19f4007797ce8c3efda7830/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81", size = 206742, upload-time = "2026-07-07T14:34:03.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/e0/47c079dd82d217c807479cd59ffd30af56307ea31c108b75758970459ad3/charset_normalizer-3.4.9-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917", size = 219191, upload-time = "2026-07-07T14:34:04.657Z" }, + { url = "https://files.pythonhosted.org/packages/42/ab/b9bc2e77d6b44a7e46ef62ec5cac1c9a6ba7b9135a5d560f002696ec9995/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41", size = 218328, upload-time = "2026-07-07T14:34:06.115Z" }, + { url = "https://files.pythonhosted.org/packages/f1/78/c9c71d599f5aa2d42bcdd35cbbd46d7f535351a57e40ff7d8e5a7e219401/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1", size = 207406, upload-time = "2026-07-07T14:34:07.554Z" }, + { url = "https://files.pythonhosted.org/packages/f6/39/c914445c321a845097ce4f6ac7de9a18228a77b766272125a1ce00d851eb/charset_normalizer-3.4.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf", size = 225157, upload-time = "2026-07-07T14:34:09.061Z" }, + { url = "https://files.pythonhosted.org/packages/9b/f2/c0d4b8508565a36bc5c624e88ed297f5b0b1095011034d7f5b83a69908b5/charset_normalizer-3.4.9-cp314-cp314-win32.whl", hash = "sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48", size = 151095, upload-time = "2026-07-07T14:34:10.901Z" }, + { url = "https://files.pythonhosted.org/packages/49/fd/a1d26144398c67486422a72bf5812cda22cb4ccfcd95a290fb41ceb4b8e2/charset_normalizer-3.4.9-cp314-cp314-win_amd64.whl", hash = "sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b", size = 162796, upload-time = "2026-07-07T14:34:12.47Z" }, + { url = "https://files.pythonhosted.org/packages/20/95/d75e82f8ce9fd323ebf059c16c9aadefb22a1ecde13b7840b35835e4886c/charset_normalizer-3.4.9-cp314-cp314-win_arm64.whl", hash = "sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519", size = 153334, upload-time = "2026-07-07T14:34:14.044Z" }, + { url = "https://files.pythonhosted.org/packages/00/5e/17398df3a139985ba9d11ed072531986f408c8fca952835ef1ab1820c02b/charset_normalizer-3.4.9-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198", size = 338848, upload-time = "2026-07-07T14:34:15.688Z" }, + { url = "https://files.pythonhosted.org/packages/cd/91/7253a32e86b7e1d1239b1b36ba6dd0f021a21107ab33054b53119cc083b9/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32", size = 223022, upload-time = "2026-07-07T14:34:17.248Z" }, + { url = "https://files.pythonhosted.org/packages/cb/32/2e64bd2be10e89c61e57ebe6a93fd98ae88eb7ebe414b5121f22c96c69eb/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632", size = 241590, upload-time = "2026-07-07T14:34:18.813Z" }, + { url = "https://files.pythonhosted.org/packages/3d/ef/d96ec496cfea0c21db43b0ad03891308b02388d054cc902cf0e5a1ad6a88/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf", size = 239584, upload-time = "2026-07-07T14:34:20.52Z" }, + { url = "https://files.pythonhosted.org/packages/d4/ce/9af95f7876194bd7a14e3dfe4a4de2e0bff02666a3910d72beafd06cc297/charset_normalizer-3.4.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990", size = 230224, upload-time = "2026-07-07T14:34:22.189Z" }, + { url = "https://files.pythonhosted.org/packages/52/94/af74dde74a3996bd959c350709bfe50e297823d70a8c1cbd54b838880863/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d", size = 212667, upload-time = "2026-07-07T14:34:23.857Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f0/f1c4fe746c395922961b5916ed1d7d6e7d4c84851d19ed43cc89980ec953/charset_normalizer-3.4.9-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e", size = 227179, upload-time = "2026-07-07T14:34:25.586Z" }, + { url = "https://files.pythonhosted.org/packages/e4/56/6c745619ac397e8871e2bcd3cea1eec86b877488f33888b3aef5c3ed506e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c", size = 225372, upload-time = "2026-07-07T14:34:27.212Z" }, + { url = "https://files.pythonhosted.org/packages/78/ad/98aae8630ac71f16711968e38a5acfecce41b778bf2f0312851020f565a8/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2", size = 215222, upload-time = "2026-07-07T14:34:28.774Z" }, + { url = "https://files.pythonhosted.org/packages/f7/40/9593d54209765207a7f11073c06494c1721e4ca4a0a426c597679bf7f91e/charset_normalizer-3.4.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534", size = 231958, upload-time = "2026-07-07T14:34:30.345Z" }, + { url = "https://files.pythonhosted.org/packages/b1/27/693ee5e8a18191eb38647360c51cd505013e2bd3b366aa43fd5344c21e3c/charset_normalizer-3.4.9-cp314-cp314t-win32.whl", hash = "sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226", size = 155580, upload-time = "2026-07-07T14:34:31.884Z" }, + { url = "https://files.pythonhosted.org/packages/80/3f/bd97d3d9c613013d07cb7733d299385b41df37f0471310f5a73dc359f0b8/charset_normalizer-3.4.9-cp314-cp314t-win_amd64.whl", hash = "sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177", size = 167620, upload-time = "2026-07-07T14:34:33.438Z" }, + { url = "https://files.pythonhosted.org/packages/3d/c6/eee9dca4439b1061f76373f06ea855678cc4a64c1c3c90b50e479edbb8eb/charset_normalizer-3.4.9-cp314-cp314t-win_arm64.whl", hash = "sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501", size = 158037, upload-time = "2026-07-07T14:34:35.018Z" }, + { url = "https://files.pythonhosted.org/packages/98/2b/f97f1c193fb855c345d678f5077d6926034db0722df74c8f057020e05a25/charset_normalizer-3.4.9-py3-none-any.whl", hash = "sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5", size = 64538, upload-time = "2026-07-07T14:34:56.993Z" }, +] + +[[package]] +name = "click" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "coverage" +version = "7.15.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/76/d0/55fe630f4cf94e3fcba868240fad8c8cdd1f764e2a932f8926347e6ec4cd/coverage-7.15.2.tar.gz", hash = "sha256:3df60dc267f0a2ca23cb7a9ab1109c62b9335ffbf519fcfe167157c28c09b81d", size = 927741, upload-time = "2026-07-15T18:56:19.558Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/3a/54536704f507d4573bf9161c4d0dd3dd59b6d85e48c664e901b6844d8e33/coverage-7.15.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2f1ec6f304b156669cfde653b4e9a953f5de87e247ea02ac599bce0ab2744036", size = 221414, upload-time = "2026-07-15T18:53:51.941Z" }, + { url = "https://files.pythonhosted.org/packages/b6/d9/8ba925d29743e3577b21e4d8c11a702b76bc93c41e7fdfd1177af63d4b8d/coverage-7.15.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4d3361879d736f469f45723c11ea1a5bbdaf1f6928f0e632c940378b5aa9b660", size = 221913, upload-time = "2026-07-15T18:53:53.682Z" }, + { url = "https://files.pythonhosted.org/packages/09/54/a855f3aa0187f2b431ade4e4791b77b56282cfb5d201c83ec26a31b5b36a/coverage-7.15.2-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:c6a98d698f9e2c8008d0370ec7fc452ebfcc530002ae2d0061170d768b992589", size = 252332, upload-time = "2026-07-15T18:53:55.467Z" }, + { url = "https://files.pythonhosted.org/packages/8e/d3/13ac97b4370640ba3452fc8559b06cc2f479ce3ba4a0b632a73e44c38a7d/coverage-7.15.2-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d50dd325e18ec25bfcc10cd7f99b04df1ab9ec76b0918c260e60817ad0643dee", size = 254243, upload-time = "2026-07-15T18:53:57.055Z" }, + { url = "https://files.pythonhosted.org/packages/88/83/5eca144942d8d0659d3f55176517f4a59cdc65eefd17146a0770935a3ebd/coverage-7.15.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:67d7602480a47bdf5b675635403625553ebaa70d5a62a657c035149fd401cea0", size = 256352, upload-time = "2026-07-15T18:53:58.83Z" }, + { url = "https://files.pythonhosted.org/packages/4e/ba/d3db2e01a50fc88cdb4c0f19542bcf6f61489e34dc9aa3538413e2459a38/coverage-7.15.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cee0f89f4767a6057c8fbf168f8135f18be651300496086bd873e3189fed0487", size = 258313, upload-time = "2026-07-15T18:54:00.497Z" }, + { url = "https://files.pythonhosted.org/packages/78/b3/aba83416e9177df28e5186d856c19158c59fc0e7e814aaa61a4a2354ad1b/coverage-7.15.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a29ec5305a7335aacee2d799e3422e91e1c8a12474986e2b3b07e315c91be82f", size = 252449, upload-time = "2026-07-15T18:54:02.456Z" }, + { url = "https://files.pythonhosted.org/packages/6a/a5/4b00ecac0194431ab451b0f6710f8e2517d04cef60f821b14dec4637d575/coverage-7.15.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:48ccc6395958eda89093ecdc35644c86f23a8b23a7f4d44958812b721aad67c1", size = 254043, upload-time = "2026-07-15T18:54:04.072Z" }, + { url = "https://files.pythonhosted.org/packages/75/b6/cfa209b4313ee7f1b34da47efcd789ea51c024ad35af390e00f5a3c10a2e/coverage-7.15.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:81f382c5a94b434ec1f6da607edb904c76d7212e618cd4d1bc9f97bed4120ef5", size = 252107, upload-time = "2026-07-15T18:54:06.745Z" }, + { url = "https://files.pythonhosted.org/packages/36/67/e8cac5a6954038c98d7fe7eb9802afe7ab3ecb637bb7cc00e69b4148b56d/coverage-7.15.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:bbc808daf4f5cd567af8075ecc72d21c6dfef9a254709a621a84c217c935ebc0", size = 255873, upload-time = "2026-07-15T18:54:08.48Z" }, + { url = "https://files.pythonhosted.org/packages/2c/92/395cca9f330a86c3fe3471d73e2c102116c4c58fdc619dbbc125c6e93a54/coverage-7.15.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:a4c46b247b5d4b78f613bd89fea926d32b25c6cc61a50bd1e99ba310348f3dad", size = 251826, upload-time = "2026-07-15T18:54:10.083Z" }, + { url = "https://files.pythonhosted.org/packages/51/60/3e91b20295439652424f426b7086ec5bf4fbe3f604c73eda22b986c4fd6b/coverage-7.15.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:094dd37f3ef7b2da8b068b583d1f4c40f91c65197e16c52a71962d5d537fc5db", size = 252735, upload-time = "2026-07-15T18:54:11.878Z" }, + { url = "https://files.pythonhosted.org/packages/a5/eb/8c07839005e5e3c6b3877d3a6e2a80ce766589f31dd2b6882b78d59a7b8c/coverage-7.15.2-cp311-cp311-win32.whl", hash = "sha256:a63b9e190711134d581c4d703df5df09851b1acf99792c7aacbbe9f41f0283c9", size = 223500, upload-time = "2026-07-15T18:54:13.525Z" }, + { url = "https://files.pythonhosted.org/packages/2e/98/59d83c257cd59f0fbaf9d9ddb26b744a576760dfd1ae16e516408894a02b/coverage-7.15.2-cp311-cp311-win_amd64.whl", hash = "sha256:8bb9f4b4279187560796a4cdaca3b0a93dd97e48ee667df005f4ed9a97403688", size = 223973, upload-time = "2026-07-15T18:54:15.163Z" }, + { url = "https://files.pythonhosted.org/packages/ea/09/2d285c8bef5c4f695d120c1c96dc11715638aa8e134069f210bb6a62a9fe/coverage-7.15.2-cp311-cp311-win_arm64.whl", hash = "sha256:8c726b232659cbd2ae57ade46509eb068c9bd7a06df9fcbff6fe484870006934", size = 223519, upload-time = "2026-07-15T18:54:16.803Z" }, + { url = "https://files.pythonhosted.org/packages/6a/50/eb5bf42e531611a9f8d272556b1ed4de503f84a91413584094487cf69f8f/coverage-7.15.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:1adac78e5abc7c5438f7a209c9ca69d06542f0bf481d728b6989ea80b813fdf9", size = 221587, upload-time = "2026-07-15T18:54:18.439Z" }, + { url = "https://files.pythonhosted.org/packages/06/d1/da99af464c335d4e023a6efcd7ec30f63b88a43c93745154ab74ffb31cea/coverage-7.15.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b868acc62aa5de3be7a9d05c2333bf8359ca987e43f9cb30ff8fbda6a024ab73", size = 221943, upload-time = "2026-07-15T18:54:20.062Z" }, + { url = "https://files.pythonhosted.org/packages/5b/8a/13c42723d61ca447eafa18732e8141dd6a63f2732e1c7e1502c182dd88d7/coverage-7.15.2-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:6f6966fc30e6f06ca8f98fb0ce51eda6b111b3ee8d066a8b1ec9e77fa06ab55d", size = 253450, upload-time = "2026-07-15T18:54:21.765Z" }, + { url = "https://files.pythonhosted.org/packages/d7/29/99021303f98fbdcb63504b4d07bea4cc025b9b2dd907c4f07c85d50a0dab/coverage-7.15.2-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:68af907f595ab01a78f794932ff3bdf929c316d3000810d38dbc247129e26f8b", size = 256187, upload-time = "2026-07-15T18:54:23.4Z" }, + { url = "https://files.pythonhosted.org/packages/f9/a8/fd503715ed6ca9c5d742923aa5209257340b367a867b2ced0c7d4ba8a0b9/coverage-7.15.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:afa29e2eff3d5729267e2cb2fd4ce9d61c952932fb2694e34ccb5d9540c6a296", size = 257301, upload-time = "2026-07-15T18:54:25.183Z" }, + { url = "https://files.pythonhosted.org/packages/da/40/3f4b8fb409810036ebc2857d36adc0498c6e957b5df0290c5036b2e143f1/coverage-7.15.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bbf44513ceb1589e31948e20eafbde9deaface90e1a1afa5f5f77b4423d17ce6", size = 259562, upload-time = "2026-07-15T18:54:27.204Z" }, + { url = "https://files.pythonhosted.org/packages/0b/8a/9bdffbef47db77cce3d6b02a28f7e919b19f0106c4b080c2c2246040f885/coverage-7.15.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9deddf09eecb717b7f980414b43d90a5b22ff3967d2949ab29cb0aa83d9e9098", size = 253841, upload-time = "2026-07-15T18:54:29.134Z" }, + { url = "https://files.pythonhosted.org/packages/1b/1e/9031efde019d31a06646261fce6dfc5c3c74e951e27a71e5c9a424563178/coverage-7.15.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ae901f7e55ba405c84ee1cab3d3e962e4e871e4a2bcb9c90911adbd69b42ac5a", size = 255221, upload-time = "2026-07-15T18:54:31.142Z" }, + { url = "https://files.pythonhosted.org/packages/56/db/787acde872389fc84a9ef9d8cd1ccc658e391ab4cb5b28092a714426a394/coverage-7.15.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a0f47002c6eeb7c280228467a4cb0cc15ca2103a8421b986b2d3ec04a0f9bd8b", size = 253366, upload-time = "2026-07-15T18:54:32.886Z" }, + { url = "https://files.pythonhosted.org/packages/2f/9b/6f57bc4b93c842eef1695f8cdaf2318e35e7ba54f5ba80d84be213ab7858/coverage-7.15.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1cd7a5beb7af3e864a13b1f0fb26efd3695da43ef0daf71e586adfffaf34d5b2", size = 257434, upload-time = "2026-07-15T18:54:34.7Z" }, + { url = "https://files.pythonhosted.org/packages/88/26/b3186a21b2acc83e451118978905c81c7072c3333707804db09a78c096a2/coverage-7.15.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:97a5c5457a9fb1d6c4e06cfb5dc835871fbfb6a6a51addc9e925bdeff5ef7440", size = 252935, upload-time = "2026-07-15T18:54:36.548Z" }, + { url = "https://files.pythonhosted.org/packages/20/c2/c9f3376b2e717ea69ed7a6e9a5fcab968fb0b290db6cf4bd9a1fc7541b75/coverage-7.15.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0901cfe6c13bcd2302da4f83e884555d2a22bda6e4c476f09ef204ba20ca536e", size = 254807, upload-time = "2026-07-15T18:54:38.296Z" }, + { url = "https://files.pythonhosted.org/packages/f0/e1/dfc15401f4a8aaeb486e1ba3e9e3c40522a6e38bd0ecf0b3f29cb8082957/coverage-7.15.2-cp312-cp312-win32.whl", hash = "sha256:b171bdd71cb7ff792bf32e376173b0ace7e7963e7e57c58dfc42063a6a7174cd", size = 223641, upload-time = "2026-07-15T18:54:40.103Z" }, + { url = "https://files.pythonhosted.org/packages/91/40/81b6d809d320cd366ec5bdf8176575e897dcb8efe7fb4b489ef9e93e4d13/coverage-7.15.2-cp312-cp312-win_amd64.whl", hash = "sha256:582edc45c2040543fef83341be23c43024a3ab3ae0c2d8bc498a06282905ad40", size = 224172, upload-time = "2026-07-15T18:54:41.882Z" }, + { url = "https://files.pythonhosted.org/packages/ef/28/9f14ec438149f7de557f45518f09b4a7917b795cc37083aa7db482693f8c/coverage-7.15.2-cp312-cp312-win_arm64.whl", hash = "sha256:a638db90c61cd219aeee65e83a24fdaa57269a741ae0cf773309208ac862cee3", size = 223556, upload-time = "2026-07-15T18:54:43.674Z" }, + { url = "https://files.pythonhosted.org/packages/fc/d5/f8c838e6b7282976f7c918884b792df7a0c42c5bba5d99c60ad2d221d56d/coverage-7.15.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1121caa19159a38b5463eaae4b1e1fde81e525b15ecc5e000cd5b1a108f743a8", size = 221606, upload-time = "2026-07-15T18:54:45.448Z" }, + { url = "https://files.pythonhosted.org/packages/bf/37/97c926376364f66298cc44893b89cdf17b8bc406376497c4061ae4b8a8ff/coverage-7.15.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a300c6934e0989c327b9e8a1e110329da4641149f872bbe9f70168be66da76c1", size = 221982, upload-time = "2026-07-15T18:54:47.341Z" }, + { url = "https://files.pythonhosted.org/packages/b7/30/a36050a6e83c2135ee0776f452ca3948224befc6d7f26acecc082d0c106a/coverage-7.15.2-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2617f8799d268fabdeef42a7e89ac3a23e1deee9025427db2df970f99a89a578", size = 252972, upload-time = "2026-07-15T18:54:49.2Z" }, + { url = "https://files.pythonhosted.org/packages/31/d3/06b5f1daf95f0f15ab05bd75f26ba5f3c8b33d0bb72f3aaa3cf41d1bad3a/coverage-7.15.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7dc2950a2992cd676d35c20ae63522836deeb034f08874699d14068710af3dc1", size = 255569, upload-time = "2026-07-15T18:54:51.098Z" }, + { url = "https://files.pythonhosted.org/packages/81/1c/9afb3f8de2b8d36960391c48559a2e3ff96594b58099f115921549ea8d0d/coverage-7.15.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9e36686f7a442185db2400b3df171aac520869faf9deb59df687d28659eda2a6", size = 256806, upload-time = "2026-07-15T18:54:53.145Z" }, + { url = "https://files.pythonhosted.org/packages/64/d8/b989f96061a5e32d82fddd1b1b9ff48a7c8f8ae7606f0e80fd9de54b1e33/coverage-7.15.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7d29ca7bd67af6e12e74632d65f026eabc1364da5c254494cd914446a28a3ef7", size = 258936, upload-time = "2026-07-15T18:54:55.015Z" }, + { url = "https://files.pythonhosted.org/packages/b8/fa/f99771f5110457c7b511c1935ca49ddf288218eaa84322e028b9334146ae/coverage-7.15.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:db9c8438057e5b0f6a22a0af99c0c1d26b57fbbdbd1be5861ddb8f897fcc3a2d", size = 253178, upload-time = "2026-07-15T18:54:57.527Z" }, + { url = "https://files.pythonhosted.org/packages/f6/96/c098a6044d119c751ceede7be91035fa8310170ec24a6523aff72f0a5793/coverage-7.15.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:63022c4c8dec1d0342f05c3ede99842fe3d007689acc45e86f123a1746e4a026", size = 254934, upload-time = "2026-07-15T18:54:59.41Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a2/1457b3a7a50c8d77500103b97a046db863e2f59a1cf6d2f814595f349885/coverage-7.15.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:6c0be82b4d4aa5b2704e08518e2252f3e3d110164bcca826816801052e48a7aa", size = 252898, upload-time = "2026-07-15T18:55:01.338Z" }, + { url = "https://files.pythonhosted.org/packages/6c/0e/76958874c471ecfcdde0d2b2747bb2c61bdbf34a40636f4ce9db9923e643/coverage-7.15.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4510fb9cdf6bb02dfa6af0be4a534b8102d086e22e4a33f8836df663da3d660d", size = 257056, upload-time = "2026-07-15T18:55:03.243Z" }, + { url = "https://files.pythonhosted.org/packages/7c/7c/3d7c4e3bf58baa40327dc7edc2272b17cf02299366d52763db1b0ca1556a/coverage-7.15.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:42ec3d989421b174a2ab607c1539f24127ad362757b7f1c0c0d7a2993f7eb37b", size = 252718, upload-time = "2026-07-15T18:55:05.029Z" }, + { url = "https://files.pythonhosted.org/packages/c8/b8/1cecffed9ce14fb25be9ba42d37b6bb61485c9a3ddd43cd3dde36b6087d8/coverage-7.15.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e8f91bce78e32343af184c3b7fa28fcf5a9e2641f4b6623d392038f804939188", size = 254490, upload-time = "2026-07-15T18:55:06.889Z" }, + { url = "https://files.pythonhosted.org/packages/6c/2c/42984561bc7f4c045dca67516a0c50ee5ef8d84352dbeb5559dc86c4823e/coverage-7.15.2-cp313-cp313-win32.whl", hash = "sha256:434e68d531858205895eb0d74b73d20b84260de426387d53c422a5acda2cf050", size = 223647, upload-time = "2026-07-15T18:55:08.941Z" }, + { url = "https://files.pythonhosted.org/packages/41/9f/39c7c9245efc583beddf89a87683574e663ed93637f3afb6cd7b88405676/coverage-7.15.2-cp313-cp313-win_amd64.whl", hash = "sha256:26c3b04a6377fd7c09800921fa934e3a17c0020439cd59df73e73ae1d4b6a78c", size = 224190, upload-time = "2026-07-15T18:55:10.789Z" }, + { url = "https://files.pythonhosted.org/packages/c7/de/3a2883cf8a213659280ef4b403059e17a9acaeb7fc7fd4105e1226ff2e6d/coverage-7.15.2-cp313-cp313-win_arm64.whl", hash = "sha256:3ed010aa1b69cda8e827aabfca9866216c980e2dca82ab9a78c5f83689964c8b", size = 223583, upload-time = "2026-07-15T18:55:12.678Z" }, + { url = "https://files.pythonhosted.org/packages/81/5f/aed265fd7a3551a394f36dfe41868aee709b7f95db4052205b4ad1563ac3/coverage-7.15.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:40f633c5c5fc783732f6312280122e859538fa24461235597c13d803ea9a108a", size = 221650, upload-time = "2026-07-15T18:55:14.527Z" }, + { url = "https://files.pythonhosted.org/packages/6b/2c/222ba12a545189017120f8eddfc1a0bd4616b47d5d4a8d99421edb2fe4c6/coverage-7.15.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:075560438765b7a2ef43bf7aa7758661b53d889df47f062a31bda6c1ade553a2", size = 221988, upload-time = "2026-07-15T18:55:16.674Z" }, + { url = "https://files.pythonhosted.org/packages/aa/38/304b5877ab46e6c290b4292cfcf3fe28245f0e5597cad7f6acc91fc7e0a4/coverage-7.15.2-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:25fd15dd40a0a2c51a500d664ca29053c09c3259d998407bf982b6e114696138", size = 253029, upload-time = "2026-07-15T18:55:18.856Z" }, + { url = "https://files.pythonhosted.org/packages/6c/58/821b533b8db9e44cf1d8a97bd525149ced40dde1d0093da02cb78e715244/coverage-7.15.2-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b9a6367e4aff723e8ee8190836836124284e8fcd4265e307c844010cfa074f3f", size = 255536, upload-time = "2026-07-15T18:55:21.027Z" }, + { url = "https://files.pythonhosted.org/packages/f1/f2/7aa06604c389d32ea7f0a6a988359a7eafc3cd3f8e7bc2e88cd2fdf0b877/coverage-7.15.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9854ca62c152874b2060772503535be2e8f53f70b8aaa7686b094888d872f984", size = 256881, upload-time = "2026-07-15T18:55:23.125Z" }, + { url = "https://files.pythonhosted.org/packages/a2/4f/1ef342339c7916d0096bc5888cc0f653882cc7bc8f897d5cb89143287c9b/coverage-7.15.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:913b6c56e110da40e035bbd168353bf7aaa2544a5eaccea5d98a4629aac156c7", size = 259196, upload-time = "2026-07-15T18:55:25.099Z" }, + { url = "https://files.pythonhosted.org/packages/fe/f4/7ed055d7a9c5ec13b161773a115a5ccc6b0081d568c31fad830806306cc7/coverage-7.15.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aaccad4129d735a8a4d526f26929894c9a4e8ef7034566f210b176749d6906e3", size = 253036, upload-time = "2026-07-15T18:55:27.018Z" }, + { url = "https://files.pythonhosted.org/packages/14/79/ea82cca18c242a3a38b6c017da39726aa62dcb64aa635abf79b92009975c/coverage-7.15.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a164b50081fc7357331c4024ef4d17b78ba325f8380d05f5a69599a7e05257ee", size = 254887, upload-time = "2026-07-15T18:55:29.084Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ba/a136db3c0d9562b00e10b72540dbf3a33cd3bc5b95060c9308e247494623/coverage-7.15.2-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:bfd341ccf78128e72c094bc70cc25b3ef309c33c7c2c66ba3ed4309549e02de1", size = 252852, upload-time = "2026-07-15T18:55:31.184Z" }, + { url = "https://files.pythonhosted.org/packages/17/17/ea334246b16b7d059953fad6fdefa11e33c68efbd3fe37b1098120a1fac2/coverage-7.15.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:1473b3ba8e7ee0f076117b1a72c23f579a2b9e2bb742f48a8d86ea27ca93f91a", size = 257128, upload-time = "2026-07-15T18:55:33.163Z" }, + { url = "https://files.pythonhosted.org/packages/ed/c3/074fb66d46d607855f710876b117cbda562c5ab08363528e78820449f937/coverage-7.15.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:17c432b5f73ad52ef46fb06019f6fa7c66ce381961cf0f7dfd1d3a4bd3a98145", size = 252668, upload-time = "2026-07-15T18:55:35.063Z" }, + { url = "https://files.pythonhosted.org/packages/e1/c1/f620850ada9b36435921c9a3a8057013422b1d964eb4bf37fe138724d192/coverage-7.15.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:77f0ef5011df53a4bd1b35211ab122287f8d9b8d7aa1c4553e5c2deb24b1d446", size = 254325, upload-time = "2026-07-15T18:55:37.125Z" }, + { url = "https://files.pythonhosted.org/packages/cc/31/a729ca3689404493af82ef8e6ff70bd88bdda8da89aeef6ca9b387aeb2b4/coverage-7.15.2-cp314-cp314-win32.whl", hash = "sha256:f653e5d7248c1191ec988a85c72edeab46c3ff44f90639a4ed4874ec0be90243", size = 223844, upload-time = "2026-07-15T18:55:39.078Z" }, + { url = "https://files.pythonhosted.org/packages/c6/83/5d809dc808fb1698c671f3e372259bb9158e64b7ea526fc6ab7de64de9fe/coverage-7.15.2-cp314-cp314-win_amd64.whl", hash = "sha256:9911f31aad8906abe337c271343485cf20df5e70df5d2f57f9f136e7b55f26bc", size = 224331, upload-time = "2026-07-15T18:55:41.346Z" }, + { url = "https://files.pythonhosted.org/packages/16/4e/35e488548e952795829e129995c4174df33bf432b591d1aa42c8d9e4e7ad/coverage-7.15.2-cp314-cp314-win_arm64.whl", hash = "sha256:e38def96ad59853824c97953fdcd2c320a84ba3ce99b417db78af8bb6c3db635", size = 223760, upload-time = "2026-07-15T18:55:43.518Z" }, + { url = "https://files.pythonhosted.org/packages/ed/49/dd2c86cd6374038f6e415fb5bfb86db5218553209c081384a020369dee79/coverage-7.15.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:835ec4e20b45f0a7f63ed78f94065aca00de033403df8377bfe8b9c6abc0a7be", size = 222384, upload-time = "2026-07-15T18:55:45.569Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/173ff17a1c0808e5a438f549f6f145d5ac7528f2791310b63523e3200ac7/coverage-7.15.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7466cc7ab6dc0db871d264bf99e8779f0917ee63d40730af0552f71535a6e072", size = 222647, upload-time = "2026-07-15T18:55:47.544Z" }, + { url = "https://files.pythonhosted.org/packages/84/f8/b8cba872162356fb44ac79c10309d987206a4461e32072fc29228dad7331/coverage-7.15.2-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:e370c12133095ff18432de8c044962be85a5a96d90c6fcbce8e17e76236d2328", size = 264013, upload-time = "2026-07-15T18:55:49.768Z" }, + { url = "https://files.pythonhosted.org/packages/ee/67/a807a7586d0b8cae485308ddd55756f0806c92f8e0b411bacbf23c48edf3/coverage-7.15.2-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fe41909c9515c3bfdb5f02c4d1f857dba322d9a9a1178069b91eea77889df63a", size = 266135, upload-time = "2026-07-15T18:55:51.941Z" }, + { url = "https://files.pythonhosted.org/packages/ce/67/cd78771dc985f7e4ebdcc82b1a96d9a932af9e806f01f2f91a89f4c72e80/coverage-7.15.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6aa28cfb6488e5453b5b762d65f73aa586380f6693a04d58078ce228a29b06c0", size = 268555, upload-time = "2026-07-15T18:55:54.065Z" }, + { url = "https://files.pythonhosted.org/packages/18/3e/10134cf81275188c58568f324fc74aedff32c63ca4d5bbc513a91944a6f0/coverage-7.15.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bcc0aae933921d03096f53b0b03eeb702129fd406dee59f08d2efacc68681fa5", size = 269674, upload-time = "2026-07-15T18:55:56.066Z" }, + { url = "https://files.pythonhosted.org/packages/75/4a/771b77de446cba985dc414bbc5844bd21604da05dbc044286df8318a48a7/coverage-7.15.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7c63387e21ab21f512c69c9756a8c7dadd322c7275edb064064433c9a09c3743", size = 263101, upload-time = "2026-07-15T18:55:58.107Z" }, + { url = "https://files.pythonhosted.org/packages/5f/b5/70a7011da15f4071943361183aefa27847f3e3aec4fd335f1cb3d3a622b1/coverage-7.15.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0e55510bc98ae943cece9e667a6c0fe94c6a92913720dea34243657a17993d0c", size = 266007, upload-time = "2026-07-15T18:56:00.468Z" }, + { url = "https://files.pythonhosted.org/packages/b4/0d/f9547e804ce7ad49646ffeffac26699510efbe6c0f751b66fdc960c4e825/coverage-7.15.2-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:2ff08701be2d1556fc78b326c80a3e8042da09352ecb3819105f8e386c8a3071", size = 263611, upload-time = "2026-07-15T18:56:02.615Z" }, + { url = "https://files.pythonhosted.org/packages/ac/59/f576a396659c0efd351f5c1544f67c3560e89c7761cabf7f65e412beeda5/coverage-7.15.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:38c9518b7103826c403a461544e3c2e77151e8676d06eaed85911a97e962584a", size = 267344, upload-time = "2026-07-15T18:56:04.622Z" }, + { url = "https://files.pythonhosted.org/packages/7c/5d/c2e4fce3579c0cb635024293f1a32bbe26df101b3e3a69f22243d1352b6c/coverage-7.15.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:dee88b1ed88587abd8c0269a1fc1f4cc77f7750d1dfde2869e2a123af420e67d", size = 262456, upload-time = "2026-07-15T18:56:06.641Z" }, + { url = "https://files.pythonhosted.org/packages/bb/dd/956287d69436b66094bc4b57ac2da71e43bfd2a5524e958900b9f582fcf8/coverage-7.15.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2fbeeeecea279727f8ac16c8e1133ddfeee793e985c86ae343d6a5ce744eef8c", size = 264771, upload-time = "2026-07-15T18:56:08.795Z" }, + { url = "https://files.pythonhosted.org/packages/2c/5a/6f979530c2734c575de77cf58f5f28d51f7123a94b5030fd9156fe5f363c/coverage-7.15.2-cp314-cp314t-win32.whl", hash = "sha256:cb0fddaa6884be6aae36ced9544b5e90f7d5f03845a2853bf47a14953a4e8688", size = 224151, upload-time = "2026-07-15T18:56:10.856Z" }, + { url = "https://files.pythonhosted.org/packages/54/7e/27f6b2a74d484742f4017553e710b01e396b23d809df3e95ca0bb9a2824b/coverage-7.15.2-cp314-cp314t-win_amd64.whl", hash = "sha256:77f091ea3a9cc611cd29f433565476bc1936c084ac8eee00ea0e7e70c27e4199", size = 224981, upload-time = "2026-07-15T18:56:12.928Z" }, + { url = "https://files.pythonhosted.org/packages/b1/48/284863423aa474240f6842bd00d680da22f4e6ea2e466618ef7c9c9e69a9/coverage-7.15.2-cp314-cp314t-win_arm64.whl", hash = "sha256:6fc448c377d6eeb00a47c673494bd9bae29280ca53987e1869e67ebedfe20658", size = 224294, upload-time = "2026-07-15T18:56:15.156Z" }, + { url = "https://files.pythonhosted.org/packages/ec/82/32e3bd191d498e64f6f911ad55d14006a0861e54869d2d32452326399e65/coverage-7.15.2-py3-none-any.whl", hash = "sha256:eb6bcae8d1a9d305351ecb108232441d11c5cfe9de840a04388ba5d2db8d735c", size = 213375, upload-time = "2026-07-15T18:56:17.305Z" }, +] + +[package.optional-dependencies] +toml = [ + { name = "tomli", marker = "python_full_version <= '3.11'" }, +] + +[[package]] +name = "cuda-bindings" +version = "13.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-pathfinder" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/6b/457ca12dad3ee9bfcc9a545cfd6b64b359ba49de40f776f6e028e678f262/cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474", size = 6053539, upload-time = "2026-05-29T23:11:43.19Z" }, + { url = "https://files.pythonhosted.org/packages/95/7a/c5e3c34a409b148f5c0f5a4ea374158f95d488862c1dffedf9aa5c639df9/cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04436a9364059c84b8f9636f359eccda1cf814341f5b670c71d80d2f79dbc708", size = 6674166, upload-time = "2026-05-29T23:11:45.478Z" }, + { url = "https://files.pythonhosted.org/packages/ce/67/5e7dba1ba576dd73da5dee894ca076ca5e959450dfff66d6d510a255d1f7/cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49", size = 6025351, upload-time = "2026-05-29T23:11:49.685Z" }, + { url = "https://files.pythonhosted.org/packages/39/2a/6d2e9047d1fb243dbaa364b01e0297534b9ed7fd27dba1c9f361519cf69b/cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a", size = 6657965, upload-time = "2026-05-29T23:11:52.227Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6e/2394f8163360f8391f8f1b7e72d300a82724edb81a7b7084c799fbd4c91f/cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9efb21c1ee64981e184b9e0ba5eb3179e5ba3d4b51665a6cb52b8ef3d01a7cbf", size = 5920504, upload-time = "2026-05-29T23:11:56.883Z" }, + { url = "https://files.pythonhosted.org/packages/34/c2/ef9b6a63f7dc432712a462c816662e662e00d38caa9b861c8c2588195d03/cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2732904099e0a4d4db774a5fc6d91ee95fae065b4d2ecabb4968c5fe2406c9d7", size = 6476660, upload-time = "2026-05-29T23:11:59.188Z" }, + { url = "https://files.pythonhosted.org/packages/b1/81/bff68ce829999c1e4209c761bbf903b1c06ec570416ddb25020864ad5907/cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ab2f74ed65bfef4163ba07a8db16f1085e0729291db12a2423aff84ee8278b8", size = 6013639, upload-time = "2026-05-29T23:12:03.509Z" }, + { url = "https://files.pythonhosted.org/packages/d4/e0/c8a1f0c8f9ffdea4f5fe6dbab89b326cef4d85caf489dad39e209da89416/cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd4c814d311ec08c981f6dded1dbe7d4b371067ee4f6c14cccec4bde9590f80", size = 6534419, upload-time = "2026-05-29T23:12:05.633Z" }, + { url = "https://files.pythonhosted.org/packages/52/b8/83b1f563925b290f2d11a01a77a84013ba56052fe3653a5bef3ccfbb43d6/cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3c772dfff49681541d59630c90f858e173ac926b9c593a2b7123f2a1043cc76", size = 5809771, upload-time = "2026-05-29T23:12:10.422Z" }, + { url = "https://files.pythonhosted.org/packages/12/20/e79b4bfe98f075195afb6343d41c498f9dbd2d161d7021d4d28bceb83581/cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36febb7c1079d68a981dbbd8d5a67235b399802b82075c9388624719607e52b9", size = 6358584, upload-time = "2026-05-29T23:12:12.767Z" }, +] + +[[package]] +name = "cuda-pathfinder" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/b4/d088047afe39827556df21118cac9ffd20cc3f968c99a7681494d1eb333c/cuda_pathfinder-1.6.0-py3-none-any.whl", hash = "sha256:1503af579d8379c24bdd65528379bc57039b0455be9f5f9686cf8e473a1fce51", size = 54591, upload-time = "2026-07-21T15:03:56.224Z" }, +] + +[[package]] +name = "cuda-toolkit" +version = "13.0.3.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/c7/a79086a62c98befcdb8349656c6f114e2db3b8b2422f6e25c97a7f2a9a3c/cuda_toolkit-13.0.3.0-py2.py3-none-any.whl", hash = "sha256:d693caaa261214ddd7dbb60d68e71cbed884e68c2be7509778f3051da0b91c3f", size = 2512, upload-time = "2026-04-14T00:50:08.173Z" }, +] + +[package.optional-dependencies] +cublas = [ + { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +cudart = [ + { name = "nvidia-cuda-runtime", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +cufft = [ + { name = "nvidia-cufft", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +cufile = [ + { name = "nvidia-cufile", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +cupti = [ + { name = "nvidia-cuda-cupti", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +curand = [ + { name = "nvidia-curand", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +cusolver = [ + { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cusolver", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +cusparse = [ + { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +nvjitlink = [ + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +nvrtc = [ + { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] +nvtx = [ + { name = "nvidia-nvtx", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, +] + +[[package]] +name = "datasets" +version = "4.8.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dill" }, + { name = "filelock" }, + { name = "fsspec", extra = ["http"] }, + { name = "httpx" }, + { name = "huggingface-hub" }, + { name = "multiprocess" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "packaging" }, + { name = "pandas" }, + { name = "pyarrow" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "tqdm" }, + { name = "xxhash" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/34/14cd8e76f907f7d4dca2334cfeec9f81d30fd15c25a015f99aaea694eaed/datasets-4.8.5.tar.gz", hash = "sha256:0f0c1c3d56ffff2c93b2f4c63c95bac94f3d7e8621aea2a2a576275233bba772", size = 605649, upload-time = "2026-04-27T15:43:57.384Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/65/99/00f3196036501b53032c4b1ab8337a0b978dee832ed276dae3815df4e8b5/datasets-4.8.5-py3-none-any.whl", hash = "sha256:5079900781719c0e063a8efdd2cd95a31ad0c63209178669cd23cf1b926149ff", size = 528973, upload-time = "2026-04-27T15:43:53.702Z" }, +] + +[[package]] +name = "dill" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/81/e1/56027a71e31b02ddc53c7d65b01e68edf64dea2932122fe7746a516f75d5/dill-0.4.1.tar.gz", hash = "sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa", size = 187315, upload-time = "2026-01-19T02:36:56.85Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, +] + +[[package]] +name = "filelock" +version = "3.32.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/57/3ba6e6cb097f85b855b00163d169f35365f44277df044dcf96d55b8f62a3/filelock-3.32.2.tar.gz", hash = "sha256:c33351e1f49cae33414acbc6d56784e6ecee82514ec90795da1161fc4836b5b8", size = 217172, upload-time = "2026-07-29T22:46:04.895Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/e8/72f8cef9fdfeffe06213fe8508039396ee48daa0e3259457ed766173bfd6/filelock-3.32.2-py3-none-any.whl", hash = "sha256:87dd94cf281e586d135fa51132b8e3d9a598b316e90377a288663c9321036c82", size = 98830, upload-time = "2026-07-29T22:46:03.52Z" }, +] + +[[package]] +name = "frozenlist" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875, upload-time = "2025-10-06T05:38:17.865Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/03/077f869d540370db12165c0aa51640a873fb661d8b315d1d4d67b284d7ac/frozenlist-1.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84", size = 86912, upload-time = "2025-10-06T05:35:45.98Z" }, + { url = "https://files.pythonhosted.org/packages/df/b5/7610b6bd13e4ae77b96ba85abea1c8cb249683217ef09ac9e0ae93f25a91/frozenlist-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9", size = 50046, upload-time = "2025-10-06T05:35:47.009Z" }, + { url = "https://files.pythonhosted.org/packages/6e/ef/0e8f1fe32f8a53dd26bdd1f9347efe0778b0fddf62789ea683f4cc7d787d/frozenlist-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93", size = 50119, upload-time = "2025-10-06T05:35:48.38Z" }, + { url = "https://files.pythonhosted.org/packages/11/b1/71a477adc7c36e5fb628245dfbdea2166feae310757dea848d02bd0689fd/frozenlist-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f", size = 231067, upload-time = "2025-10-06T05:35:49.97Z" }, + { url = "https://files.pythonhosted.org/packages/45/7e/afe40eca3a2dc19b9904c0f5d7edfe82b5304cb831391edec0ac04af94c2/frozenlist-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695", size = 233160, upload-time = "2025-10-06T05:35:51.729Z" }, + { url = "https://files.pythonhosted.org/packages/a6/aa/7416eac95603ce428679d273255ffc7c998d4132cfae200103f164b108aa/frozenlist-1.8.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52", size = 228544, upload-time = "2025-10-06T05:35:53.246Z" }, + { url = "https://files.pythonhosted.org/packages/8b/3d/2a2d1f683d55ac7e3875e4263d28410063e738384d3adc294f5ff3d7105e/frozenlist-1.8.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581", size = 243797, upload-time = "2025-10-06T05:35:54.497Z" }, + { url = "https://files.pythonhosted.org/packages/78/1e/2d5565b589e580c296d3bb54da08d206e797d941a83a6fdea42af23be79c/frozenlist-1.8.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567", size = 247923, upload-time = "2025-10-06T05:35:55.861Z" }, + { url = "https://files.pythonhosted.org/packages/aa/c3/65872fcf1d326a7f101ad4d86285c403c87be7d832b7470b77f6d2ed5ddc/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b", size = 230886, upload-time = "2025-10-06T05:35:57.399Z" }, + { url = "https://files.pythonhosted.org/packages/a0/76/ac9ced601d62f6956f03cc794f9e04c81719509f85255abf96e2510f4265/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92", size = 245731, upload-time = "2025-10-06T05:35:58.563Z" }, + { url = "https://files.pythonhosted.org/packages/b9/49/ecccb5f2598daf0b4a1415497eba4c33c1e8ce07495eb07d2860c731b8d5/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d", size = 241544, upload-time = "2025-10-06T05:35:59.719Z" }, + { url = "https://files.pythonhosted.org/packages/53/4b/ddf24113323c0bbcc54cb38c8b8916f1da7165e07b8e24a717b4a12cbf10/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd", size = 241806, upload-time = "2025-10-06T05:36:00.959Z" }, + { url = "https://files.pythonhosted.org/packages/a7/fb/9b9a084d73c67175484ba2789a59f8eebebd0827d186a8102005ce41e1ba/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967", size = 229382, upload-time = "2025-10-06T05:36:02.22Z" }, + { url = "https://files.pythonhosted.org/packages/95/a3/c8fb25aac55bf5e12dae5c5aa6a98f85d436c1dc658f21c3ac73f9fa95e5/frozenlist-1.8.0-cp311-cp311-win32.whl", hash = "sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25", size = 39647, upload-time = "2025-10-06T05:36:03.409Z" }, + { url = "https://files.pythonhosted.org/packages/0a/f5/603d0d6a02cfd4c8f2a095a54672b3cf967ad688a60fb9faf04fc4887f65/frozenlist-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b", size = 44064, upload-time = "2025-10-06T05:36:04.368Z" }, + { url = "https://files.pythonhosted.org/packages/5d/16/c2c9ab44e181f043a86f9a8f84d5124b62dbcb3a02c0977ec72b9ac1d3e0/frozenlist-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a", size = 39937, upload-time = "2025-10-06T05:36:05.669Z" }, + { url = "https://files.pythonhosted.org/packages/69/29/948b9aa87e75820a38650af445d2ef2b6b8a6fab1a23b6bb9e4ef0be2d59/frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1", size = 87782, upload-time = "2025-10-06T05:36:06.649Z" }, + { url = "https://files.pythonhosted.org/packages/64/80/4f6e318ee2a7c0750ed724fa33a4bdf1eacdc5a39a7a24e818a773cd91af/frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b", size = 50594, upload-time = "2025-10-06T05:36:07.69Z" }, + { url = "https://files.pythonhosted.org/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4", size = 50448, upload-time = "2025-10-06T05:36:08.78Z" }, + { url = "https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", size = 242411, upload-time = "2025-10-06T05:36:09.801Z" }, + { url = "https://files.pythonhosted.org/packages/8f/83/f61505a05109ef3293dfb1ff594d13d64a2324ac3482be2cedc2be818256/frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4", size = 243014, upload-time = "2025-10-06T05:36:11.394Z" }, + { url = "https://files.pythonhosted.org/packages/d8/cb/cb6c7b0f7d4023ddda30cf56b8b17494eb3a79e3fda666bf735f63118b35/frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8", size = 234909, upload-time = "2025-10-06T05:36:12.598Z" }, + { url = "https://files.pythonhosted.org/packages/31/c5/cd7a1f3b8b34af009fb17d4123c5a778b44ae2804e3ad6b86204255f9ec5/frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b", size = 250049, upload-time = "2025-10-06T05:36:14.065Z" }, + { url = "https://files.pythonhosted.org/packages/c0/01/2f95d3b416c584a1e7f0e1d6d31998c4a795f7544069ee2e0962a4b60740/frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52", size = 256485, upload-time = "2025-10-06T05:36:15.39Z" }, + { url = "https://files.pythonhosted.org/packages/ce/03/024bf7720b3abaebcff6d0793d73c154237b85bdf67b7ed55e5e9596dc9a/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29", size = 237619, upload-time = "2025-10-06T05:36:16.558Z" }, + { url = "https://files.pythonhosted.org/packages/69/fa/f8abdfe7d76b731f5d8bd217827cf6764d4f1d9763407e42717b4bed50a0/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3", size = 250320, upload-time = "2025-10-06T05:36:17.821Z" }, + { url = "https://files.pythonhosted.org/packages/f5/3c/b051329f718b463b22613e269ad72138cc256c540f78a6de89452803a47d/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143", size = 246820, upload-time = "2025-10-06T05:36:19.046Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ae/58282e8f98e444b3f4dd42448ff36fa38bef29e40d40f330b22e7108f565/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608", size = 250518, upload-time = "2025-10-06T05:36:20.763Z" }, + { url = "https://files.pythonhosted.org/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", size = 239096, upload-time = "2025-10-06T05:36:22.129Z" }, + { url = "https://files.pythonhosted.org/packages/66/bb/852b9d6db2fa40be96f29c0d1205c306288f0684df8fd26ca1951d461a56/frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf", size = 39985, upload-time = "2025-10-06T05:36:23.661Z" }, + { url = "https://files.pythonhosted.org/packages/b8/af/38e51a553dd66eb064cdf193841f16f077585d4d28394c2fa6235cb41765/frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746", size = 44591, upload-time = "2025-10-06T05:36:24.958Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1dc65480ab147339fecc70797e9c2f69d9cea9cf38934ce08df070fdb9cb/frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd", size = 40102, upload-time = "2025-10-06T05:36:26.333Z" }, + { url = "https://files.pythonhosted.org/packages/2d/40/0832c31a37d60f60ed79e9dfb5a92e1e2af4f40a16a29abcc7992af9edff/frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a", size = 85717, upload-time = "2025-10-06T05:36:27.341Z" }, + { url = "https://files.pythonhosted.org/packages/30/ba/b0b3de23f40bc55a7057bd38434e25c34fa48e17f20ee273bbde5e0650f3/frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7", size = 49651, upload-time = "2025-10-06T05:36:28.855Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ab/6e5080ee374f875296c4243c381bbdef97a9ac39c6e3ce1d5f7d42cb78d6/frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40", size = 49417, upload-time = "2025-10-06T05:36:29.877Z" }, + { url = "https://files.pythonhosted.org/packages/d5/4e/e4691508f9477ce67da2015d8c00acd751e6287739123113a9fca6f1604e/frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027", size = 234391, upload-time = "2025-10-06T05:36:31.301Z" }, + { url = "https://files.pythonhosted.org/packages/40/76/c202df58e3acdf12969a7895fd6f3bc016c642e6726aa63bd3025e0fc71c/frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822", size = 233048, upload-time = "2025-10-06T05:36:32.531Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c0/8746afb90f17b73ca5979c7a3958116e105ff796e718575175319b5bb4ce/frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121", size = 226549, upload-time = "2025-10-06T05:36:33.706Z" }, + { url = "https://files.pythonhosted.org/packages/7e/eb/4c7eefc718ff72f9b6c4893291abaae5fbc0c82226a32dcd8ef4f7a5dbef/frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5", size = 239833, upload-time = "2025-10-06T05:36:34.947Z" }, + { url = "https://files.pythonhosted.org/packages/c2/4e/e5c02187cf704224f8b21bee886f3d713ca379535f16893233b9d672ea71/frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e", size = 245363, upload-time = "2025-10-06T05:36:36.534Z" }, + { url = "https://files.pythonhosted.org/packages/1f/96/cb85ec608464472e82ad37a17f844889c36100eed57bea094518bf270692/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11", size = 229314, upload-time = "2025-10-06T05:36:38.582Z" }, + { url = "https://files.pythonhosted.org/packages/5d/6f/4ae69c550e4cee66b57887daeebe006fe985917c01d0fff9caab9883f6d0/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1", size = 243365, upload-time = "2025-10-06T05:36:40.152Z" }, + { url = "https://files.pythonhosted.org/packages/7a/58/afd56de246cf11780a40a2c28dc7cbabbf06337cc8ddb1c780a2d97e88d8/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1", size = 237763, upload-time = "2025-10-06T05:36:41.355Z" }, + { url = "https://files.pythonhosted.org/packages/cb/36/cdfaf6ed42e2644740d4a10452d8e97fa1c062e2a8006e4b09f1b5fd7d63/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8", size = 240110, upload-time = "2025-10-06T05:36:42.716Z" }, + { url = "https://files.pythonhosted.org/packages/03/a8/9ea226fbefad669f11b52e864c55f0bd57d3c8d7eb07e9f2e9a0b39502e1/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed", size = 233717, upload-time = "2025-10-06T05:36:44.251Z" }, + { url = "https://files.pythonhosted.org/packages/1e/0b/1b5531611e83ba7d13ccc9988967ea1b51186af64c42b7a7af465dcc9568/frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496", size = 39628, upload-time = "2025-10-06T05:36:45.423Z" }, + { url = "https://files.pythonhosted.org/packages/d8/cf/174c91dbc9cc49bc7b7aab74d8b734e974d1faa8f191c74af9b7e80848e6/frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231", size = 43882, upload-time = "2025-10-06T05:36:46.796Z" }, + { url = "https://files.pythonhosted.org/packages/c1/17/502cd212cbfa96eb1388614fe39a3fc9ab87dbbe042b66f97acb57474834/frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62", size = 39676, upload-time = "2025-10-06T05:36:47.8Z" }, + { url = "https://files.pythonhosted.org/packages/d2/5c/3bbfaa920dfab09e76946a5d2833a7cbdf7b9b4a91c714666ac4855b88b4/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94", size = 89235, upload-time = "2025-10-06T05:36:48.78Z" }, + { url = "https://files.pythonhosted.org/packages/d2/d6/f03961ef72166cec1687e84e8925838442b615bd0b8854b54923ce5b7b8a/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c", size = 50742, upload-time = "2025-10-06T05:36:49.837Z" }, + { url = "https://files.pythonhosted.org/packages/1e/bb/a6d12b7ba4c3337667d0e421f7181c82dda448ce4e7ad7ecd249a16fa806/frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52", size = 51725, upload-time = "2025-10-06T05:36:50.851Z" }, + { url = "https://files.pythonhosted.org/packages/bc/71/d1fed0ffe2c2ccd70b43714c6cab0f4188f09f8a67a7914a6b46ee30f274/frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51", size = 284533, upload-time = "2025-10-06T05:36:51.898Z" }, + { url = "https://files.pythonhosted.org/packages/c9/1f/fb1685a7b009d89f9bf78a42d94461bc06581f6e718c39344754a5d9bada/frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65", size = 292506, upload-time = "2025-10-06T05:36:53.101Z" }, + { url = "https://files.pythonhosted.org/packages/e6/3b/b991fe1612703f7e0d05c0cf734c1b77aaf7c7d321df4572e8d36e7048c8/frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82", size = 274161, upload-time = "2025-10-06T05:36:54.309Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ec/c5c618767bcdf66e88945ec0157d7f6c4a1322f1473392319b7a2501ded7/frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714", size = 294676, upload-time = "2025-10-06T05:36:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/7c/ce/3934758637d8f8a88d11f0585d6495ef54b2044ed6ec84492a91fa3b27aa/frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d", size = 300638, upload-time = "2025-10-06T05:36:56.758Z" }, + { url = "https://files.pythonhosted.org/packages/fc/4f/a7e4d0d467298f42de4b41cbc7ddaf19d3cfeabaf9ff97c20c6c7ee409f9/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506", size = 283067, upload-time = "2025-10-06T05:36:57.965Z" }, + { url = "https://files.pythonhosted.org/packages/dc/48/c7b163063d55a83772b268e6d1affb960771b0e203b632cfe09522d67ea5/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51", size = 292101, upload-time = "2025-10-06T05:36:59.237Z" }, + { url = "https://files.pythonhosted.org/packages/9f/d0/2366d3c4ecdc2fd391e0afa6e11500bfba0ea772764d631bbf82f0136c9d/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e", size = 289901, upload-time = "2025-10-06T05:37:00.811Z" }, + { url = "https://files.pythonhosted.org/packages/b8/94/daff920e82c1b70e3618a2ac39fbc01ae3e2ff6124e80739ce5d71c9b920/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0", size = 289395, upload-time = "2025-10-06T05:37:02.115Z" }, + { url = "https://files.pythonhosted.org/packages/e3/20/bba307ab4235a09fdcd3cc5508dbabd17c4634a1af4b96e0f69bfe551ebd/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41", size = 283659, upload-time = "2025-10-06T05:37:03.711Z" }, + { url = "https://files.pythonhosted.org/packages/fd/00/04ca1c3a7a124b6de4f8a9a17cc2fcad138b4608e7a3fc5877804b8715d7/frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b", size = 43492, upload-time = "2025-10-06T05:37:04.915Z" }, + { url = "https://files.pythonhosted.org/packages/59/5e/c69f733a86a94ab10f68e496dc6b7e8bc078ebb415281d5698313e3af3a1/frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888", size = 48034, upload-time = "2025-10-06T05:37:06.343Z" }, + { url = "https://files.pythonhosted.org/packages/16/6c/be9d79775d8abe79b05fa6d23da99ad6e7763a1d080fbae7290b286093fd/frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042", size = 41749, upload-time = "2025-10-06T05:37:07.431Z" }, + { url = "https://files.pythonhosted.org/packages/f1/c8/85da824b7e7b9b6e7f7705b2ecaf9591ba6f79c1177f324c2735e41d36a2/frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0", size = 86127, upload-time = "2025-10-06T05:37:08.438Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e8/a1185e236ec66c20afd72399522f142c3724c785789255202d27ae992818/frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f", size = 49698, upload-time = "2025-10-06T05:37:09.48Z" }, + { url = "https://files.pythonhosted.org/packages/a1/93/72b1736d68f03fda5fdf0f2180fb6caaae3894f1b854d006ac61ecc727ee/frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c", size = 49749, upload-time = "2025-10-06T05:37:10.569Z" }, + { url = "https://files.pythonhosted.org/packages/a7/b2/fabede9fafd976b991e9f1b9c8c873ed86f202889b864756f240ce6dd855/frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2", size = 231298, upload-time = "2025-10-06T05:37:11.993Z" }, + { url = "https://files.pythonhosted.org/packages/3a/3b/d9b1e0b0eed36e70477ffb8360c49c85c8ca8ef9700a4e6711f39a6e8b45/frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8", size = 232015, upload-time = "2025-10-06T05:37:13.194Z" }, + { url = "https://files.pythonhosted.org/packages/dc/94/be719d2766c1138148564a3960fc2c06eb688da592bdc25adcf856101be7/frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686", size = 225038, upload-time = "2025-10-06T05:37:14.577Z" }, + { url = "https://files.pythonhosted.org/packages/e4/09/6712b6c5465f083f52f50cf74167b92d4ea2f50e46a9eea0523d658454ae/frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e", size = 240130, upload-time = "2025-10-06T05:37:15.781Z" }, + { url = "https://files.pythonhosted.org/packages/f8/d4/cd065cdcf21550b54f3ce6a22e143ac9e4836ca42a0de1022da8498eac89/frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a", size = 242845, upload-time = "2025-10-06T05:37:17.037Z" }, + { url = "https://files.pythonhosted.org/packages/62/c3/f57a5c8c70cd1ead3d5d5f776f89d33110b1addae0ab010ad774d9a44fb9/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128", size = 229131, upload-time = "2025-10-06T05:37:18.221Z" }, + { url = "https://files.pythonhosted.org/packages/6c/52/232476fe9cb64f0742f3fde2b7d26c1dac18b6d62071c74d4ded55e0ef94/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f", size = 240542, upload-time = "2025-10-06T05:37:19.771Z" }, + { url = "https://files.pythonhosted.org/packages/5f/85/07bf3f5d0fb5414aee5f47d33c6f5c77bfe49aac680bfece33d4fdf6a246/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7", size = 237308, upload-time = "2025-10-06T05:37:20.969Z" }, + { url = "https://files.pythonhosted.org/packages/11/99/ae3a33d5befd41ac0ca2cc7fd3aa707c9c324de2e89db0e0f45db9a64c26/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30", size = 238210, upload-time = "2025-10-06T05:37:22.252Z" }, + { url = "https://files.pythonhosted.org/packages/b2/60/b1d2da22f4970e7a155f0adde9b1435712ece01b3cd45ba63702aea33938/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7", size = 231972, upload-time = "2025-10-06T05:37:23.5Z" }, + { url = "https://files.pythonhosted.org/packages/3f/ab/945b2f32de889993b9c9133216c068b7fcf257d8595a0ac420ac8677cab0/frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806", size = 40536, upload-time = "2025-10-06T05:37:25.581Z" }, + { url = "https://files.pythonhosted.org/packages/59/ad/9caa9b9c836d9ad6f067157a531ac48b7d36499f5036d4141ce78c230b1b/frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0", size = 44330, upload-time = "2025-10-06T05:37:26.928Z" }, + { url = "https://files.pythonhosted.org/packages/82/13/e6950121764f2676f43534c555249f57030150260aee9dcf7d64efda11dd/frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b", size = 40627, upload-time = "2025-10-06T05:37:28.075Z" }, + { url = "https://files.pythonhosted.org/packages/c0/c7/43200656ecc4e02d3f8bc248df68256cd9572b3f0017f0a0c4e93440ae23/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d", size = 89238, upload-time = "2025-10-06T05:37:29.373Z" }, + { url = "https://files.pythonhosted.org/packages/d1/29/55c5f0689b9c0fb765055629f472c0de484dcaf0acee2f7707266ae3583c/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed", size = 50738, upload-time = "2025-10-06T05:37:30.792Z" }, + { url = "https://files.pythonhosted.org/packages/ba/7d/b7282a445956506fa11da8c2db7d276adcbf2b17d8bb8407a47685263f90/frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930", size = 51739, upload-time = "2025-10-06T05:37:32.127Z" }, + { url = "https://files.pythonhosted.org/packages/62/1c/3d8622e60d0b767a5510d1d3cf21065b9db874696a51ea6d7a43180a259c/frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c", size = 284186, upload-time = "2025-10-06T05:37:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/2d/14/aa36d5f85a89679a85a1d44cd7a6657e0b1c75f61e7cad987b203d2daca8/frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24", size = 292196, upload-time = "2025-10-06T05:37:36.107Z" }, + { url = "https://files.pythonhosted.org/packages/05/23/6bde59eb55abd407d34f77d39a5126fb7b4f109a3f611d3929f14b700c66/frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37", size = 273830, upload-time = "2025-10-06T05:37:37.663Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3f/22cff331bfad7a8afa616289000ba793347fcd7bc275f3b28ecea2a27909/frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a", size = 294289, upload-time = "2025-10-06T05:37:39.261Z" }, + { url = "https://files.pythonhosted.org/packages/a4/89/5b057c799de4838b6c69aa82b79705f2027615e01be996d2486a69ca99c4/frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2", size = 300318, upload-time = "2025-10-06T05:37:43.213Z" }, + { url = "https://files.pythonhosted.org/packages/30/de/2c22ab3eb2a8af6d69dc799e48455813bab3690c760de58e1bf43b36da3e/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef", size = 282814, upload-time = "2025-10-06T05:37:45.337Z" }, + { url = "https://files.pythonhosted.org/packages/59/f7/970141a6a8dbd7f556d94977858cfb36fa9b66e0892c6dd780d2219d8cd8/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe", size = 291762, upload-time = "2025-10-06T05:37:46.657Z" }, + { url = "https://files.pythonhosted.org/packages/c1/15/ca1adae83a719f82df9116d66f5bb28bb95557b3951903d39135620ef157/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8", size = 289470, upload-time = "2025-10-06T05:37:47.946Z" }, + { url = "https://files.pythonhosted.org/packages/ac/83/dca6dc53bf657d371fbc88ddeb21b79891e747189c5de990b9dfff2ccba1/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a", size = 289042, upload-time = "2025-10-06T05:37:49.499Z" }, + { url = "https://files.pythonhosted.org/packages/96/52/abddd34ca99be142f354398700536c5bd315880ed0a213812bc491cff5e4/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e", size = 283148, upload-time = "2025-10-06T05:37:50.745Z" }, + { url = "https://files.pythonhosted.org/packages/af/d3/76bd4ed4317e7119c2b7f57c3f6934aba26d277acc6309f873341640e21f/frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df", size = 44676, upload-time = "2025-10-06T05:37:52.222Z" }, + { url = "https://files.pythonhosted.org/packages/89/76/c615883b7b521ead2944bb3480398cbb07e12b7b4e4d073d3752eb721558/frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd", size = 49451, upload-time = "2025-10-06T05:37:53.425Z" }, + { url = "https://files.pythonhosted.org/packages/e0/a3/5982da14e113d07b325230f95060e2169f5311b1017ea8af2a29b374c289/frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79", size = 42507, upload-time = "2025-10-06T05:37:54.513Z" }, + { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, +] + +[[package]] +name = "fsspec" +version = "2026.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/51/7c/f60c259dcbf4f0c47cc4ddb8f7720d2dcdc8888c8e5ad84c73ea4531cc5b/fsspec-2026.2.0.tar.gz", hash = "sha256:6544e34b16869f5aacd5b90bdf1a71acb37792ea3ddf6125ee69a22a53fb8bff", size = 313441, upload-time = "2026-02-05T21:50:53.743Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e6/ab/fb21f4c939bb440104cc2b396d3be1d9b7a9fd3c6c2a53d98c45b3d7c954/fsspec-2026.2.0-py3-none-any.whl", hash = "sha256:98de475b5cb3bd66bedd5c4679e87b4fdfe1a3bf4d707b151b3c07e58c9a2437", size = 202505, upload-time = "2026-02-05T21:50:51.819Z" }, +] + +[package.optional-dependencies] +http = [ + { name = "aiohttp" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "hf-xet" +version = "1.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/63/39/67be8d71f900d9a55761b6022821d6679fb56c64f1b6063d5af2c2606727/hf_xet-1.5.2.tar.gz", hash = "sha256:73044bd31bae33c984af832d19c752a0dffb67518fee9ddbd91d616e1101cf47", size = 903674, upload-time = "2026-07-16T17:29:56.833Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/be/525eabac5d1736b679c39e342ecd4292534012546a2d18f0043c8e3b6021/hf_xet-1.5.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:4a5ecb9cda8512ba2aa8ee5d37c87a1422992165892d653098c7b90247481c3b", size = 4064284, upload-time = "2026-07-16T17:29:29.907Z" }, + { url = "https://files.pythonhosted.org/packages/c5/3f/699749dd78442480eda4e4fca494284b0e3542e4063cc37654d5fdc929e6/hf_xet-1.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8764488197c1d7b1378c8438c18d2eea902e150dbca0b0f0d2d32603fb9b5576", size = 3828537, upload-time = "2026-07-16T17:29:31.549Z" }, + { url = "https://files.pythonhosted.org/packages/22/d7/2658ac0a5b9f4664ca27ce31bd015044fe9dea50ed455fb5197aba819c11/hf_xet-1.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8d7446f72abbf7e01ca5ff131786bc2e74a56393462c17a6bf1e303fbab81db4", size = 4417133, upload-time = "2026-07-16T17:29:33.391Z" }, + { url = "https://files.pythonhosted.org/packages/d9/58/8343f3cb63c8fa058d576136df3871550f7d5214a8f048a7ea2eab6ac906/hf_xet-1.5.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:580e59e29bf37aece1f2b68537de1e3fb04f43a23d910dcf6f128280b5bfbba4", size = 4212613, upload-time = "2026-07-16T17:29:34.989Z" }, + { url = "https://files.pythonhosted.org/packages/0c/33/a968f4e4535037b36941ec00714625fb60e026302407e7e26ca9f3e65f4e/hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bee28c619622d36968056532fd49cf2b35ca75099b1d616c31a618a893491380", size = 4412710, upload-time = "2026-07-16T17:29:36.646Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d9/9e33981173dbaf194ba0015202b02d467b624d44d4eba89e1bf06c0d2995/hf_xet-1.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e396ab0faf6298199ad7a95305c3ca8498cb825978a6485be6d00587ee4ec577", size = 4628455, upload-time = "2026-07-16T17:29:38.352Z" }, + { url = "https://files.pythonhosted.org/packages/e9/4b/cc682832de4264a03880a2d1b5ec3e1fab3bf307f508817250baafdb9996/hf_xet-1.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:fd3add255549e8ef58fa35b2e42dc016961c050600444e7d77d030ba6b57120e", size = 3979044, upload-time = "2026-07-16T17:29:40.329Z" }, + { url = "https://files.pythonhosted.org/packages/ea/09/b2cdf2a0fb39a08af3222b96092a36bd3b40c54123eef07de4422e870971/hf_xet-1.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d6f9c58549407b84b9a5383afd68db0acc42345326a3159990b36a5ca8a20e4e", size = 3808037, upload-time = "2026-07-16T17:29:42.357Z" }, + { url = "https://files.pythonhosted.org/packages/de/ba/2b70603c7552db82baeb2623e2336898304a17328845151be4fe1f48d420/hf_xet-1.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f922b8f5fb84f1dd3d7ab7a1316354a1bca9b1c73ecfc19c76e51a2a49d29799", size = 4033760, upload-time = "2026-07-16T17:29:43.884Z" }, + { url = "https://files.pythonhosted.org/packages/60/ac/b097a86a1e4a6098f3a79382643ab09d5733d87ccc864877ad1e12b49b70/hf_xet-1.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:045f84440c55cdeb659cf1a1dd48c77bcd0d2e93632e2fea8f2c3bdee79f38ed", size = 3841438, upload-time = "2026-07-16T17:29:45.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/35/db860aa3a0780660324a506ad4b3d322ddc6ecbba4b9340aed0942cbf21c/hf_xet-1.5.2-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db78c39c83d6279daddc98e2238f373ab8980685556d42472b4ec51abcf03e8c", size = 4428006, upload-time = "2026-07-16T17:29:46.996Z" }, + { url = "https://files.pythonhosted.org/packages/af/6b/832dd980af4b0c3ae0660e309285f2ffcdff2faa38129390dbb47aa4a3f9/hf_xet-1.5.2-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:7db73c810500c54c6760be8c39d4b2e476974de85424c50063efc22fdda13025", size = 4221099, upload-time = "2026-07-16T17:29:48.525Z" }, + { url = "https://files.pythonhosted.org/packages/9e/05/ae50f0d34e3254e6c3e208beb2519f6b8673016fc4b3643badaf6450d186/hf_xet-1.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6395cfe3c9cbead4f16b31808b0e67eac428b66c656f856e99636adaddea878f", size = 4420766, upload-time = "2026-07-16T17:29:50.092Z" }, + { url = "https://files.pythonhosted.org/packages/07/a9/c050bc2743a2bcd68928bfee157b08681667a164a24ec95fbfcfcd717e08/hf_xet-1.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cde8cd167126bb6109b2ceb19b844433a4988643e8f3e01dd9dd0e4a34535097", size = 4636716, upload-time = "2026-07-16T17:29:51.62Z" }, + { url = "https://files.pythonhosted.org/packages/e9/f8/68b01c5c2edb56ac9a67b3d076ffddcb90867abaee923923eb34e7a14e76/hf_xet-1.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:ecf63d1cb69a9a7319910f8f83fcf9b46e7a32dfcf4b8f8eeddb55f647306e65", size = 3988373, upload-time = "2026-07-16T17:29:53.395Z" }, + { url = "https://files.pythonhosted.org/packages/39/c6/988383e9dc17294d536fcbcd6fd16eed882e411ad16c954984a53e47b09c/hf_xet-1.5.2-cp38-abi3-win_arm64.whl", hash = "sha256:1da28519496eb7c8094c11e4d25509b4a468457a0302d58136099db2fd9a671d", size = 3816957, upload-time = "2026-07-16T17:29:54.991Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "huggingface-hub" +version = "1.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, + { name = "httpx" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/82/db/3582597f8be0d34bd6881365a26d390854f12893eabdd62dd36de9df5a47/huggingface_hub-1.26.0.tar.gz", hash = "sha256:c8cd4e2df1ba9402f77fce9b509ec1d52debb502551789473f34016acc14e361", size = 936665, upload-time = "2026-07-30T14:12:04.156Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/bb/63a644c75b545f3ff394b822e9bd1c4a9586489c618b77a4d8a44a33a23b/huggingface_hub-1.26.0-py3-none-any.whl", hash = "sha256:e8cca670caa5d8dfa7e45bf45e86b466698198cd8150c021bcdb4a86b9252364", size = 780357, upload-time = "2026-07-30T14:12:01.998Z" }, +] + +[[package]] +name = "idna" +version = "3.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631, upload-time = "2025-09-27T18:36:18.185Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058, upload-time = "2025-09-27T18:36:19.444Z" }, + { url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287, upload-time = "2025-09-27T18:36:20.768Z" }, + { url = "https://files.pythonhosted.org/packages/30/ac/0273f6fcb5f42e314c6d8cd99effae6a5354604d461b8d392b5ec9530a54/markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf", size = 22940, upload-time = "2025-09-27T18:36:22.249Z" }, + { url = "https://files.pythonhosted.org/packages/19/ae/31c1be199ef767124c042c6c3e904da327a2f7f0cd63a0337e1eca2967a8/markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f", size = 21887, upload-time = "2025-09-27T18:36:23.535Z" }, + { url = "https://files.pythonhosted.org/packages/b2/76/7edcab99d5349a4532a459e1fe64f0b0467a3365056ae550d3bcf3f79e1e/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a", size = 23692, upload-time = "2025-09-27T18:36:24.823Z" }, + { url = "https://files.pythonhosted.org/packages/a4/28/6e74cdd26d7514849143d69f0bf2399f929c37dc2b31e6829fd2045b2765/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115", size = 21471, upload-time = "2025-09-27T18:36:25.95Z" }, + { url = "https://files.pythonhosted.org/packages/62/7e/a145f36a5c2945673e590850a6f8014318d5577ed7e5920a4b3448e0865d/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a", size = 22923, upload-time = "2025-09-27T18:36:27.109Z" }, + { url = "https://files.pythonhosted.org/packages/0f/62/d9c46a7f5c9adbeeeda52f5b8d802e1094e9717705a645efc71b0913a0a8/markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19", size = 14572, upload-time = "2025-09-27T18:36:28.045Z" }, + { url = "https://files.pythonhosted.org/packages/83/8a/4414c03d3f891739326e1783338e48fb49781cc915b2e0ee052aa490d586/markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01", size = 15077, upload-time = "2025-09-27T18:36:29.025Z" }, + { url = "https://files.pythonhosted.org/packages/35/73/893072b42e6862f319b5207adc9ae06070f095b358655f077f69a35601f0/markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c", size = 13876, upload-time = "2025-09-27T18:36:29.954Z" }, + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "mpmath" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, +] + +[[package]] +name = "multidict" +version = "6.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/c2/c2d94cbe6ac1753f3fc980da97b3d930efe1da3af3c9f5125354436c073d/multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d", size = 102010, upload-time = "2026-01-26T02:46:45.979Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/f1/a90635c4f88fb913fbf4ce660b83b7445b7a02615bda034b2f8eb38fd597/multidict-6.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d", size = 76626, upload-time = "2026-01-26T02:43:26.485Z" }, + { url = "https://files.pythonhosted.org/packages/a6/9b/267e64eaf6fc637a15b35f5de31a566634a2740f97d8d094a69d34f524a4/multidict-6.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e", size = 44706, upload-time = "2026-01-26T02:43:27.607Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a4/d45caf2b97b035c57267791ecfaafbd59c68212004b3842830954bb4b02e/multidict-6.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855", size = 44356, upload-time = "2026-01-26T02:43:28.661Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d2/0a36c8473f0cbaeadd5db6c8b72d15bbceeec275807772bfcd059bef487d/multidict-6.7.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3", size = 244355, upload-time = "2026-01-26T02:43:31.165Z" }, + { url = "https://files.pythonhosted.org/packages/5d/16/8c65be997fd7dd311b7d39c7b6e71a0cb449bad093761481eccbbe4b42a2/multidict-6.7.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e", size = 246433, upload-time = "2026-01-26T02:43:32.581Z" }, + { url = "https://files.pythonhosted.org/packages/01/fb/4dbd7e848d2799c6a026ec88ad39cf2b8416aa167fcc903baa55ecaa045c/multidict-6.7.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a", size = 225376, upload-time = "2026-01-26T02:43:34.417Z" }, + { url = "https://files.pythonhosted.org/packages/b6/8a/4a3a6341eac3830f6053062f8fbc9a9e54407c80755b3f05bc427295c2d0/multidict-6.7.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8", size = 257365, upload-time = "2026-01-26T02:43:35.741Z" }, + { url = "https://files.pythonhosted.org/packages/f7/a2/dd575a69c1aa206e12d27d0770cdf9b92434b48a9ef0cd0d1afdecaa93c4/multidict-6.7.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0", size = 254747, upload-time = "2026-01-26T02:43:36.976Z" }, + { url = "https://files.pythonhosted.org/packages/5a/56/21b27c560c13822ed93133f08aa6372c53a8e067f11fbed37b4adcdac922/multidict-6.7.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144", size = 246293, upload-time = "2026-01-26T02:43:38.258Z" }, + { url = "https://files.pythonhosted.org/packages/5a/a4/23466059dc3854763423d0ad6c0f3683a379d97673b1b89ec33826e46728/multidict-6.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49", size = 242962, upload-time = "2026-01-26T02:43:40.034Z" }, + { url = "https://files.pythonhosted.org/packages/1f/67/51dd754a3524d685958001e8fa20a0f5f90a6a856e0a9dcabff69be3dbb7/multidict-6.7.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71", size = 237360, upload-time = "2026-01-26T02:43:41.752Z" }, + { url = "https://files.pythonhosted.org/packages/64/3f/036dfc8c174934d4b55d86ff4f978e558b0e585cef70cfc1ad01adc6bf18/multidict-6.7.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3", size = 245940, upload-time = "2026-01-26T02:43:43.042Z" }, + { url = "https://files.pythonhosted.org/packages/3d/20/6214d3c105928ebc353a1c644a6ef1408bc5794fcb4f170bb524a3c16311/multidict-6.7.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c", size = 253502, upload-time = "2026-01-26T02:43:44.371Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e2/c653bc4ae1be70a0f836b82172d643fcf1dade042ba2676ab08ec08bff0f/multidict-6.7.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0", size = 247065, upload-time = "2026-01-26T02:43:45.745Z" }, + { url = "https://files.pythonhosted.org/packages/c8/11/a854b4154cd3bd8b1fd375e8a8ca9d73be37610c361543d56f764109509b/multidict-6.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa", size = 241870, upload-time = "2026-01-26T02:43:47.054Z" }, + { url = "https://files.pythonhosted.org/packages/13/bf/9676c0392309b5fdae322333d22a829715b570edb9baa8016a517b55b558/multidict-6.7.1-cp311-cp311-win32.whl", hash = "sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a", size = 41302, upload-time = "2026-01-26T02:43:48.753Z" }, + { url = "https://files.pythonhosted.org/packages/c9/68/f16a3a8ba6f7b6dc92a1f19669c0810bd2c43fc5a02da13b1cbf8e253845/multidict-6.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b", size = 45981, upload-time = "2026-01-26T02:43:49.921Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ad/9dd5305253fa00cd3c7555dbef69d5bf4133debc53b87ab8d6a44d411665/multidict-6.7.1-cp311-cp311-win_arm64.whl", hash = "sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6", size = 43159, upload-time = "2026-01-26T02:43:51.635Z" }, + { url = "https://files.pythonhosted.org/packages/8d/9c/f20e0e2cf80e4b2e4b1c365bf5fe104ee633c751a724246262db8f1a0b13/multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172", size = 76893, upload-time = "2026-01-26T02:43:52.754Z" }, + { url = "https://files.pythonhosted.org/packages/fe/cf/18ef143a81610136d3da8193da9d80bfe1cb548a1e2d1c775f26b23d024a/multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd", size = 45456, upload-time = "2026-01-26T02:43:53.893Z" }, + { url = "https://files.pythonhosted.org/packages/a9/65/1caac9d4cd32e8433908683446eebc953e82d22b03d10d41a5f0fefe991b/multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7", size = 43872, upload-time = "2026-01-26T02:43:55.041Z" }, + { url = "https://files.pythonhosted.org/packages/cf/3b/d6bd75dc4f3ff7c73766e04e705b00ed6dbbaccf670d9e05a12b006f5a21/multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53", size = 251018, upload-time = "2026-01-26T02:43:56.198Z" }, + { url = "https://files.pythonhosted.org/packages/fd/80/c959c5933adedb9ac15152e4067c702a808ea183a8b64cf8f31af8ad3155/multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75", size = 258883, upload-time = "2026-01-26T02:43:57.499Z" }, + { url = "https://files.pythonhosted.org/packages/86/85/7ed40adafea3d4f1c8b916e3b5cc3a8e07dfcdcb9cd72800f4ed3ca1b387/multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b", size = 242413, upload-time = "2026-01-26T02:43:58.755Z" }, + { url = "https://files.pythonhosted.org/packages/d2/57/b8565ff533e48595503c785f8361ff9a4fde4d67de25c207cd0ba3befd03/multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733", size = 268404, upload-time = "2026-01-26T02:44:00.216Z" }, + { url = "https://files.pythonhosted.org/packages/e0/50/9810c5c29350f7258180dfdcb2e52783a0632862eb334c4896ac717cebcb/multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a", size = 269456, upload-time = "2026-01-26T02:44:02.202Z" }, + { url = "https://files.pythonhosted.org/packages/f3/8d/5e5be3ced1d12966fefb5c4ea3b2a5b480afcea36406559442c6e31d4a48/multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961", size = 256322, upload-time = "2026-01-26T02:44:03.56Z" }, + { url = "https://files.pythonhosted.org/packages/31/6e/d8a26d81ac166a5592782d208dd90dfdc0a7a218adaa52b45a672b46c122/multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582", size = 253955, upload-time = "2026-01-26T02:44:04.845Z" }, + { url = "https://files.pythonhosted.org/packages/59/4c/7c672c8aad41534ba619bcd4ade7a0dc87ed6b8b5c06149b85d3dd03f0cd/multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e", size = 251254, upload-time = "2026-01-26T02:44:06.133Z" }, + { url = "https://files.pythonhosted.org/packages/7b/bd/84c24de512cbafbdbc39439f74e967f19570ce7924e3007174a29c348916/multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3", size = 252059, upload-time = "2026-01-26T02:44:07.518Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ba/f5449385510825b73d01c2d4087bf6d2fccc20a2d42ac34df93191d3dd03/multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6", size = 263588, upload-time = "2026-01-26T02:44:09.382Z" }, + { url = "https://files.pythonhosted.org/packages/d7/11/afc7c677f68f75c84a69fe37184f0f82fce13ce4b92f49f3db280b7e92b3/multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a", size = 259642, upload-time = "2026-01-26T02:44:10.73Z" }, + { url = "https://files.pythonhosted.org/packages/2b/17/ebb9644da78c4ab36403739e0e6e0e30ebb135b9caf3440825001a0bddcb/multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba", size = 251377, upload-time = "2026-01-26T02:44:12.042Z" }, + { url = "https://files.pythonhosted.org/packages/ca/a4/840f5b97339e27846c46307f2530a2805d9d537d8b8bd416af031cad7fa0/multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511", size = 41887, upload-time = "2026-01-26T02:44:14.245Z" }, + { url = "https://files.pythonhosted.org/packages/80/31/0b2517913687895f5904325c2069d6a3b78f66cc641a86a2baf75a05dcbb/multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19", size = 46053, upload-time = "2026-01-26T02:44:15.371Z" }, + { url = "https://files.pythonhosted.org/packages/0c/5b/aba28e4ee4006ae4c7df8d327d31025d760ffa992ea23812a601d226e682/multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf", size = 43307, upload-time = "2026-01-26T02:44:16.852Z" }, + { url = "https://files.pythonhosted.org/packages/f2/22/929c141d6c0dba87d3e1d38fbdf1ba8baba86b7776469f2bc2d3227a1e67/multidict-6.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23", size = 76174, upload-time = "2026-01-26T02:44:18.509Z" }, + { url = "https://files.pythonhosted.org/packages/c7/75/bc704ae15fee974f8fccd871305e254754167dce5f9e42d88a2def741a1d/multidict-6.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2", size = 45116, upload-time = "2026-01-26T02:44:19.745Z" }, + { url = "https://files.pythonhosted.org/packages/79/76/55cd7186f498ed080a18440c9013011eb548f77ae1b297206d030eb1180a/multidict-6.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445", size = 43524, upload-time = "2026-01-26T02:44:21.571Z" }, + { url = "https://files.pythonhosted.org/packages/e9/3c/414842ef8d5a1628d68edee29ba0e5bcf235dbfb3ccd3ea303a7fe8c72ff/multidict-6.7.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177", size = 249368, upload-time = "2026-01-26T02:44:22.803Z" }, + { url = "https://files.pythonhosted.org/packages/f6/32/befed7f74c458b4a525e60519fe8d87eef72bb1e99924fa2b0f9d97a221e/multidict-6.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23", size = 256952, upload-time = "2026-01-26T02:44:24.306Z" }, + { url = "https://files.pythonhosted.org/packages/03/d6/c878a44ba877f366630c860fdf74bfb203c33778f12b6ac274936853c451/multidict-6.7.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060", size = 240317, upload-time = "2026-01-26T02:44:25.772Z" }, + { url = "https://files.pythonhosted.org/packages/68/49/57421b4d7ad2e9e60e25922b08ceb37e077b90444bde6ead629095327a6f/multidict-6.7.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d", size = 267132, upload-time = "2026-01-26T02:44:27.648Z" }, + { url = "https://files.pythonhosted.org/packages/b7/fe/ec0edd52ddbcea2a2e89e174f0206444a61440b40f39704e64dc807a70bd/multidict-6.7.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed", size = 268140, upload-time = "2026-01-26T02:44:29.588Z" }, + { url = "https://files.pythonhosted.org/packages/b0/73/6e1b01cbeb458807aa0831742232dbdd1fa92bfa33f52a3f176b4ff3dc11/multidict-6.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429", size = 254277, upload-time = "2026-01-26T02:44:30.902Z" }, + { url = "https://files.pythonhosted.org/packages/6a/b2/5fb8c124d7561a4974c342bc8c778b471ebbeb3cc17df696f034a7e9afe7/multidict-6.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6", size = 252291, upload-time = "2026-01-26T02:44:32.31Z" }, + { url = "https://files.pythonhosted.org/packages/5a/96/51d4e4e06bcce92577fcd488e22600bd38e4fd59c20cb49434d054903bd2/multidict-6.7.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9", size = 250156, upload-time = "2026-01-26T02:44:33.734Z" }, + { url = "https://files.pythonhosted.org/packages/db/6b/420e173eec5fba721a50e2a9f89eda89d9c98fded1124f8d5c675f7a0c0f/multidict-6.7.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c", size = 249742, upload-time = "2026-01-26T02:44:35.222Z" }, + { url = "https://files.pythonhosted.org/packages/44/a3/ec5b5bd98f306bc2aa297b8c6f11a46714a56b1e6ef5ebda50a4f5d7c5fb/multidict-6.7.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84", size = 262221, upload-time = "2026-01-26T02:44:36.604Z" }, + { url = "https://files.pythonhosted.org/packages/cd/f7/e8c0d0da0cd1e28d10e624604e1a36bcc3353aaebdfdc3a43c72bc683a12/multidict-6.7.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d", size = 258664, upload-time = "2026-01-26T02:44:38.008Z" }, + { url = "https://files.pythonhosted.org/packages/52/da/151a44e8016dd33feed44f730bd856a66257c1ee7aed4f44b649fb7edeb3/multidict-6.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33", size = 249490, upload-time = "2026-01-26T02:44:39.386Z" }, + { url = "https://files.pythonhosted.org/packages/87/af/a3b86bf9630b732897f6fc3f4c4714b90aa4361983ccbdcd6c0339b21b0c/multidict-6.7.1-cp313-cp313-win32.whl", hash = "sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3", size = 41695, upload-time = "2026-01-26T02:44:41.318Z" }, + { url = "https://files.pythonhosted.org/packages/b2/35/e994121b0e90e46134673422dd564623f93304614f5d11886b1b3e06f503/multidict-6.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5", size = 45884, upload-time = "2026-01-26T02:44:42.488Z" }, + { url = "https://files.pythonhosted.org/packages/ca/61/42d3e5dbf661242a69c97ea363f2d7b46c567da8eadef8890022be6e2ab0/multidict-6.7.1-cp313-cp313-win_arm64.whl", hash = "sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df", size = 43122, upload-time = "2026-01-26T02:44:43.664Z" }, + { url = "https://files.pythonhosted.org/packages/6d/b3/e6b21c6c4f314bb956016b0b3ef2162590a529b84cb831c257519e7fde44/multidict-6.7.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1", size = 83175, upload-time = "2026-01-26T02:44:44.894Z" }, + { url = "https://files.pythonhosted.org/packages/fb/76/23ecd2abfe0957b234f6c960f4ade497f55f2c16aeb684d4ecdbf1c95791/multidict-6.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963", size = 48460, upload-time = "2026-01-26T02:44:46.106Z" }, + { url = "https://files.pythonhosted.org/packages/c4/57/a0ed92b23f3a042c36bc4227b72b97eca803f5f1801c1ab77c8a212d455e/multidict-6.7.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34", size = 46930, upload-time = "2026-01-26T02:44:47.278Z" }, + { url = "https://files.pythonhosted.org/packages/b5/66/02ec7ace29162e447f6382c495dc95826bf931d3818799bbef11e8f7df1a/multidict-6.7.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65", size = 242582, upload-time = "2026-01-26T02:44:48.604Z" }, + { url = "https://files.pythonhosted.org/packages/58/18/64f5a795e7677670e872673aca234162514696274597b3708b2c0d276cce/multidict-6.7.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292", size = 250031, upload-time = "2026-01-26T02:44:50.544Z" }, + { url = "https://files.pythonhosted.org/packages/c8/ed/e192291dbbe51a8290c5686f482084d31bcd9d09af24f63358c3d42fd284/multidict-6.7.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43", size = 228596, upload-time = "2026-01-26T02:44:51.951Z" }, + { url = "https://files.pythonhosted.org/packages/1e/7e/3562a15a60cf747397e7f2180b0a11dc0c38d9175a650e75fa1b4d325e15/multidict-6.7.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca", size = 257492, upload-time = "2026-01-26T02:44:53.902Z" }, + { url = "https://files.pythonhosted.org/packages/24/02/7d0f9eae92b5249bb50ac1595b295f10e263dd0078ebb55115c31e0eaccd/multidict-6.7.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd", size = 255899, upload-time = "2026-01-26T02:44:55.316Z" }, + { url = "https://files.pythonhosted.org/packages/00/e3/9b60ed9e23e64c73a5cde95269ef1330678e9c6e34dd4eb6b431b85b5a10/multidict-6.7.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7", size = 247970, upload-time = "2026-01-26T02:44:56.783Z" }, + { url = "https://files.pythonhosted.org/packages/3e/06/538e58a63ed5cfb0bd4517e346b91da32fde409d839720f664e9a4ae4f9d/multidict-6.7.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3", size = 245060, upload-time = "2026-01-26T02:44:58.195Z" }, + { url = "https://files.pythonhosted.org/packages/b2/2f/d743a3045a97c895d401e9bd29aaa09b94f5cbdf1bd561609e5a6c431c70/multidict-6.7.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4", size = 235888, upload-time = "2026-01-26T02:44:59.57Z" }, + { url = "https://files.pythonhosted.org/packages/38/83/5a325cac191ab28b63c52f14f1131f3b0a55ba3b9aa65a6d0bf2a9b921a0/multidict-6.7.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8", size = 243554, upload-time = "2026-01-26T02:45:01.054Z" }, + { url = "https://files.pythonhosted.org/packages/20/1f/9d2327086bd15da2725ef6aae624208e2ef828ed99892b17f60c344e57ed/multidict-6.7.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c", size = 252341, upload-time = "2026-01-26T02:45:02.484Z" }, + { url = "https://files.pythonhosted.org/packages/e8/2c/2a1aa0280cf579d0f6eed8ee5211c4f1730bd7e06c636ba2ee6aafda302e/multidict-6.7.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52", size = 246391, upload-time = "2026-01-26T02:45:03.862Z" }, + { url = "https://files.pythonhosted.org/packages/e5/03/7ca022ffc36c5a3f6e03b179a5ceb829be9da5783e6fe395f347c0794680/multidict-6.7.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108", size = 243422, upload-time = "2026-01-26T02:45:05.296Z" }, + { url = "https://files.pythonhosted.org/packages/dc/1d/b31650eab6c5778aceed46ba735bd97f7c7d2f54b319fa916c0f96e7805b/multidict-6.7.1-cp313-cp313t-win32.whl", hash = "sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32", size = 47770, upload-time = "2026-01-26T02:45:06.754Z" }, + { url = "https://files.pythonhosted.org/packages/ac/5b/2d2d1d522e51285bd61b1e20df8f47ae1a9d80839db0b24ea783b3832832/multidict-6.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8", size = 53109, upload-time = "2026-01-26T02:45:08.044Z" }, + { url = "https://files.pythonhosted.org/packages/3d/a3/cc409ba012c83ca024a308516703cf339bdc4b696195644a7215a5164a24/multidict-6.7.1-cp313-cp313t-win_arm64.whl", hash = "sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118", size = 45573, upload-time = "2026-01-26T02:45:09.349Z" }, + { url = "https://files.pythonhosted.org/packages/91/cc/db74228a8be41884a567e88a62fd589a913708fcf180d029898c17a9a371/multidict-6.7.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee", size = 75190, upload-time = "2026-01-26T02:45:10.651Z" }, + { url = "https://files.pythonhosted.org/packages/d5/22/492f2246bb5b534abd44804292e81eeaf835388901f0c574bac4eeec73c5/multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2", size = 44486, upload-time = "2026-01-26T02:45:11.938Z" }, + { url = "https://files.pythonhosted.org/packages/f1/4f/733c48f270565d78b4544f2baddc2fb2a245e5a8640254b12c36ac7ac68e/multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1", size = 43219, upload-time = "2026-01-26T02:45:14.346Z" }, + { url = "https://files.pythonhosted.org/packages/24/bb/2c0c2287963f4259c85e8bcbba9182ced8d7fca65c780c38e99e61629d11/multidict-6.7.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d", size = 245132, upload-time = "2026-01-26T02:45:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/a7/f9/44d4b3064c65079d2467888794dea218d1601898ac50222ab8a9a8094460/multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31", size = 252420, upload-time = "2026-01-26T02:45:17.293Z" }, + { url = "https://files.pythonhosted.org/packages/8b/13/78f7275e73fa17b24c9a51b0bd9d73ba64bb32d0ed51b02a746eb876abe7/multidict-6.7.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048", size = 233510, upload-time = "2026-01-26T02:45:19.356Z" }, + { url = "https://files.pythonhosted.org/packages/4b/25/8167187f62ae3cbd52da7893f58cb036b47ea3fb67138787c76800158982/multidict-6.7.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362", size = 264094, upload-time = "2026-01-26T02:45:20.834Z" }, + { url = "https://files.pythonhosted.org/packages/a1/e7/69a3a83b7b030cf283fb06ce074a05a02322359783424d7edf0f15fe5022/multidict-6.7.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37", size = 260786, upload-time = "2026-01-26T02:45:22.818Z" }, + { url = "https://files.pythonhosted.org/packages/fe/3b/8ec5074bcfc450fe84273713b4b0a0dd47c0249358f5d82eb8104ffe2520/multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709", size = 248483, upload-time = "2026-01-26T02:45:24.368Z" }, + { url = "https://files.pythonhosted.org/packages/48/5a/d5a99e3acbca0e29c5d9cba8f92ceb15dce78bab963b308ae692981e3a5d/multidict-6.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0", size = 248403, upload-time = "2026-01-26T02:45:25.982Z" }, + { url = "https://files.pythonhosted.org/packages/35/48/e58cd31f6c7d5102f2a4bf89f96b9cf7e00b6c6f3d04ecc44417c00a5a3c/multidict-6.7.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb", size = 240315, upload-time = "2026-01-26T02:45:27.487Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/1cd210229559cb90b6786c30676bb0c58249ff42f942765f88793b41fdce/multidict-6.7.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd", size = 245528, upload-time = "2026-01-26T02:45:28.991Z" }, + { url = "https://files.pythonhosted.org/packages/64/f2/6e1107d226278c876c783056b7db43d800bb64c6131cec9c8dfb6903698e/multidict-6.7.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601", size = 258784, upload-time = "2026-01-26T02:45:30.503Z" }, + { url = "https://files.pythonhosted.org/packages/4d/c1/11f664f14d525e4a1b5327a82d4de61a1db604ab34c6603bb3c2cc63ad34/multidict-6.7.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1", size = 251980, upload-time = "2026-01-26T02:45:32.603Z" }, + { url = "https://files.pythonhosted.org/packages/e1/9f/75a9ac888121d0c5bbd4ecf4eead45668b1766f6baabfb3b7f66a410e231/multidict-6.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b", size = 243602, upload-time = "2026-01-26T02:45:34.043Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e7/50bf7b004cc8525d80dbbbedfdc7aed3e4c323810890be4413e589074032/multidict-6.7.1-cp314-cp314-win32.whl", hash = "sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d", size = 40930, upload-time = "2026-01-26T02:45:36.278Z" }, + { url = "https://files.pythonhosted.org/packages/e0/bf/52f25716bbe93745595800f36fb17b73711f14da59ed0bb2eba141bc9f0f/multidict-6.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f", size = 45074, upload-time = "2026-01-26T02:45:37.546Z" }, + { url = "https://files.pythonhosted.org/packages/97/ab/22803b03285fa3a525f48217963da3a65ae40f6a1b6f6cf2768879e208f9/multidict-6.7.1-cp314-cp314-win_arm64.whl", hash = "sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5", size = 42471, upload-time = "2026-01-26T02:45:38.889Z" }, + { url = "https://files.pythonhosted.org/packages/e0/6d/f9293baa6146ba9507e360ea0292b6422b016907c393e2f63fc40ab7b7b5/multidict-6.7.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581", size = 82401, upload-time = "2026-01-26T02:45:40.254Z" }, + { url = "https://files.pythonhosted.org/packages/7a/68/53b5494738d83558d87c3c71a486504d8373421c3e0dbb6d0db48ad42ee0/multidict-6.7.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a", size = 48143, upload-time = "2026-01-26T02:45:41.635Z" }, + { url = "https://files.pythonhosted.org/packages/37/e8/5284c53310dcdc99ce5d66563f6e5773531a9b9fe9ec7a615e9bc306b05f/multidict-6.7.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c", size = 46507, upload-time = "2026-01-26T02:45:42.99Z" }, + { url = "https://files.pythonhosted.org/packages/e4/fc/6800d0e5b3875568b4083ecf5f310dcf91d86d52573160834fb4bfcf5e4f/multidict-6.7.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262", size = 239358, upload-time = "2026-01-26T02:45:44.376Z" }, + { url = "https://files.pythonhosted.org/packages/41/75/4ad0973179361cdf3a113905e6e088173198349131be2b390f9fa4da5fc6/multidict-6.7.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59", size = 246884, upload-time = "2026-01-26T02:45:47.167Z" }, + { url = "https://files.pythonhosted.org/packages/c3/9c/095bb28b5da139bd41fb9a5d5caff412584f377914bd8787c2aa98717130/multidict-6.7.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889", size = 225878, upload-time = "2026-01-26T02:45:48.698Z" }, + { url = "https://files.pythonhosted.org/packages/07/d0/c0a72000243756e8f5a277b6b514fa005f2c73d481b7d9e47cd4568aa2e4/multidict-6.7.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4", size = 253542, upload-time = "2026-01-26T02:45:50.164Z" }, + { url = "https://files.pythonhosted.org/packages/c0/6b/f69da15289e384ecf2a68837ec8b5ad8c33e973aa18b266f50fe55f24b8c/multidict-6.7.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d", size = 252403, upload-time = "2026-01-26T02:45:51.779Z" }, + { url = "https://files.pythonhosted.org/packages/a2/76/b9669547afa5a1a25cd93eaca91c0da1c095b06b6d2d8ec25b713588d3a1/multidict-6.7.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609", size = 244889, upload-time = "2026-01-26T02:45:53.27Z" }, + { url = "https://files.pythonhosted.org/packages/7e/a9/a50d2669e506dad33cfc45b5d574a205587b7b8a5f426f2fbb2e90882588/multidict-6.7.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489", size = 241982, upload-time = "2026-01-26T02:45:54.919Z" }, + { url = "https://files.pythonhosted.org/packages/c5/bb/1609558ad8b456b4827d3c5a5b775c93b87878fd3117ed3db3423dfbce1b/multidict-6.7.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c", size = 232415, upload-time = "2026-01-26T02:45:56.981Z" }, + { url = "https://files.pythonhosted.org/packages/d8/59/6f61039d2aa9261871e03ab9dc058a550d240f25859b05b67fd70f80d4b3/multidict-6.7.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e", size = 240337, upload-time = "2026-01-26T02:45:58.698Z" }, + { url = "https://files.pythonhosted.org/packages/a1/29/fdc6a43c203890dc2ae9249971ecd0c41deaedfe00d25cb6564b2edd99eb/multidict-6.7.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c", size = 248788, upload-time = "2026-01-26T02:46:00.862Z" }, + { url = "https://files.pythonhosted.org/packages/a9/14/a153a06101323e4cf086ecee3faadba52ff71633d471f9685c42e3736163/multidict-6.7.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9", size = 242842, upload-time = "2026-01-26T02:46:02.824Z" }, + { url = "https://files.pythonhosted.org/packages/41/5f/604ae839e64a4a6efc80db94465348d3b328ee955e37acb24badbcd24d83/multidict-6.7.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2", size = 240237, upload-time = "2026-01-26T02:46:05.898Z" }, + { url = "https://files.pythonhosted.org/packages/5f/60/c3a5187bf66f6fb546ff4ab8fb5a077cbdd832d7b1908d4365c7f74a1917/multidict-6.7.1-cp314-cp314t-win32.whl", hash = "sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7", size = 48008, upload-time = "2026-01-26T02:46:07.468Z" }, + { url = "https://files.pythonhosted.org/packages/0c/f7/addf1087b860ac60e6f382240f64fb99f8bfb532bb06f7c542b83c29ca61/multidict-6.7.1-cp314-cp314t-win_amd64.whl", hash = "sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5", size = 53542, upload-time = "2026-01-26T02:46:08.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/81/4629d0aa32302ef7b2ec65c75a728cc5ff4fa410c50096174c1632e70b3e/multidict-6.7.1-cp314-cp314t-win_arm64.whl", hash = "sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2", size = 44719, upload-time = "2026-01-26T02:46:11.146Z" }, + { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, +] + +[[package]] +name = "multiprocess" +version = "0.70.19" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dill" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a2/f2/e783ac7f2aeeed14e9e12801f22529cc7e6b7ab80928d6dcce4e9f00922d/multiprocess-0.70.19.tar.gz", hash = "sha256:952021e0e6c55a4a9fe4cd787895b86e239a40e76802a789d6305398d3975897", size = 2079989, upload-time = "2026-01-19T06:47:39.744Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/aa/714635c727dbfc251139226fa4eaf1b07f00dc12d9cd2eb25f931adaf873/multiprocess-0.70.19-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1bbf1b69af1cf64cd05f65337d9215b88079ec819cd0ea7bac4dab84e162efe7", size = 144743, upload-time = "2026-01-19T06:47:24.562Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e1/155f6abf5e6b5d9cef29b6d0167c180846157a4aca9b9bee1a217f67c959/multiprocess-0.70.19-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:5be9ec7f0c1c49a4f4a6fd20d5dda4aeabc2d39a50f4ad53720f1cd02b3a7c2e", size = 144738, upload-time = "2026-01-19T06:47:26.636Z" }, + { url = "https://files.pythonhosted.org/packages/af/cb/f421c2869d75750a4f32301cc20c4b63fab6376e9a75c8e5e655bdeb3d9b/multiprocess-0.70.19-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:1c3dce098845a0db43b32a0b76a228ca059a668071cfeaa0f40c36c0b1585d45", size = 144741, upload-time = "2026-01-19T06:47:27.985Z" }, + { url = "https://files.pythonhosted.org/packages/e3/45/8004d1e6b9185c1a444d6b55ac5682acf9d98035e54386d967366035a03a/multiprocess-0.70.19-py310-none-any.whl", hash = "sha256:97404393419dcb2a8385910864eedf47a3cadf82c66345b44f036420eb0b5d87", size = 134948, upload-time = "2026-01-19T06:47:32.325Z" }, + { url = "https://files.pythonhosted.org/packages/86/c2/dec9722dc3474c164a0b6bcd9a7ed7da542c98af8cabce05374abab35edd/multiprocess-0.70.19-py311-none-any.whl", hash = "sha256:928851ae7973aea4ce0eaf330bbdafb2e01398a91518d5c8818802845564f45c", size = 144457, upload-time = "2026-01-19T06:47:33.711Z" }, + { url = "https://files.pythonhosted.org/packages/71/70/38998b950a97ea279e6bd657575d22d1a2047256caf707d9a10fbce4f065/multiprocess-0.70.19-py312-none-any.whl", hash = "sha256:3a56c0e85dd5025161bac5ce138dcac1e49174c7d8e74596537e729fd5c53c28", size = 150281, upload-time = "2026-01-19T06:47:35.037Z" }, + { url = "https://files.pythonhosted.org/packages/7f/74/d2c27e03cb84251dfe7249b8e82923643c6d48fa4883b9476b025e7dc7eb/multiprocess-0.70.19-py313-none-any.whl", hash = "sha256:8d5eb4ec5017ba2fab4e34a747c6d2c2b6fecfe9e7236e77988db91580ada952", size = 156414, upload-time = "2026-01-19T06:47:35.915Z" }, + { url = "https://files.pythonhosted.org/packages/a0/61/af9115673a5870fd885247e2f1b68c4f1197737da315b520a91c757a861a/multiprocess-0.70.19-py314-none-any.whl", hash = "sha256:e8cc7fbdff15c0613f0a1f1f8744bef961b0a164c0ca29bdff53e9d2d93c5e5f", size = 160318, upload-time = "2026-01-19T06:47:37.497Z" }, + { url = "https://files.pythonhosted.org/packages/7e/82/69e539c4c2027f1e1697e09aaa2449243085a0edf81ae2c6341e84d769b6/multiprocess-0.70.19-py39-none-any.whl", hash = "sha256:0d4b4397ed669d371c81dcd1ef33fd384a44d6c3de1bd0ca7ac06d837720d3c5", size = 133477, upload-time = "2026-01-19T06:47:38.619Z" }, +] + +[[package]] +name = "networkx" +version = "3.6.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, +] + +[[package]] +name = "numpy" +version = "2.4.6" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.12' and sys_platform == 'win32'", + "python_full_version < '3.12' and sys_platform == 'emscripten'", + "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/49/ec46835a70be8fa6446c495126ac84fdb28cb2558e1620ffb87a10c8b64c/numpy-2.4.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4", size = 16969194, upload-time = "2026-05-18T23:33:13.503Z" }, + { url = "https://files.pythonhosted.org/packages/0e/0d/f5957185c0ee2f3e12f78715aa9e3b353fd83633316c8532b38faa37e3f6/numpy-2.4.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d", size = 14964111, upload-time = "2026-05-18T23:33:17.795Z" }, + { url = "https://files.pythonhosted.org/packages/ad/40/40a40ee0ddf7ceb782c49af278894b686e586d65d8c1889c8b5da01a3d7d/numpy-2.4.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8", size = 5469159, upload-time = "2026-05-18T23:33:20.654Z" }, + { url = "https://files.pythonhosted.org/packages/63/13/f9a8046535cb21deae82f8d03de9617e08882d274fad2539630761888228/numpy-2.4.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538", size = 6798936, upload-time = "2026-05-18T23:33:22.987Z" }, + { url = "https://files.pythonhosted.org/packages/33/a8/6fa8c1a345a8c85dbb21932c447bee07c30a2c2a3f31e369c0a84b300147/numpy-2.4.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47", size = 15966692, upload-time = "2026-05-18T23:33:26.62Z" }, + { url = "https://files.pythonhosted.org/packages/02/03/74fe2a4cb3817d94d86402f2506554130a2f01414e299b5a843e5a8a957f/numpy-2.4.6-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93", size = 16918164, upload-time = "2026-05-18T23:33:29.955Z" }, + { url = "https://files.pythonhosted.org/packages/c5/80/3615be3313f7e7696609bc194b9f0101da809df79e859bdb84e0cd043f46/numpy-2.4.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8", size = 17322877, upload-time = "2026-05-18T23:33:34.724Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ac/a691e0fe2675e370d0e08ff905adc49a1c8830e8cae03efe4477e92cd55d/numpy-2.4.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6", size = 18651487, upload-time = "2026-05-18T23:33:38.217Z" }, + { url = "https://files.pythonhosted.org/packages/15/a7/9bc1cd626d7bf6869bfedf27b91b6ab5dd607758bf8e959d6fa80c6a59cb/numpy-2.4.6-cp311-cp311-win32.whl", hash = "sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8", size = 6233945, upload-time = "2026-05-18T23:33:41.331Z" }, + { url = "https://files.pythonhosted.org/packages/c5/31/7fc6239c12bce7e931463251cca4426c465e1876ba3cc785402ef4dd8f4e/numpy-2.4.6-cp311-cp311-win_amd64.whl", hash = "sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147", size = 12608406, upload-time = "2026-05-18T23:33:44.131Z" }, + { url = "https://files.pythonhosted.org/packages/27/83/140f85a466595a16382996a1bf06b2b54bcd597488921b0c9daaeeda72af/numpy-2.4.6-cp311-cp311-win_arm64.whl", hash = "sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577", size = 10479528, upload-time = "2026-05-18T23:33:50.725Z" }, + { url = "https://files.pythonhosted.org/packages/95/2a/3d7b5ac8aac24feaf9ad7ed58f45b0bbc06d37e4338ae84c9f2298b570f9/numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1", size = 16689119, upload-time = "2026-05-18T23:33:54.065Z" }, + { url = "https://files.pythonhosted.org/packages/ea/12/92c4c131527599e8288d6918e888d88726f84d805d784b771f32408aeaef/numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb", size = 14699246, upload-time = "2026-05-18T23:33:57.621Z" }, + { url = "https://files.pythonhosted.org/packages/ad/fe/c0a6b7b2ca128a8fb228575147073b660656734b8ebe4d76c8fd748dcc79/numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41", size = 5204410, upload-time = "2026-05-18T23:34:00.302Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d4/9770d14ba719432bb90a421bfd443872ed0f70f7264b64bec12ea363d5fd/numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698", size = 6551240, upload-time = "2026-05-18T23:34:02.852Z" }, + { url = "https://files.pythonhosted.org/packages/c9/c6/50a46a6205feba2343f1d6d17438107c5dc491ed1c736e6ea68689fd906b/numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f", size = 15671012, upload-time = "2026-05-18T23:34:05.485Z" }, + { url = "https://files.pythonhosted.org/packages/99/60/14115e6364fa676c5397c2ad3004e527e9aa487abf5d0706ec81bbd08529/numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853", size = 16645538, upload-time = "2026-05-18T23:34:09.265Z" }, + { url = "https://files.pythonhosted.org/packages/ae/c5/693cbe59e57db94d2231fa519ca3978dc9e19da5a8f088588f5c6e947ff2/numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a", size = 17020706, upload-time = "2026-05-18T23:34:13.053Z" }, + { url = "https://files.pythonhosted.org/packages/ef/fc/85b7c4eff9b4966ade25c2273cf7e7012e92366c032058653934b37de044/numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2", size = 18368541, upload-time = "2026-05-18T23:34:17.024Z" }, + { url = "https://files.pythonhosted.org/packages/f6/81/e1b27545deedce7f4a0b348618c6b62d74e36a4dc9ccd42f3eb2f85eee32/numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45", size = 5962825, upload-time = "2026-05-18T23:34:20.3Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ca/feab00bd44aa5fe1ad2c18f08b4d3bb92e26484b0b1d1443897809ed528c/numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751", size = 12321687, upload-time = "2026-05-18T23:34:23.095Z" }, + { url = "https://files.pythonhosted.org/packages/63/cf/5a6d34850a39d1093558564f77ee8e8e0bee5061151b8f05a55711001ec7/numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8", size = 10221482, upload-time = "2026-05-18T23:34:25.876Z" }, + { url = "https://files.pythonhosted.org/packages/fb/82/bdab26d7438c6791ca31b7c024ca37c1eab8b726ba236129005cd4a06e45/numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0", size = 16684648, upload-time = "2026-05-18T23:34:29.41Z" }, + { url = "https://files.pythonhosted.org/packages/1b/30/a80189bcc7f5e4258b3fbc3968d909d1756f54d023299ecc39ad6fdb9ef8/numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb", size = 14693902, upload-time = "2026-05-18T23:34:33.013Z" }, + { url = "https://files.pythonhosted.org/packages/97/12/70b5d0d7c15e1ebb8a6a84a8caa1d19e181d84fb58bb6d70aca29099dec1/numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f", size = 5198992, upload-time = "2026-05-18T23:34:36.132Z" }, + { url = "https://files.pythonhosted.org/packages/ba/8c/ebd2a8f8a83541f8d38cc5667e8c2b69cecfd30da6e45693e8158857d44b/numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3", size = 6546944, upload-time = "2026-05-18T23:34:38.484Z" }, + { url = "https://files.pythonhosted.org/packages/bb/c5/7b863a97a91671a0338f4253bd3b5a3d3852f0692dae91711c9f4a10e787/numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b", size = 15669392, upload-time = "2026-05-18T23:34:41.257Z" }, + { url = "https://files.pythonhosted.org/packages/a5/9d/3584b9984ca4c047aea75214ce1a4c4c73d849bd71b604264b7f5653f8a8/numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089", size = 16633220, upload-time = "2026-05-18T23:34:45.075Z" }, + { url = "https://files.pythonhosted.org/packages/05/ae/7c67fba23bd98caec7c99261f3a16072ade14813486b0282cb29846de832/numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a", size = 17020800, upload-time = "2026-05-18T23:34:49.065Z" }, + { url = "https://files.pythonhosted.org/packages/d9/5d/3b6725cb31d983c5e66916f5d36f6d7e5521129e4c4404d64f918292a5b6/numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605", size = 18357600, upload-time = "2026-05-18T23:34:52.709Z" }, + { url = "https://files.pythonhosted.org/packages/f7/da/2ccc6c2fe8898dee01d90c75c5f5f914a23daf99e3e0f59516a08760c8b5/numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91", size = 5961134, upload-time = "2026-05-18T23:34:55.618Z" }, + { url = "https://files.pythonhosted.org/packages/b5/cd/9cc4dc876fb065d5c220aae4d5e14826b2715331bb7618ce1fb07a679d99/numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359", size = 12318598, upload-time = "2026-05-18T23:34:58.928Z" }, + { url = "https://files.pythonhosted.org/packages/39/1e/c0bcba1f8694116485fe28fd1be698c278fcda4141c5b0e53a2aed8b12a8/numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778", size = 10222272, upload-time = "2026-05-18T23:35:02.167Z" }, + { url = "https://files.pythonhosted.org/packages/63/6d/cc5619247c8f4204e507f5883528372e4ac4bb189e579fb859a12e480b1f/numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1", size = 14821197, upload-time = "2026-05-18T23:35:05.468Z" }, + { url = "https://files.pythonhosted.org/packages/00/58/f1c39161c87d9e9bed660f1ed4bafc0e403d5ec9650b6dd77aead07d489b/numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe", size = 5326287, upload-time = "2026-05-18T23:35:08.693Z" }, + { url = "https://files.pythonhosted.org/packages/af/57/3917ab0fd97f271a8694513581b8a36c655f111c446852c302f04ccdb6fc/numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997", size = 6646763, upload-time = "2026-05-18T23:35:11.459Z" }, + { url = "https://files.pythonhosted.org/packages/eb/0f/037e64c494b67581ae18193d770adef354c41f3f2c8ebf865602d949bf8f/numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20", size = 15728070, upload-time = "2026-05-18T23:35:14.79Z" }, + { url = "https://files.pythonhosted.org/packages/21/a6/5d2bae9c9542eb4df16dc9c46dc79c186e9bad53805dfa5399a6023c6db0/numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d", size = 16681752, upload-time = "2026-05-18T23:35:18.836Z" }, + { url = "https://files.pythonhosted.org/packages/92/14/23d1dfb410ae362cd59ce53e936b1513d545eb40db3949ced632e19a459e/numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67", size = 17086024, upload-time = "2026-05-18T23:35:22.52Z" }, + { url = "https://files.pythonhosted.org/packages/4b/6e/23595a2c642cdf3bc567877064bdd7f91c8b0038a4453cf2daf7248eafe9/numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd", size = 18403398, upload-time = "2026-05-18T23:35:26.398Z" }, + { url = "https://files.pythonhosted.org/packages/8a/90/0ac3bc947217e66dec77e7cbc6a1979d1af70b6461b82f620d3bccd5e4c8/numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab", size = 6084971, upload-time = "2026-05-18T23:35:29.387Z" }, + { url = "https://files.pythonhosted.org/packages/77/71/5673e351671a1d2bd6063b91b44f70c0affea7d1516fa7a6572941ba4aa1/numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75", size = 12458532, upload-time = "2026-05-18T23:35:32.175Z" }, + { url = "https://files.pythonhosted.org/packages/3f/88/19d3503c5046e688f049274b27a3ef3d771152fa80d3ba3d01a3dff61abe/numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd", size = 10291881, upload-time = "2026-05-18T23:35:35.465Z" }, + { url = "https://files.pythonhosted.org/packages/f8/91/3ab2044d05fd16d343c5ac2e69b127f1b2854040dd20b193257c78028bd3/numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079", size = 16683458, upload-time = "2026-05-18T23:35:38.353Z" }, + { url = "https://files.pythonhosted.org/packages/8e/62/764ce66fa4147ae6d73071a3abf804ffe606f174618697c571acdf26a7c9/numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7", size = 14704559, upload-time = "2026-05-18T23:35:42.14Z" }, + { url = "https://files.pythonhosted.org/packages/60/61/23f27c172f022e04025b7dc2367f4d63c1a398120607ec896228649a6f48/numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5", size = 5209716, upload-time = "2026-05-18T23:35:45.377Z" }, + { url = "https://files.pythonhosted.org/packages/03/71/21cf70dc6ea3e3acb95fc53a265b2fc248b981f0194ceb5b475271b8809d/numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096", size = 6543947, upload-time = "2026-05-18T23:35:47.926Z" }, + { url = "https://files.pythonhosted.org/packages/d5/91/64288395ee1799bd2e0b04a305dce9666da90c961e1f3fe982a05ee1c036/numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b", size = 15685197, upload-time = "2026-05-18T23:35:50.863Z" }, + { url = "https://files.pythonhosted.org/packages/f3/eb/ebffaa97dc55502df69584a8f0dcf07f69a3e0b3e2323670a2722db9aa39/numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8", size = 16638245, upload-time = "2026-05-18T23:35:54.752Z" }, + { url = "https://files.pythonhosted.org/packages/b8/0b/54f9da33128d7e350fab89c7455902eeae70349ee52bddb448dc4a576f45/numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402", size = 17036587, upload-time = "2026-05-18T23:35:58.355Z" }, + { url = "https://files.pythonhosted.org/packages/b6/f0/fdebc1052db1cc37c64beb22072d67cd6d1c71adca1299f53dec2b5e20d3/numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb", size = 18363226, upload-time = "2026-05-18T23:36:02.845Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b4/298628d98c72b57e57f7165ae6a481a1deaf6f3c28262a6e4c739c275930/numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1", size = 6010196, upload-time = "2026-05-18T23:36:05.92Z" }, + { url = "https://files.pythonhosted.org/packages/df/ac/46de6dda46478f7942f839e094970be2d4a861e005c4b3bf07c92e291a09/numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261", size = 12450334, upload-time = "2026-05-18T23:36:09.107Z" }, + { url = "https://files.pythonhosted.org/packages/78/92/b8b798ac784102c0da830d2257d59358e3d3d90d1e2b3f2575dad976c5cf/numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6", size = 10495678, upload-time = "2026-05-18T23:36:12.766Z" }, + { url = "https://files.pythonhosted.org/packages/30/34/ec28d1aa8115971537c01469ab2011ee96827930f0a124de1000cc2a7ed7/numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a", size = 14823672, upload-time = "2026-05-18T23:36:16.473Z" }, + { url = "https://files.pythonhosted.org/packages/16/bd/f6d1fede4e54e8042a7ff97bb495510f3c220f94bcd9e8b228e87c92cc0d/numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e", size = 5328731, upload-time = "2026-05-18T23:36:19.767Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f0/e105b9e2fd728a9910103884decd6951d9dd73896b914a98d9a231de02ee/numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e", size = 6649805, upload-time = "2026-05-18T23:36:22.266Z" }, + { url = "https://files.pythonhosted.org/packages/82/dd/1206a7ca6ab15e3f02069707ca96222e202af681bb73756da7527f3cb837/numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43", size = 15730496, upload-time = "2026-05-18T23:36:25.713Z" }, + { url = "https://files.pythonhosted.org/packages/51/e7/38d3ea825dcab85a591734decb2f6c67caa7c8367d374df1a1c3842f9b07/numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e", size = 16679616, upload-time = "2026-05-18T23:36:29.652Z" }, + { url = "https://files.pythonhosted.org/packages/93/b7/caabfdf53edf663e0b4eb74d7d405d83baef09eb5e83bcd32d601d72b93e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895", size = 17085145, upload-time = "2026-05-18T23:36:33.449Z" }, + { url = "https://files.pythonhosted.org/packages/f9/45/68d7c33a6bcf3e5aa3bdbd57a367e6f615286dfd6482f97e8ffeb734306e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4", size = 18403813, upload-time = "2026-05-18T23:36:37.369Z" }, + { url = "https://files.pythonhosted.org/packages/9c/50/0753655aa844c99cd9e018aacf76f130f1bd81d881bb74bc0aef5d73a8ba/numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063", size = 6156982, upload-time = "2026-05-18T23:36:40.817Z" }, + { url = "https://files.pythonhosted.org/packages/b2/d4/7c67becf668f973cb490cec3e98dfd799d866f9c989a54d355672cfa0db6/numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627", size = 12638908, upload-time = "2026-05-18T23:36:43.996Z" }, + { url = "https://files.pythonhosted.org/packages/43/bb/e1c71a4295b1b1d1393d50dbb4f2a36283c6859d9d3892e84f00ec5a91d5/numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66", size = 10565867, upload-time = "2026-05-18T23:36:47.114Z" }, + { url = "https://files.pythonhosted.org/packages/de/12/b422cc84439adc0d00de605bf4a308890ae5c26f2c71fbd73e5d08fbb0dd/numpy-2.4.6-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662", size = 16847511, upload-time = "2026-05-18T23:36:50.673Z" }, + { url = "https://files.pythonhosted.org/packages/44/53/f481bef68011740f8849418d82db07230e825013f31f4eef5ba5b805316a/numpy-2.4.6-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7", size = 14889064, upload-time = "2026-05-18T23:36:53.879Z" }, + { url = "https://files.pythonhosted.org/packages/7f/57/42ed575c10ced8af951d426bc4e1f8aff16fd851db33f067036215a7f860/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f", size = 5394157, upload-time = "2026-05-18T23:36:57.194Z" }, + { url = "https://files.pythonhosted.org/packages/6a/ef/f66cc724fcc36c1e364c67f51ae9146090b8b584f27d58b97fdae3edd737/numpy-2.4.6-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c", size = 6708728, upload-time = "2026-05-18T23:36:59.575Z" }, + { url = "https://files.pythonhosted.org/packages/1a/9c/c531f2293b91265d8b48e9b329f54fdd7ffae73cb4134ea10cca4237e9cc/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0", size = 15798374, upload-time = "2026-05-18T23:37:02.674Z" }, + { url = "https://files.pythonhosted.org/packages/1a/b0/413077f6b1153ed3cba361401c6783bbad6114804a000cc22eb71c13e190/numpy-2.4.6-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02", size = 16747286, upload-time = "2026-05-18T23:37:06.327Z" }, + { url = "https://files.pythonhosted.org/packages/15/ce/e5ec180bc41812edcd8daeb8639d205622c0e8c02259d8ab25a0201b3c2a/numpy-2.4.6-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73", size = 12504263, upload-time = "2026-05-18T23:37:09.715Z" }, +] + +[[package]] +name = "numpy" +version = "2.5.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +sdist = { url = "https://files.pythonhosted.org/packages/22/fd/89965aa4ac08c74998539fcbf24fa3540f3e15237fbeb6bcf9c908f4aade/numpy-2.5.1.tar.gz", hash = "sha256:a48a113e6afea91f5608793bafa7ef2ad481fefbda87ec5069f483de61cb9fa3", size = 20755553, upload-time = "2026-07-04T17:08:00.933Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/7b/14687aa674250e5e546f616f486b0d56d3631cd5b2415739141ce40bdcea/numpy-2.5.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2c889b56fe48b1018f764b0eec8df59ab654e9148aa91faa12596043500de277", size = 16801574, upload-time = "2026-07-04T17:06:12.423Z" }, + { url = "https://files.pythonhosted.org/packages/e1/19/cc5bb2a3f2913d27d6dbb2c78d25921fabaedc6741d4a5a615a11f3c5bf3/numpy-2.5.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab451b59c5643c570974c43aef780703ef1d3b4965d2be07afd530615a9358d1", size = 11772250, upload-time = "2026-07-04T17:06:15.726Z" }, + { url = "https://files.pythonhosted.org/packages/42/77/fdf34a71dd30f54979b18603bee915e0aaf825b07afe79acd60b04b691e2/numpy-2.5.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:78798bd5b9ad744056af8efa90e3b9ddaa53272a0848a483084a1cc0a13b2dc0", size = 5331516, upload-time = "2026-07-04T17:06:17.913Z" }, + { url = "https://files.pythonhosted.org/packages/ce/e2/eb7efa015b4cce41e2517bf182a7fce0d7d5b9d9ed76a29bfa0f4fe4505c/numpy-2.5.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:2ae0ca40bcb22d6ba59c1dfd5446f49940b0f2d821fde133f10dda11f816b84e", size = 6664863, upload-time = "2026-07-04T17:06:20.02Z" }, + { url = "https://files.pythonhosted.org/packages/a9/4b/a2b32dd94ee9ffbeecb28152240042a3949db33b1c834d44090b80e1b3b8/numpy-2.5.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:61ac47e772e6b8ea489e1d2f441a34c5c3ac17327e7ce294cbdf535795ad4e75", size = 15167977, upload-time = "2026-07-04T17:06:21.621Z" }, + { url = "https://files.pythonhosted.org/packages/b8/a9/6e73d68500f80773f65f0654ea932019d6694329a0eb0ed0533de38df376/numpy-2.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59fda5e192b570217ec2580c96f00e9a7e12ef6866a900eb089b62c1a32545ca", size = 16672469, upload-time = "2026-07-04T17:06:24.064Z" }, + { url = "https://files.pythonhosted.org/packages/24/7d/ad3e59015135f5261c95fd4cafeff159c955febd83a99a1d9250c4233815/numpy-2.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f7119ebff1a9829e9f431a4f9d28e703023bb6b9fe7c8f724467dbfc27c94ab3", size = 16527531, upload-time = "2026-07-04T17:06:26.69Z" }, + { url = "https://files.pythonhosted.org/packages/83/d0/a39b2fbcde9cb17a1dac678f254b33a6336298af9df338824c685425d5e8/numpy-2.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e824c2acf8862052246be5a44c15da1777940c60d010dd2aab897824d9c430f9", size = 18431940, upload-time = "2026-07-04T17:06:29.521Z" }, + { url = "https://files.pythonhosted.org/packages/04/12/cff070947791c1ed425ff76413189adbdc2fbe215eba7ce7fa454a03c7f8/numpy-2.5.1-cp312-cp312-win32.whl", hash = "sha256:08d60c810432eb83360958dea0999ac4cfb94531ea8efcbf0b7f277c2068aeb2", size = 6066764, upload-time = "2026-07-04T17:06:32.571Z" }, + { url = "https://files.pythonhosted.org/packages/65/66/53f31807a48a750f9d748da273bc3fcedd12b27ff1f3e373bfec55ef2dc0/numpy-2.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:f7d60026c0bdb1380e83bfa7a0419c4577ee4b9a08880afcb6dadeb74c649fa2", size = 12430966, upload-time = "2026-07-04T17:06:34.926Z" }, + { url = "https://files.pythonhosted.org/packages/2b/2a/d1a88066b1c14186f5d3c0d18c94f17b064511982bab0578d49ee9d43c29/numpy-2.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:17a25e09640602e10bc8de0e6fa2b3fd68eedd84ba6d7842dc8f32f9ab87bd0b", size = 10350488, upload-time = "2026-07-04T17:06:37.785Z" }, + { url = "https://files.pythonhosted.org/packages/eb/07/ec2a3f0c91761581d4b7104a740791800025983f9a4dc4e73f91a99aeac4/numpy-2.5.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0bfebd8695f9863592fe744be833a258120b14a9f39da255e8aa8fade2c0ddd1", size = 16796419, upload-time = "2026-07-04T17:06:40.37Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ab/ddb499fc4f8780354395face5b65c7fd107bcd6e1d667a5f07d046956f6f/numpy-2.5.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:30b44a6b53a7ae63c54c089a8726e5563ed302716c5b7ccc85afade40b0e7ff6", size = 11765832, upload-time = "2026-07-04T17:06:42.768Z" }, + { url = "https://files.pythonhosted.org/packages/88/b3/3c28c558a09fc72100c646dac6d2fce8e834c471b0edca01a29996706117/numpy-2.5.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6165343f81b56ef8f514f396989e529b61d9dc709b99421b07e9f3e698e2287d", size = 5325143, upload-time = "2026-07-04T17:06:45.466Z" }, + { url = "https://files.pythonhosted.org/packages/5e/0e/ce19b985bb15c596f4f05954e76cccc77c845083b3b8f938a6c68e523128/numpy-2.5.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:4939237038ada79308dda3204ac6462df056b5672b2e25db1149cf873668b3e1", size = 6659749, upload-time = "2026-07-04T17:06:47.288Z" }, + { url = "https://files.pythonhosted.org/packages/2e/20/1ee6614d64332a1bba6411f38e68cb79eec1b2459e20a623777c5c5492a2/numpy-2.5.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c6759f538fb912fc46de0a6b1758ccf7b57bc7c7ebebc23974fdac3de8db0cd", size = 15164716, upload-time = "2026-07-04T17:06:49.494Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a7/2bcd3fdbb87804755c35b729bf8709d62025c5f4cfd7d5b2415997097515/numpy-2.5.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9726558e8db4a5bf7929a70ae50f63abda4daf0efe810e3bfbab95976f75fc1a", size = 16661440, upload-time = "2026-07-04T17:06:52.061Z" }, + { url = "https://files.pythonhosted.org/packages/fc/d7/a41e3310c886fe457d36e670bbf24fae411aca8a7b6ad92a32afd924077c/numpy-2.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3935f3b419b244a02732676fa5317a9193cc596a4c0646db07e5b421229ac9f7", size = 16526305, upload-time = "2026-07-04T17:06:54.605Z" }, + { url = "https://files.pythonhosted.org/packages/53/75/4333a9a707c1edd3a4e1a0c58eca52c0f31e55089fa80db02b5565b24df7/numpy-2.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dc932a65ded7ce9013d120845a2514dcccb1a67bfc8deb8d37633762951904a6", size = 18423008, upload-time = "2026-07-04T17:06:57.54Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/e314a32b1c11a2ffe818ddad3a57b50b4b6e1b6c487192eb50cdef0415d0/numpy-2.5.1-cp313-cp313-win32.whl", hash = "sha256:4b4ff1608417eb7a59da7b967bbb798cacfe071d2caf526a24281cd562072ed9", size = 6063885, upload-time = "2026-07-04T17:07:00.14Z" }, + { url = "https://files.pythonhosted.org/packages/10/70/800b3fca480af32df9e8ea9f3d4a0c8feb4b32d7f195d174eabbda4829ad/numpy-2.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:6c3fe51bc6a16453d452997053454f309e8e0ed7b42d6b361ce4ac8c32913d74", size = 12425674, upload-time = "2026-07-04T17:07:02.387Z" }, + { url = "https://files.pythonhosted.org/packages/8b/0b/196350c122f50f6ca56846f2d71efd5e0d24b7b2e07355e019b2e2c7a11e/numpy-2.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:f7feb014281029e628ba2d5a007407443b06e418b6fe451d1e2adcbc8eba0107", size = 10350256, upload-time = "2026-07-04T17:07:04.878Z" }, + { url = "https://files.pythonhosted.org/packages/db/f4/731b6085a83faf6ca843394cbd5e217280c214399f7e8b21b9f552af0ae2/numpy-2.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c786fe9a5bbe360022e584c5a34cf6b54265c71bd7ec8ac3d8fec38968071f8", size = 16795063, upload-time = "2026-07-04T17:07:07.374Z" }, + { url = "https://files.pythonhosted.org/packages/bf/64/0e215f2048dd11a55bb989ed41b3585ef57452404e638d703a211a3e4157/numpy-2.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:32985c896d897419ef8da6917872d80b78ad0ea26d85b23245c7366ffde76d75", size = 11776652, upload-time = "2026-07-04T17:07:09.907Z" }, + { url = "https://files.pythonhosted.org/packages/b5/59/2b844c7a6e9deff69b404a66221e1542937734f65d5e6e39411876053862/numpy-2.5.1-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:efd736408cc97c79b9e6917338dfc8f06013b2274f992e96b1d9a81a71e2a2c2", size = 5335944, upload-time = "2026-07-04T17:07:12.227Z" }, + { url = "https://files.pythonhosted.org/packages/86/51/9bf7cb2cabcebc9e017e4ec7e6322b378317a542c08b4cb68479c1efc716/numpy-2.5.1-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:ab84dc6b074fa881cae55bea94cc4f68e285181ba7f32497bf7dee6b1496165b", size = 6656266, upload-time = "2026-07-04T17:07:14.368Z" }, + { url = "https://files.pythonhosted.org/packages/83/3e/fb7615b211b82a32f44d5180a6d421b61f84d4fadd578b48ba4ac34e189f/numpy-2.5.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:caf3e317d33d60c37986b452613f4ab51246d0691350c03d0cb4a898627f4a95", size = 15179720, upload-time = "2026-07-04T17:07:16.272Z" }, + { url = "https://files.pythonhosted.org/packages/41/5f/0f992cb24560673496c5d68de61913b57166ce530ffda07c1f280e0cc464/numpy-2.5.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54ad769f17bc2d833b620851989f62054fb9ab93c969d9e1dc3c8e3d56beea21", size = 16664835, upload-time = "2026-07-04T17:07:19.021Z" }, + { url = "https://files.pythonhosted.org/packages/a2/2f/97d6475ee91afe2587797d09446f9d3e475ad4cb681662d824809327b75a/numpy-2.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c12afb53450fa976d4c681c50a7423729a4c51c0465ed9f32b8a9cabbc472373", size = 16539135, upload-time = "2026-07-04T17:07:22.015Z" }, + { url = "https://files.pythonhosted.org/packages/c4/5b/4db81e4ba0be7e2776b1de68c82aa862c7f8ec27e1b4927d4ae075e20678/numpy-2.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e8c11c405efc5ff6816d5983c96cdfa215bab3428961243af3ff59b228490438", size = 18426684, upload-time = "2026-07-04T17:07:24.941Z" }, + { url = "https://files.pythonhosted.org/packages/1f/64/c0ba2d90724d450279a7df8f32057241070250a26a7e2b5337d77347f481/numpy-2.5.1-cp314-cp314-win32.whl", hash = "sha256:f2479a47f8d5932d1718168a681ad6e536a9df484c83cfcf9de365e164537ace", size = 6116103, upload-time = "2026-07-04T17:07:27.622Z" }, + { url = "https://files.pythonhosted.org/packages/c1/1a/837f9ed7405adcd7a40538792eb169eddd8fa5630c16a1ef49dae71a30f4/numpy-2.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:24d0eb82c0541d3415a33425db64ae439dffccd7b4dbcb30e7c35120205c506a", size = 12562177, upload-time = "2026-07-04T17:07:29.887Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/49707938b6dd0a78a9178dd93227dc89e4c11af47f5c798d70366e8d0483/numpy-2.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5a4c988b38d261deeeaad9954e3deb091ad905c94e8bb6708654ef1d97f286b0", size = 10627739, upload-time = "2026-07-04T17:07:32.568Z" }, + { url = "https://files.pythonhosted.org/packages/a6/c7/bb4b882cfe7f299cbc8b66e42e7dd78cf9d14e40f9469fc5e3db7e15b3bd/numpy-2.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a33276be12fa045805f477f22482088b66bb758ffbe89a9d21457de863a32e22", size = 11894709, upload-time = "2026-07-04T17:07:34.941Z" }, + { url = "https://files.pythonhosted.org/packages/40/3f/5af7f4a7f6224aef48017aa82bb6174c7a659d724be0c75017b7e64a55b4/numpy-2.5.1-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:f089d7b00756190aacf1f5d34bdf38c3c430ac82b4f868f8cede73380460fce7", size = 5453810, upload-time = "2026-07-04T17:07:37.495Z" }, + { url = "https://files.pythonhosted.org/packages/20/c9/3474309bc94d634d3f9c3eddf03250ecb8c22cd948ef16fef69a77cc5d7b/numpy-2.5.1-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:09e9bfd8d2cf479c7d174804fb3811c53a8e9f20a37444008606b57d6b7a826d", size = 6761189, upload-time = "2026-07-04T17:07:39.563Z" }, + { url = "https://files.pythonhosted.org/packages/90/8a/558ae39fdd55d7e7f7fef9a84a6e964ac6b23edbd2a07e52bb084500507d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e68d8dd1e7eba712948f2053a29ec86917bc70ba1358df869d9f06649ef9cf09", size = 15225039, upload-time = "2026-07-04T17:07:41.682Z" }, + { url = "https://files.pythonhosted.org/packages/63/27/ca7392b2d030277bdf0273e7d23255b3ee57d57a7c170a6f4fb3981e1e5d/numpy-2.5.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:99d5095fa265a0c4152e7bb12759e14381ef5496152f1ce58f44bdf55c44beb4", size = 16701306, upload-time = "2026-07-04T17:07:44.611Z" }, + { url = "https://files.pythonhosted.org/packages/02/42/03d53ae7996c44d4374a8262e9dc41671fd56cbb98f7d47ef85cf5da4c6b/numpy-2.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ab87a91b3cc3382b8956095bd8f95e00cf679bb81554339be1a2ba404a1473c1", size = 16589955, upload-time = "2026-07-04T17:07:47.694Z" }, + { url = "https://files.pythonhosted.org/packages/7b/15/6c1784ae469640e65db111e9a34b3d0f14d91e8a38b9ce34810ced370dbb/numpy-2.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:224ca51130ef7da85bea2191625181cb4f337f9cb64b471f10c1a12aa8b60077", size = 18464252, upload-time = "2026-07-04T17:07:50.684Z" }, + { url = "https://files.pythonhosted.org/packages/94/a8/f98e50356cf167df656c526c2dfeec2d7dde182f2a3da4b458a5938e2776/numpy-2.5.1-cp314-cp314t-win32.whl", hash = "sha256:6eab239876581b2b3c5a242281b6007bbdbcd1c7085d7709bb57c5929b11e6bf", size = 6263298, upload-time = "2026-07-04T17:07:53.445Z" }, + { url = "https://files.pythonhosted.org/packages/72/ac/96ae880cdecad0b3275d9359fcec72667b49a4863c9f12942e43679dda02/numpy-2.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:83ce9c80d5b521b0d77ddcbe5447c218d247929b6cc056ca5351342accfff0af", size = 12748623, upload-time = "2026-07-04T17:07:55.384Z" }, + { url = "https://files.pythonhosted.org/packages/a1/5a/4d2b1601df3602dba7a14f3348ba9bfe94a18adb428e693df6154c293831/numpy-2.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:5a6db61f9aaa57e369905c67d852045d3c4f7126405b29d09b19dec118e9c9cb", size = 10697674, upload-time = "2026-07-04T17:07:58.506Z" }, +] + +[[package]] +name = "nvidia-cublas" +version = "13.1.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cuda-nvrtc" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" }, + { url = "https://files.pythonhosted.org/packages/3b/cd/154ca20c38269e05eff77c1464e6c1da89f50a6390b565e9d82e06bc11e1/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:37936a16db8fe4ac1f065c2139360608a543a09275cb1a1af612e08cfa065436", size = 423138758, upload-time = "2026-04-08T18:46:58.655Z" }, +] + +[[package]] +name = "nvidia-cuda-cupti" +version = "13.0.85" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/2a/80353b103fc20ce05ef51e928daed4b6015db4aaa9162ed0997090fe2250/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151", size = 10310827, upload-time = "2025-09-04T08:26:42.012Z" }, + { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597, upload-time = "2025-09-04T08:26:51.312Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc" +version = "13.0.88" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575", size = 90215200, upload-time = "2025-09-04T08:28:44.204Z" }, + { url = "https://files.pythonhosted.org/packages/b7/dc/6bb80850e0b7edd6588d560758f17e0550893a1feaf436807d64d2da040f/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b", size = 43015449, upload-time = "2025-09-04T08:28:20.239Z" }, +] + +[[package]] +name = "nvidia-cuda-runtime" +version = "13.0.96" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/4f/17d7b9b8e285199c58ce28e31b5c5bbaa4d8271af06a89b6405258245de2/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55", size = 2261060, upload-time = "2025-10-09T08:55:15.78Z" }, + { url = "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548", size = 2243632, upload-time = "2025-10-09T08:55:36.117Z" }, +] + +[[package]] +name = "nvidia-cudnn-cu13" +version = "9.20.0.48" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" }, + { url = "https://files.pythonhosted.org/packages/6e/5e/edb9c0ae051602c3ccaffe424256463636d639e27d7f302dde9975ef9e7a/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0c45dd8eeb50b603f07995b1b300c62ffe6a1980482b82b3bcf94a4ca9d49304", size = 366173588, upload-time = "2026-03-09T19:29:34.474Z" }, +] + +[[package]] +name = "nvidia-cufft" +version = "12.0.0.61" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3", size = 214085489, upload-time = "2025-09-04T08:31:56.044Z" }, +] + +[[package]] +name = "nvidia-cufile" +version = "1.15.1.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44", size = 1223672, upload-time = "2025-09-04T08:32:22.779Z" }, + { url = "https://files.pythonhosted.org/packages/ab/73/cc4a14c9813a8a0d509417cf5f4bdaba76e924d58beb9864f5a7baceefbf/nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1", size = 1136992, upload-time = "2025-09-04T08:32:14.119Z" }, +] + +[[package]] +name = "nvidia-curand" +version = "10.4.0.35" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/72/7c2ae24fb6b63a32e6ae5d241cc65263ea18d08802aaae087d9f013335a2/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a", size = 61962106, upload-time = "2025-08-04T10:21:41.128Z" }, + { url = "https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc", size = 59544258, upload-time = "2025-08-04T10:22:03.992Z" }, +] + +[[package]] +name = "nvidia-cusolver" +version = "12.0.4.66" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas" }, + { name = "nvidia-cusparse" }, + { name = "nvidia-nvjitlink" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, + { url = "https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112", size = 200941980, upload-time = "2025-09-04T08:33:22.767Z" }, +] + +[[package]] +name = "nvidia-cusparse" +version = "12.6.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, + { url = "https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b", size = 145942937, upload-time = "2025-09-04T08:33:58.029Z" }, +] + +[[package]] +name = "nvidia-cusparselt-cu13" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/e1/cdc1797eadf82d3a9a575a19b33fdc871a97edbec42c00b5b5e914f4aff4/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4dca476c50bf4780d46cd0bfbd82e2bc10a08e4fef7950917ce8d7578d22a23f", size = 221051344, upload-time = "2025-09-05T18:49:51.289Z" }, + { url = "https://files.pythonhosted.org/packages/34/7d/2661f2fb3ac4302f3a246f5fc030213ac60c1fe0bce84f9783dbd831dbb7/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:786ce87568c303fadb5afcc7102d454cd3040d75f6f8626f5db460d1871f4dd0", size = 170148586, upload-time = "2025-09-05T18:50:50.248Z" }, +] + +[[package]] +name = "nvidia-nccl-cu13" +version = "2.29.7" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/72/0d/daf50d44177ee0cbc7ff0a0c91eb5ff676c82be42f9a970bc7597f440c3a/nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:674a12383e3c38a1bcccae7d4f3633b37852230b6047883cb2f4c2d1b36d9bf5", size = 206014712, upload-time = "2026-03-03T05:34:20.843Z" }, + { url = "https://files.pythonhosted.org/packages/67/f4/58e4e91b6919367c7aafb8e36fce9aad1a3047e536bf7e2fd560927d3a4c/nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:edd81538446786ec3b73972543e53bb43bcaf0bfc8ef76cb679fcc390ffe136d", size = 205976000, upload-time = "2026-03-03T05:36:24.472Z" }, +] + +[[package]] +name = "nvidia-nvjitlink" +version = "13.3.33" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/ee/580ca6f29dcab0221db8706badca1bbbb084f1975c4d4e83329c3a7e31f0/nvidia_nvjitlink-13.3.33-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:26a6de7fb4c8fdaa7703d3dad720d6d427ddfea5c48a528fd97c11733ad830e5", size = 40742423, upload-time = "2026-05-26T16:54:51.613Z" }, + { url = "https://files.pythonhosted.org/packages/69/30/45414e35ff2eee7db3da037e5707037ccf9d2b5218ffbdb055ea4d5aa98a/nvidia_nvjitlink-13.3.33-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ce48b37dfeb3cb1eae4cf85adacb47d7a6539ea2272870c9a3628ce275c2037e", size = 39168635, upload-time = "2026-05-26T16:54:13.906Z" }, +] + +[[package]] +name = "nvidia-nvshmem-cu13" +version = "3.4.5" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/0f/05cc9c720236dcd2db9c1ab97fff629e96821be2e63103569da0c9b72f19/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9", size = 60215947, upload-time = "2025-09-06T00:32:20.022Z" }, + { url = "https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80", size = 60412546, upload-time = "2025-09-06T00:32:41.564Z" }, +] + +[[package]] +name = "nvidia-nvtx" +version = "13.0.85" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4", size = 148047, upload-time = "2025-09-04T08:29:01.761Z" }, + { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pandas" +version = "3.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/4f/5f3422a2afec5ffc46308b79e53291365a93748b498ac2e58bead0197916/pandas-3.0.5.tar.gz", hash = "sha256:dca3734d6ab7c906e6730f0788b0a1dbb9f2467731f9711f77995c8e9d62d712", size = 4658219, upload-time = "2026-07-22T22:19:28.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/ef/f1fd7431d635bf20015489bf0bd69c17fff1018de773540f651455a3916b/pandas-3.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2946e77e4a53cd248cbde631a12f0e51c8324ce354c3eba4d20147c1ad6f4282", size = 10397178, upload-time = "2026-07-22T22:17:48.274Z" }, + { url = "https://files.pythonhosted.org/packages/31/b4/0eafac990a431561187694126de01f9b12559549b4d86360c0c4bd870fde/pandas-3.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:71ecc8fb7ed1a7aa4392316b5309a6347e8e7f832f38fd897846b3a1457a9298", size = 9990736, upload-time = "2026-07-22T22:17:52.388Z" }, + { url = "https://files.pythonhosted.org/packages/de/21/359880af3ea9b7cb23bea5b51e8e70ef3866c03be09da9a2787e18e330a8/pandas-3.0.5-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b173f5951ff6b8b0ec7675e20dff3c97b7e7a57dfcce387c2d7c5afe87cb7899", size = 10814438, upload-time = "2026-07-22T22:17:54.708Z" }, + { url = "https://files.pythonhosted.org/packages/d1/50/d6cc4d7e508bbccf5d6027314a8312bc7ac73d0ec7f195f53838daafab40/pandas-3.0.5-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2c0cf1dd9b55a22d105fc46c1b489af3bd42264fcba7c66297bf47a9a1d9c78a", size = 11323634, upload-time = "2026-07-22T22:17:56.858Z" }, + { url = "https://files.pythonhosted.org/packages/70/2b/d5f0a8c90dd0ae04e64ba53b871afb796ec026b615086d382ddc2ade729b/pandas-3.0.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0fac0010c75e4efb6b99e249c183a8993ce0dc95c240f9b120a5e67c727b7928", size = 11850860, upload-time = "2026-07-22T22:17:59.1Z" }, + { url = "https://files.pythonhosted.org/packages/5c/30/183aec2e19adf778a98d29b5729a0a68f4cc4ebf9b9c3b70d0297355bcb1/pandas-3.0.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:08d24fe11a17dc33bd6e937dc9c665f9cba08fbdc9f657f405713515febe300d", size = 12411100, upload-time = "2026-07-22T22:18:01.485Z" }, + { url = "https://files.pythonhosted.org/packages/fa/9a/31f4983f191af51ab2a8f2d0c7b33dff3a84da26533f982fff02c2f9e28b/pandas-3.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:b1261758dfb6cf12c3cff8300e21cefad30e7ec709abb4c24ac7318e6a52462a", size = 9968804, upload-time = "2026-07-22T22:18:03.903Z" }, + { url = "https://files.pythonhosted.org/packages/49/97/7886c89a39045c69ad82cbceaf3343810480c8ef49a216319ce8183860a6/pandas-3.0.5-cp311-cp311-win_arm64.whl", hash = "sha256:679f4e85b30ddb1515458ab1e788d3e260eae369b1f78da7a3aa4cac8ebf4a2a", size = 9205447, upload-time = "2026-07-22T22:18:06.134Z" }, + { url = "https://files.pythonhosted.org/packages/1c/54/1dc810ea558d1320b597aa140a514f2fdf1d2ea09c38cf556f13ea712ec9/pandas-3.0.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fa290c16964d4963fbfbc358928239cf3bd755b20e988ce944877def2f44471d", size = 10411717, upload-time = "2026-07-22T22:18:08.307Z" }, + { url = "https://files.pythonhosted.org/packages/68/56/fbe81c09195924d8b7b8d4461a20458fe80a6a5ed6b24f0314da684277e1/pandas-3.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c2e26bb46934b8a2ca0c3de1d3d606fc5f6746584791b2db264d58cf370e08dc", size = 9957095, upload-time = "2026-07-22T22:18:10.6Z" }, + { url = "https://files.pythonhosted.org/packages/e0/51/fac252f4a913ed5eabf3c11b880a9e8d5a6c10f0b2129d0462212d238b4d/pandas-3.0.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73fa87b08a7ef706f8aafda39ddaccf2a99047bea62d8c88a0361bcafb2237bc", size = 10485458, upload-time = "2026-07-22T22:18:12.834Z" }, + { url = "https://files.pythonhosted.org/packages/12/98/e976540c1addf70442be7842a18cf70884a964abbf69442504f4d2939989/pandas-3.0.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d373ce03ffd84010ed9839fa73672a9c8256990532e158440c0085db7d914b34", size = 10998091, upload-time = "2026-07-22T22:18:15.209Z" }, + { url = "https://files.pythonhosted.org/packages/a4/8c/1f29b5be8d3fc47dd7567eb167fabba2085879b31e0287ce7cba6d3d2ff4/pandas-3.0.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2a29c53d85ea98c5e792c59ef82ee9fbe6ca902c0d0adb6b23f45ef894cd7bf6", size = 11499501, upload-time = "2026-07-22T22:18:17.689Z" }, + { url = "https://files.pythonhosted.org/packages/9d/e2/bd9c98ad2df7b38bde002adde4cdf353519da51881634323b126c55997f9/pandas-3.0.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a5ad3b02ed6bc7d7ae9b70804b2c6aa31827489d150f8e623ce82491b82085d7", size = 12060559, upload-time = "2026-07-22T22:18:20.147Z" }, + { url = "https://files.pythonhosted.org/packages/f3/9a/ffbd852d58bd74a617fe2f8ee6a58a96982271ce41cf981eab22190b4a4b/pandas-3.0.5-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:b2acb4650527eec6822c3dadb2b771277b65e7dae7a267d4bccf65fd1bb3fbce", size = 7197652, upload-time = "2026-07-22T22:18:22.502Z" }, + { url = "https://files.pythonhosted.org/packages/70/b5/d2d3e9ae73362ba4229651b0ee1455cf78073a1ce585f6ff693782ce263e/pandas-3.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:80a611068e8a3ac23f7398c6c14eb46dc974e5cc9997f653e2dcfd1da74edd41", size = 9831691, upload-time = "2026-07-22T22:18:24.534Z" }, + { url = "https://files.pythonhosted.org/packages/52/51/dea1e89d6a6796b9c43f85a09b484ee03edb8a4c4842e73e200a8c11301c/pandas-3.0.5-cp312-cp312-win_arm64.whl", hash = "sha256:25ff585b972a18ef1fe9ffa3ac6544d9950508aa76832e5147640b6022821e49", size = 9105796, upload-time = "2026-07-22T22:18:27.064Z" }, + { url = "https://files.pythonhosted.org/packages/bf/09/7b95c4a0025227d6f118c4039b423412ac6a982db02864166185d812fbc7/pandas-3.0.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c1c05a767fe8e5b4fe9e1c29806829c582052eaedb9120a3da83ba3f69e24a5b", size = 10385742, upload-time = "2026-07-22T22:18:29.346Z" }, + { url = "https://files.pythonhosted.org/packages/8d/0c/dc78fd8c4da477b4b5e8ad37295af352190d21ef63a9ee1bc071753074cc/pandas-3.0.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b86765f268b56f7e665b93bce9d5df69dee7f99e595cf8fb839483ab315942a3", size = 9932067, upload-time = "2026-07-22T22:18:31.833Z" }, + { url = "https://files.pythonhosted.org/packages/3e/71/3592c055cf44df9808550f9368ceda80ff2b224d355ef73fe251dcda1802/pandas-3.0.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c597ecf5616b5c420372c1d4d4c00dbbfba7398bea857dcc984347e1ea48417b", size = 10466756, upload-time = "2026-07-22T22:18:34.195Z" }, + { url = "https://files.pythonhosted.org/packages/e3/70/4363150359f95b4cb4bcbb34ca23572bb5495749a621a8f3d5a1ddfd293c/pandas-3.0.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b11c36e218331d0387cbe3a0a5f75162357a1d92d57b2b08a336ff94b19b2be", size = 10938525, upload-time = "2026-07-22T22:18:36.81Z" }, + { url = "https://files.pythonhosted.org/packages/f7/d0/317e7a0c67c0e69fa905a0161409397a7dc2d46ff611f6ca4803352c042b/pandas-3.0.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cf52e1f61d229496da17dc7ab54acdee627357e7008fd4fecba3d0ba2937fa58", size = 11489303, upload-time = "2026-07-22T22:18:39.287Z" }, + { url = "https://files.pythonhosted.org/packages/f1/8d/36dade89b49e4f9d5cbdbe863772581f98c0c6d78fc39ad4c557f6f2e17e/pandas-3.0.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:db172144bb56422bd157812f3b021eacc255451470b31e2c633c349490a1cfee", size = 11989004, upload-time = "2026-07-22T22:18:42.208Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ba/18c4ec8a746e177da05a9e7a7963781d8ea195780724f854601b6ebd6b78/pandas-3.0.5-cp313-cp313-win_amd64.whl", hash = "sha256:0d298e951f23016ce4699951d044ae6418dbc91bf68cefca0f77666fcbb4e5c6", size = 9826896, upload-time = "2026-07-22T22:18:44.539Z" }, + { url = "https://files.pythonhosted.org/packages/de/ec/28a57266b753799a87b8bc79e7887ac6fd981b8c6d2978a0b7e7b6bd708c/pandas-3.0.5-cp313-cp313-win_arm64.whl", hash = "sha256:66266d3442a5e8b3c90274c2b8b230bee42dd1c286bc822cc2f9f2c7e12b883e", size = 9094790, upload-time = "2026-07-22T22:18:47.468Z" }, + { url = "https://files.pythonhosted.org/packages/51/2f/cf6aae281264f4463f0875bcbb15fd2bb6d291cc535187dad1732475e4a9/pandas-3.0.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2f264fc46911cc8131a7322a16199bbf8e353d27c10bb211f5bd0c814324dc36", size = 10390034, upload-time = "2026-07-22T22:18:49.818Z" }, + { url = "https://files.pythonhosted.org/packages/06/ec/5189518c7a7659c4bdcc6b1eb32c46c6f3c86b0661ffd84143d1112c7732/pandas-3.0.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:53730687fcd161883b24e10411c06d6a4c0f2275d2faf3bb2bc25deb4ba8007c", size = 9980065, upload-time = "2026-07-22T22:18:52.249Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f1/598503ce8d7e3c35601e0747ba288c7864baae66380725bc12f13f884dfe/pandas-3.0.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:960d3ebcf249f75206899fcd2c6de53f736b7265759ced0d3e559df0b8b709b0", size = 10545532, upload-time = "2026-07-22T22:18:54.813Z" }, + { url = "https://files.pythonhosted.org/packages/fa/de/ceae2adf7034e07e9910299fe412e1819c4f0dd520700a888bcb03625448/pandas-3.0.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e94c2c5ca43bd3ca32bf64d32308887b65e5f9bfd8023ea52755107a999f93b", size = 10963120, upload-time = "2026-07-22T22:18:57.42Z" }, + { url = "https://files.pythonhosted.org/packages/66/25/86e0f4451874eb79e688deeebe3c451fec4557f8952005818d800ee8ac7e/pandas-3.0.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e819dd5f62966b481a8cb649d3299ebd886a1ea91ed5a99bf7ce77c98d18ab94", size = 11563178, upload-time = "2026-07-22T22:18:59.729Z" }, + { url = "https://files.pythonhosted.org/packages/f3/45/8643daa3b4147e433adfcccefdd0380d3aad79d86b15d8999730fe1944d5/pandas-3.0.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3c5ed2e7c06e91d340dfd091d7934f9bc82e4a36b95f647f090b9d1c9ac649da", size = 12028708, upload-time = "2026-07-22T22:19:02.164Z" }, + { url = "https://files.pythonhosted.org/packages/96/58/ad979ae617615576e8aafd569c9d4b62f1191d896e38f51d66ba06f3b89a/pandas-3.0.5-cp314-cp314-win_amd64.whl", hash = "sha256:cd8f7c6dc98527058ee6264219343f5392240a6f1bfa654fc5d79023020d0c92", size = 9951806, upload-time = "2026-07-22T22:19:04.596Z" }, + { url = "https://files.pythonhosted.org/packages/69/32/7ac03886b304049a9d2625ee88f59af760d8a93bd30ed9239bce7b9869a8/pandas-3.0.5-cp314-cp314-win_arm64.whl", hash = "sha256:5183427f5a8156d480f30333777bc978be93650a49a7c01db26adffe95b31e85", size = 9238297, upload-time = "2026-07-22T22:19:06.836Z" }, + { url = "https://files.pythonhosted.org/packages/be/ed/1d1f2ee5547d5167face2376d11c8b2a4c7bfff5a416ee7a9046891fab1e/pandas-3.0.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:303da736987d481074ca720ada325f8bd80c64ebc2d45ed79b29df3aaa4a26ca", size = 10849690, upload-time = "2026-07-22T22:19:09.391Z" }, + { url = "https://files.pythonhosted.org/packages/57/55/17e17152e98fbb0c4b1e562bc65387a2f20a80db0f4a86bf8d3a0e4248d4/pandas-3.0.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:3b2801bbb049d0136f6c213eae02b5fca969384fc2064dd728d8620552aa49da", size = 10509945, upload-time = "2026-07-22T22:19:11.773Z" }, + { url = "https://files.pythonhosted.org/packages/88/90/817d44dbf83facf9556f33576d9af0a241981e7bb5c00606c0bcb5df8dda/pandas-3.0.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cce3a9d11d2b1f82c69a27ec1f4948a170e2c403c4bbfa8cca62e3fdebe2ef3a", size = 10392197, upload-time = "2026-07-22T22:19:14.024Z" }, + { url = "https://files.pythonhosted.org/packages/f1/da/889f00c0a6f5aa1545add70abbf01502dff87ab577adb855bd631c54d2f2/pandas-3.0.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef01af4d8dc6cd2c8d6c7736f149574ef93fe043811eeb5e445f2647154b5040", size = 10862726, upload-time = "2026-07-22T22:19:16.351Z" }, + { url = "https://files.pythonhosted.org/packages/bc/98/f1e934fb3c98fce859c6147c6785816c7b5b9ab7821115c5d8c4de9842b9/pandas-3.0.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e2759e890db96dfcffdbd9b86c3c2cb6afaf58def482820317e06163ec1066cd", size = 11414864, upload-time = "2026-07-22T22:19:18.981Z" }, + { url = "https://files.pythonhosted.org/packages/fe/be/d448af7d657d82e1888dd8551f79c6d6fb161080b5b9752d84d910ec2319/pandas-3.0.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b58b1b39d46a5862e3fb18f50d1a201398619d16a0f9f73f57eea5583cf0e63c", size = 11925105, upload-time = "2026-07-22T22:19:21.515Z" }, + { url = "https://files.pythonhosted.org/packages/29/c1/ccb4238212c8c4f496c584f3044d94e0c030ed8e1d68999db46c91c2242f/pandas-3.0.5-cp314-cp314t-win_amd64.whl", hash = "sha256:1c10461f6eeb35d8f05b6184c65c8b9991663b66c46b1d559b682cb34ae7c6ea", size = 10387612, upload-time = "2026-07-22T22:19:24.257Z" }, + { url = "https://files.pythonhosted.org/packages/d2/cf/6a51b2c38980e04c279fd2fa908a1b0982064e860444acfca4ec2e2c8359/pandas-3.0.5-cp314-cp314t-win_arm64.whl", hash = "sha256:3c5015fd1730fbf883647e88068176c839c102cea883ba1769a6f4593bfc1f8c", size = 9509776, upload-time = "2026-07-22T22:19:26.694Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "propcache" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/44/c87281c333769159c50594f22610f77398a47ccbfbbf23074e744e86f87c/propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427", size = 50208, upload-time = "2026-05-08T21:02:12.199Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/f1/8a8cc1c2c7e7934ab77e0163414f736fadbc0f5e8dd9673b952355ac175b/propcache-0.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78", size = 90744, upload-time = "2026-05-08T20:59:45.799Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f4/651b1225e976bd1a2ba5cfba0c29d096581c2636b437e3a9a7ab6276270a/propcache-0.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959", size = 52033, upload-time = "2026-05-08T20:59:47.408Z" }, + { url = "https://files.pythonhosted.org/packages/15/a8/8ede85d6aa1f79fc7dc2f8fd2c8d65920b8272c3892903c8a1affde48cfb/propcache-0.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7", size = 52754, upload-time = "2026-05-08T20:59:49.202Z" }, + { url = "https://files.pythonhosted.org/packages/7d/fe/b3551b41bbc2f5b5bb088fc6920567cd43101253e68fbaa261339eb96fe1/propcache-0.5.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511", size = 57573, upload-time = "2026-05-08T20:59:50.778Z" }, + { url = "https://files.pythonhosted.org/packages/83/27/ab851ebd1b7172e3e161f5f8d39e315d54a91bea246f01f4d872d3376aef/propcache-0.5.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660", size = 60645, upload-time = "2026-05-08T20:59:52.227Z" }, + { url = "https://files.pythonhosted.org/packages/95/7d/466b3d18022e9897cbda9c735c493c5bd747d7a4c6f5ea1480b4cec434b6/propcache-0.5.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66", size = 61563, upload-time = "2026-05-08T20:59:53.866Z" }, + { url = "https://files.pythonhosted.org/packages/27/1b/16ab7f2cf2041da2f60d156ba64c2484eadf9168075b4ff43c3ef60045af/propcache-0.5.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b", size = 58888, upload-time = "2026-05-08T20:59:55.457Z" }, + { url = "https://files.pythonhosted.org/packages/0a/67/bb777ffd907633563bf35fd859c4ce97b0512c32f4633cf5d1eb7c33512b/propcache-0.5.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67", size = 59253, upload-time = "2026-05-08T20:59:57.075Z" }, + { url = "https://files.pythonhosted.org/packages/b9/42/64f8d90b73fd9cdc1499b48057ff6d9cd2a98a25734c9bb62ecf07e87061/propcache-0.5.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f", size = 57558, upload-time = "2026-05-08T20:59:58.602Z" }, + { url = "https://files.pythonhosted.org/packages/eb/02/dba5bc03c9041f2092ea55a449caf5dfe68352c6654511b29ba0654ddb69/propcache-0.5.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c", size = 55007, upload-time = "2026-05-08T20:59:59.837Z" }, + { url = "https://files.pythonhosted.org/packages/14/c0/43f649c7aa2a77a3b100d84e9dea3a483120ecb608bfe36ce49eaff517fe/propcache-0.5.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0", size = 60355, upload-time = "2026-05-08T21:00:01.144Z" }, + { url = "https://files.pythonhosted.org/packages/83/c0/435dafd27f1cb4a495381dae60e25883ccfe4020bb72818e8184c1678092/propcache-0.5.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6", size = 59057, upload-time = "2026-05-08T21:00:02.401Z" }, + { url = "https://files.pythonhosted.org/packages/53/ae/6e292df9135d659944e96cb3389258e4a663e5b2b5f6c217ef0ddc8d2f73/propcache-0.5.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27", size = 61938, upload-time = "2026-05-08T21:00:03.638Z" }, + { url = "https://files.pythonhosted.org/packages/0b/42/314ebc50d8159055411fd6b0bda322ff510e4b1f7d2e4927940ad0f6af20/propcache-0.5.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f", size = 59731, upload-time = "2026-05-08T21:00:04.881Z" }, + { url = "https://files.pythonhosted.org/packages/b8/9b/2da6dee38871c3c8772fabc2758325a5c9077d6d18c597737dc04dd884cd/propcache-0.5.2-cp311-cp311-win32.whl", hash = "sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0", size = 38966, upload-time = "2026-05-08T21:00:06.511Z" }, + { url = "https://files.pythonhosted.org/packages/42/4e/f17363fb58c0afe05b067361cb6d86ed2d29de6506779a27547c4d183075/propcache-0.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82", size = 42135, upload-time = "2026-05-08T21:00:08.088Z" }, + { url = "https://files.pythonhosted.org/packages/c6/eb/6af6685077d22e8b33358d3c548e3282706a0b3cd85044ffba4e5dd08e3b/propcache-0.5.2-cp311-cp311-win_arm64.whl", hash = "sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab", size = 38381, upload-time = "2026-05-08T21:00:09.692Z" }, + { url = "https://files.pythonhosted.org/packages/4a/cb/e27bc2b2737a0bb49962b275efa051e8f1c35a936df7d5139b6b658b7dc9/propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba", size = 95887, upload-time = "2026-05-08T21:00:11.277Z" }, + { url = "https://files.pythonhosted.org/packages/e6/13/b8ae04c59392f8d11c6cd9fb4011d1dc7c86b81225c770280300e259ffe1/propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a", size = 54654, upload-time = "2026-05-08T21:00:12.604Z" }, + { url = "https://files.pythonhosted.org/packages/2c/7d/49777a3e20b55863d4794384a38acd460c04157b0a00f8602b0d508b8431/propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf", size = 55190, upload-time = "2026-05-08T21:00:13.935Z" }, + { url = "https://files.pythonhosted.org/packages/44/c7/085d0cd63062e84044e3f05797749c3f8e3938ff3aeb0eb2f69d43fafc91/propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144", size = 59995, upload-time = "2026-05-08T21:00:15.526Z" }, + { url = "https://files.pythonhosted.org/packages/9c/42/32cf8e3009e92b2645cf1e944f701e8ea4e924dffde1ee26db860bcbf7e4/propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9", size = 63422, upload-time = "2026-05-08T21:00:16.824Z" }, + { url = "https://files.pythonhosted.org/packages/9e/1b/f112433f99fc979431b87a39ef169e3f8df070d99a72792c56d6937ac48b/propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42", size = 64342, upload-time = "2026-05-08T21:00:18.362Z" }, + { url = "https://files.pythonhosted.org/packages/14/15/5574111ae50dd6e879456888c0eadd4c5a869959775854e18e18a6b345f3/propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476", size = 61639, upload-time = "2026-05-08T21:00:19.692Z" }, + { url = "https://files.pythonhosted.org/packages/cc/da/4d775080b1490c0ae604acda868bd71aabe3a89ed16f2aa4339eb8a283e7/propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba", size = 61588, upload-time = "2026-05-08T21:00:21.155Z" }, + { url = "https://files.pythonhosted.org/packages/04/ac/f076982cbe2195ee9cf32de5a1e46951d9fb399fc207f390562dd0fd8fb2/propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a", size = 60029, upload-time = "2026-05-08T21:00:22.713Z" }, + { url = "https://files.pythonhosted.org/packages/70/60/189be62e0dd898dce3b331e1b8c7a543cd3a405ac0c81fe8ee8a9d5d77e1/propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64", size = 56774, upload-time = "2026-05-08T21:00:24.001Z" }, + { url = "https://files.pythonhosted.org/packages/ea/9e/93377b9c7939c1ffae98f878dee955efadfd638078bc86dbc21f9d52f651/propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913", size = 63532, upload-time = "2026-05-08T21:00:25.545Z" }, + { url = "https://files.pythonhosted.org/packages/14/f9/590ef6cfb9b8028d516d287812ece32bb0bc5f11fbb9c8bf6b2e6313fec8/propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1", size = 61592, upload-time = "2026-05-08T21:00:27.186Z" }, + { url = "https://files.pythonhosted.org/packages/b4/5e/70958b3034c297a630bba2f17ca7abc2d5f39a803ad7e370ab79d1ecd022/propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33", size = 64788, upload-time = "2026-05-08T21:00:28.8Z" }, + { url = "https://files.pythonhosted.org/packages/12/fd/77fe5936d8c3086ca9048f7f415f122ed82e53884a9ec193646b42deef06/propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a", size = 62514, upload-time = "2026-05-08T21:00:30.098Z" }, + { url = "https://files.pythonhosted.org/packages/cf/74/66bd798b5b3be70aa1b391f5cc9d6a0a5532d7fd3b19ec0b213e72e6ad9d/propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031", size = 39018, upload-time = "2026-05-08T21:00:31.622Z" }, + { url = "https://files.pythonhosted.org/packages/61/7c/5c0d34aa3024694d6dcb9271cdbdd08c4e47c1c0ad95ec7e7bc74cdea145/propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42", size = 42322, upload-time = "2026-05-08T21:00:32.918Z" }, + { url = "https://files.pythonhosted.org/packages/4d/91/875812f1a3feb20ceba818ef39fbe4d92f1081e04ac815c822496d0d038b/propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84", size = 38172, upload-time = "2026-05-08T21:00:35.124Z" }, + { url = "https://files.pythonhosted.org/packages/c5/09/f049e45385503fe67db75a6b6186a7b9f0c3930366dc960522c312a825b1/propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a", size = 94457, upload-time = "2026-05-08T21:00:36.355Z" }, + { url = "https://files.pythonhosted.org/packages/6b/65/83d1d05655baf63113731bd5a1008435e14f8d1e5a06cbe4ec5b23ad7a31/propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117", size = 53835, upload-time = "2026-05-08T21:00:38.072Z" }, + { url = "https://files.pythonhosted.org/packages/a9/12/a6ba6482bb5ea3260c000c9b20881c95fa11c6b30173715668259f844ed7/propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098", size = 54545, upload-time = "2026-05-08T21:00:39.319Z" }, + { url = "https://files.pythonhosted.org/packages/a9/19/7fa086f5764c59ec8a8e157cd93aa8497acc00aba9dcdec56bfffb32602d/propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4", size = 59886, upload-time = "2026-05-08T21:00:40.621Z" }, + { url = "https://files.pythonhosted.org/packages/a1/e4/5d7663dc8235956c8f5281698a3af1d351d8820341ddd890f59d9a9127f2/propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e", size = 63261, upload-time = "2026-05-08T21:00:41.775Z" }, + { url = "https://files.pythonhosted.org/packages/4a/4a/15a03adee24d6350da4292caeac44c34c033d2afe5e87eb370f38854560f/propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7", size = 64184, upload-time = "2026-05-08T21:00:43.018Z" }, + { url = "https://files.pythonhosted.org/packages/8b/c6/979176efdaa3d239e36d503d5af63a0a773b36662ed8f52e5b6a6d9fd40e/propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d", size = 61534, upload-time = "2026-05-08T21:00:44.507Z" }, + { url = "https://files.pythonhosted.org/packages/c8/22/63e8cd1bae4c2d2be6493b6b7d10566ddafad88137cfbc99964a1119853c/propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a", size = 61500, upload-time = "2026-05-08T21:00:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/60/5a/28e5d9acbac1cc9ccb67045e8c1b943aa8d79fdf39c93bd73cacd68008ea/propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2", size = 59994, upload-time = "2026-05-08T21:00:47.093Z" }, + { url = "https://files.pythonhosted.org/packages/f3/40/db650677f554a95b9c01a7c9d93d629e93a15562f5deb4573c9ee136fed2/propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa", size = 56884, upload-time = "2026-05-08T21:00:48.376Z" }, + { url = "https://files.pythonhosted.org/packages/80/45/70b39b89516ff8b96bf732fa6fded8cef20f293cb1508690101c3c07ec51/propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853", size = 63464, upload-time = "2026-05-08T21:00:49.954Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e2/fa59d3a89eac5534293124af4f1d0d0ada091ce4a0ab4610ce03fd2bdd8d/propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a", size = 61588, upload-time = "2026-05-08T21:00:51.281Z" }, + { url = "https://files.pythonhosted.org/packages/0b/97/efb547a55c4bc7381cfb202d6a2239ac621045277bc1ea5dfd3a7f0516c0/propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704", size = 64667, upload-time = "2026-05-08T21:00:52.602Z" }, + { url = "https://files.pythonhosted.org/packages/92/56/f5c7d9b4b7595d5127da38974d791b2153f3d1eae6c674af3583ace92ad3/propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4", size = 62463, upload-time = "2026-05-08T21:00:54.303Z" }, + { url = "https://files.pythonhosted.org/packages/bd/3b/484a3a65fc9f9f60c41dcd17b428bace5389544e2c680994534a20755066/propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d", size = 38621, upload-time = "2026-05-08T21:00:55.808Z" }, + { url = "https://files.pythonhosted.org/packages/1c/fd/3f0f10dba4dabad3bf53102be007abf55481067952bde0fdddff439e7c61/propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757", size = 41649, upload-time = "2026-05-08T21:00:57.061Z" }, + { url = "https://files.pythonhosted.org/packages/90/ec/6ce619cc32bb500a482f811f9cd509368b4e58e638d13f2c68f370d6b475/propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f", size = 37636, upload-time = "2026-05-08T21:00:58.646Z" }, + { url = "https://files.pythonhosted.org/packages/1b/82/c1d268bbbf2ef981c5bf0fbbe746db617c66e3bcefe431a1aa8943fbe23a/propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d", size = 98872, upload-time = "2026-05-08T21:00:59.889Z" }, + { url = "https://files.pythonhosted.org/packages/f4/d4/52c871e73e864e6b34c0e2d58ac1ec5ccd149497ddc7ad2137ae98323a35/propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa", size = 56257, upload-time = "2026-05-08T21:01:01.195Z" }, + { url = "https://files.pythonhosted.org/packages/67/f0/9b90ca2a210b3d09bcfcd96ecd0f55545c091535abce2a45de2775cfd357/propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94", size = 56696, upload-time = "2026-05-08T21:01:02.941Z" }, + { url = "https://files.pythonhosted.org/packages/9d/0e/6e9d4ba07c8e56e21ddec1e75f12148142b21ca83a51871babce095334f4/propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164", size = 62378, upload-time = "2026-05-08T21:01:04.475Z" }, + { url = "https://files.pythonhosted.org/packages/65/19/c10badaa463dde8a27ce884f8ee2ec37e6035b7c9f5ff0c8f74f06f08dac/propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f", size = 65283, upload-time = "2026-05-08T21:01:05.959Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b6/93bea99ca80e19cef6512a8580e5b7857bbe09422d9daa7fd4ef5723306c/propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c", size = 66616, upload-time = "2026-05-08T21:01:07.228Z" }, + { url = "https://files.pythonhosted.org/packages/83/e4/5c7462e50625f051f37fb38b8224f7639f667184bbd34424ec83819bb1b7/propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc", size = 63773, upload-time = "2026-05-08T21:01:08.514Z" }, + { url = "https://files.pythonhosted.org/packages/ca/b6/99238894047b13c823be25027e736626cd414a52a5e30d2c3347c2733529/propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f", size = 63664, upload-time = "2026-05-08T21:01:09.874Z" }, + { url = "https://files.pythonhosted.org/packages/85/1e/a3a1a63116a2b8edb415a8bb9a6f0c34bd03830b1e18e8ce2904e1dc1cf4/propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb", size = 62643, upload-time = "2026-05-08T21:01:11.132Z" }, + { url = "https://files.pythonhosted.org/packages/e4/03/893cf147de2fc6543c5eaa07ad833170e7e2a2385725bbebe8c0503723bb/propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751", size = 59595, upload-time = "2026-05-08T21:01:12.387Z" }, + { url = "https://files.pythonhosted.org/packages/86/3b/04c1a2e12c57766568ba75ba72b3bf2042818d4c1425fab6fc07155c7cff/propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836", size = 65711, upload-time = "2026-05-08T21:01:13.676Z" }, + { url = "https://files.pythonhosted.org/packages/1c/34/80f8d0099f8d6bacc4de1624c85672681c8cd1149ca2da0e38fd120b817f/propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f", size = 64247, upload-time = "2026-05-08T21:01:14.936Z" }, + { url = "https://files.pythonhosted.org/packages/f3/1a/8b08f3a5f1037e9e370c55883ceeeee0f6dd0416fb2d2d67b8bfc91f2a79/propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55", size = 67102, upload-time = "2026-05-08T21:01:16.281Z" }, + { url = "https://files.pythonhosted.org/packages/34/68/8bdb7bb7756d76e005490649d10e4a8369e610c74d619f71e1aedf889e9c/propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568", size = 64964, upload-time = "2026-05-08T21:01:17.57Z" }, + { url = "https://files.pythonhosted.org/packages/0a/aa/50fb0b5d3968b61a510926ff8b8465f1d6e976b3ab74496d7a4b9fc42515/propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191", size = 42546, upload-time = "2026-05-08T21:01:18.946Z" }, + { url = "https://files.pythonhosted.org/packages/ae/4c/0ddbae64321bd4a95bcbfc19307238016b5b1fee645c84626c8d539e5b74/propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7", size = 46330, upload-time = "2026-05-08T21:01:20.162Z" }, + { url = "https://files.pythonhosted.org/packages/00/d9/9cddc8efb78d8af264c5ec9f6d10b62f57c515feda8d321595f56010fb23/propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96", size = 40521, upload-time = "2026-05-08T21:01:21.399Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ea/23ee535d90ce8bcc465a3028eb3cc0ce3bd1005f4bb27710b30587de798d/propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999", size = 94662, upload-time = "2026-05-08T21:01:22.683Z" }, + { url = "https://files.pythonhosted.org/packages/b5/06/c5a52f419b5d8972f8d46a7577476090d8e3263ff589ce40b5ca4968d5be/propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e", size = 53928, upload-time = "2026-05-08T21:01:23.986Z" }, + { url = "https://files.pythonhosted.org/packages/63/b1/4260d67d6bd85e58a66b72d54ce15d5de789b6f3870cc6bedf8ff9667401/propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539", size = 54650, upload-time = "2026-05-08T21:01:25.305Z" }, + { url = "https://files.pythonhosted.org/packages/70/06/2f46c318e3307cd7a6a7481def374ce838c0fe20084b39dd54b0879d0e99/propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e", size = 59912, upload-time = "2026-05-08T21:01:26.545Z" }, + { url = "https://files.pythonhosted.org/packages/4c/29/fe1aebec2ce57ab985a9c382bded1124431f85078113aa222c5d278430d4/propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979", size = 63300, upload-time = "2026-05-08T21:01:27.937Z" }, + { url = "https://files.pythonhosted.org/packages/b4/18/2334b26768b6c82be8c69e83671b767d5ef426aa09b0cba6c2ea47816774/propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80", size = 64208, upload-time = "2026-05-08T21:01:29.484Z" }, + { url = "https://files.pythonhosted.org/packages/2b/76/7f1bfd6afff4c5e38e36a3c6d68eb5f4b7311ea80baf693db78d95b603c4/propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825", size = 61633, upload-time = "2026-05-08T21:01:31.068Z" }, + { url = "https://files.pythonhosted.org/packages/c4/46/b3ff8aba2b4953a3e50de2cf72f1b5748b8eca93b15f3dc2c84339084c09/propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39", size = 61724, upload-time = "2026-05-08T21:01:32.374Z" }, + { url = "https://files.pythonhosted.org/packages/c5/01/814cfcafbcff954f94c01cf30e097ddc88a076b5440fbcf4570753437d40/propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4", size = 60069, upload-time = "2026-05-08T21:01:33.67Z" }, + { url = "https://files.pythonhosted.org/packages/da/68/5c6f7622d510cc666a300687e06fd060c1a43361c0c9b20d284f06d8096a/propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5", size = 57099, upload-time = "2026-05-08T21:01:34.915Z" }, + { url = "https://files.pythonhosted.org/packages/55/27/9cb0b4c679124085327957d42521c99dba04c88c90c3e55a6f0b633ebccc/propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702", size = 63391, upload-time = "2026-05-08T21:01:36.231Z" }, + { url = "https://files.pythonhosted.org/packages/f0/9d/7258aaa5bdf60fc6f27591eef6fe52768cb0beda7140be477c8b12c9794a/propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3", size = 61626, upload-time = "2026-05-08T21:01:37.545Z" }, + { url = "https://files.pythonhosted.org/packages/8e/0d/41c602003e8a9b16fe1e7eadf62c7bfba9d5474370b24200bf48b315f45f/propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5", size = 64781, upload-time = "2026-05-08T21:01:38.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f3/38e66b1856e9bd079deea015bc4a55f7767c0e4db2f7dcf69e7e680ba4ce/propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4", size = 62570, upload-time = "2026-05-08T21:01:40.415Z" }, + { url = "https://files.pythonhosted.org/packages/95/ca/bbfe9b910ce57dde8bb4876b4520fc02a4e89497c10de26be936758a3aaa/propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0", size = 39436, upload-time = "2026-05-08T21:01:41.654Z" }, + { url = "https://files.pythonhosted.org/packages/61/d2/45c9defbaa1ea297035d9d4cce9e8f80daafbf19319c6007f157c6256ea9/propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c", size = 42373, upload-time = "2026-05-08T21:01:43.041Z" }, + { url = "https://files.pythonhosted.org/packages/44/68/9ea5103f41d5217d7d6ec24db90018e23aebec070c3f9a6e54d12b841fd8/propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0", size = 38554, upload-time = "2026-05-08T21:01:44.336Z" }, + { url = "https://files.pythonhosted.org/packages/8a/81/fadf555f42d3b762eea8a53950b0489fdc0aa9da5f8ed9e10ce0a4e01b48/propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb", size = 99395, upload-time = "2026-05-08T21:01:45.883Z" }, + { url = "https://files.pythonhosted.org/packages/f5/c9/c61e134a686949cf7971af3a390148b1156f7be81c73bc0cd12c873e2d48/propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078", size = 56653, upload-time = "2026-05-08T21:01:47.307Z" }, + { url = "https://files.pythonhosted.org/packages/cb/73/daf935ea7048ddd7ec8eec5345b4a40b619d2d178b3c0a0900796bc3c794/propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa", size = 56914, upload-time = "2026-05-08T21:01:48.573Z" }, + { url = "https://files.pythonhosted.org/packages/79/9f/aba959b435ea18617edd7cf0a7ad0b9c574b8fc7e3d2cd55fb59cb255d33/propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917", size = 62567, upload-time = "2026-05-08T21:01:49.903Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a1/859942de9a791ff42f6141736f5b37749b8f53e65edfa49638c67dd67e6a/propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe", size = 65542, upload-time = "2026-05-08T21:01:51.204Z" }, + { url = "https://files.pythonhosted.org/packages/b5/61/315bc0fd6c0fc7f80a528b8afd209e5fc4a875ea79571b91b8f50f442907/propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03", size = 66845, upload-time = "2026-05-08T21:01:52.539Z" }, + { url = "https://files.pythonhosted.org/packages/47/f7/9f8122e3132e8e354ac41975ef8f1099be7d5a16bc7ae562734e993665c0/propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335", size = 63985, upload-time = "2026-05-08T21:01:53.847Z" }, + { url = "https://files.pythonhosted.org/packages/c8/54/c317819ec157cbf6f35df9df9657a6f82daf34d5faf15948b2f639c2192e/propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285", size = 63999, upload-time = "2026-05-08T21:01:55.179Z" }, + { url = "https://files.pythonhosted.org/packages/5a/56/387e3f7dfce0a9233df41fb888aa1c30222cb4bbbf09537c02dd9bd85fe2/propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837", size = 62779, upload-time = "2026-05-08T21:01:57.489Z" }, + { url = "https://files.pythonhosted.org/packages/a1/9c/596784cb5824ed61ee960d3f8655a3f0993e107c6e98ab6c818b7fb92ccb/propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8", size = 59796, upload-time = "2026-05-08T21:01:58.736Z" }, + { url = "https://files.pythonhosted.org/packages/c2/3d/1a6cfa1726a48542c1e8784a0761421476a5b68e09b7f36bf95eb954aaba/propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366", size = 66023, upload-time = "2026-05-08T21:02:00.228Z" }, + { url = "https://files.pythonhosted.org/packages/e4/0e/05fd6990369477076e4e280bcb970de760fddf0161a46e988bc95f7940ec/propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56", size = 64448, upload-time = "2026-05-08T21:02:01.888Z" }, + { url = "https://files.pythonhosted.org/packages/cd/86/5f8da315a4309c62c10c0b2516b17492d5d3bbe1bb862b96604db67e2a37/propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d", size = 67329, upload-time = "2026-05-08T21:02:03.484Z" }, + { url = "https://files.pythonhosted.org/packages/da/d3/3368efe79ab21f0cdf86ef49895811c9cc933131d4cde1f28a624e22e712/propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2", size = 65172, upload-time = "2026-05-08T21:02:04.745Z" }, + { url = "https://files.pythonhosted.org/packages/d5/07/127e8b0bacfb325396196f9d976a22453049b89b9b2b08477cc3145faa44/propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821", size = 43813, upload-time = "2026-05-08T21:02:06.025Z" }, + { url = "https://files.pythonhosted.org/packages/88/fb/46dad6c0ae49ed230ab1b16c890c2b6314e2403e6c412976f4a72d64a527/propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370", size = 47764, upload-time = "2026-05-08T21:02:07.353Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c4/a47d0a63aa309d10d59ede6e9d4cff03a344a79d1f0f4cd0cd74997b53e0/propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6", size = 41140, upload-time = "2026-05-08T21:02:09.065Z" }, + { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, +] + +[[package]] +name = "pyarrow" +version = "25.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/f3/95428098d1fa7d04432fb750eed06b41304c2f6a5d3319985e64db2d9d41/pyarrow-25.0.0.tar.gz", hash = "sha256:d2d697008b5ec06d75952ef260c2e9a8a0f6ccfce24266c04c9c8ade927cb3b4", size = 1199181, upload-time = "2026-07-10T08:29:50.116Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/98/ae2b5acf9876dbeffa6f320776242c52caab062df55c8ac5501ed2679e74/pyarrow-25.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:2e3b6544e26e393fe2cd530f523e36c1c8d3c345bbbb60cca3fd866be8322517", size = 35939080, upload-time = "2026-07-10T08:26:04.53Z" }, + { url = "https://files.pythonhosted.org/packages/80/09/3de2a968edbd496c86cb8b932cdbee2d4b08c4a28e9884a15e5c705a646b/pyarrow-25.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:b724d127783b4c19f088fcdfc844cbc318809246a30307bcabd5ed02045e890e", size = 37633420, upload-time = "2026-07-10T08:26:10.354Z" }, + { url = "https://files.pythonhosted.org/packages/19/86/8399243a4ce080426ec37db18d5e29148b7ec960a8a8c7f9059a7bf6ef0a/pyarrow-25.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:244f98a595f70fa4fd35faa7508c4ae67e14a173397a4b3b49d2b3c360fb0062", size = 46861050, upload-time = "2026-07-10T08:26:16.397Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/72d704b02bc5fc6d06954d76a0208c1e79cad3ab370f6d6a91ffe5078870/pyarrow-25.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:0222f0071d13313962a88d21bf28b80d355ac39d81bfa6ff3fe00eeaf748e4be", size = 50056458, upload-time = "2026-07-10T08:26:23.271Z" }, + { url = "https://files.pythonhosted.org/packages/06/5d/3c31a60b6403d63cad2e0f829096f5fc5763a129ead4207a5d4690b96448/pyarrow-25.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b58726f118c079f9d4ed7e904975d4f15fd69d0741ba511a4e2dcaa4ef16354f", size = 49957793, upload-time = "2026-07-10T08:26:30.232Z" }, + { url = "https://files.pythonhosted.org/packages/34/f7/8f8a019061f9863a831915329264372a87ed25eaf9109ce56eb0e84012c5/pyarrow-25.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:38a2c887cb3883e241b70201688db34133b6dfadd04f03c8f9213df53770c18e", size = 53100544, upload-time = "2026-07-10T08:26:36.414Z" }, + { url = "https://files.pythonhosted.org/packages/f1/e2/738071e95c5ddad7b3dfc12f569ffa992db89d7d7b4a95258fd184191249/pyarrow-25.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:161649d60a7a46c613a19fd795763ea8a88c36ba997dd99d9bc66e6794ee36e8", size = 27848311, upload-time = "2026-07-10T08:26:41.429Z" }, + { url = "https://files.pythonhosted.org/packages/73/44/fdd3a4377807b7dcabe2d4b5aa99dbbc98e2e5df3f1ca4e7f0aec492d987/pyarrow-25.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:149730a3d1f0fb59d663a0b8aa210adfd9c17c27cd94a0d143e60daea8320d4e", size = 35850884, upload-time = "2026-07-10T08:26:47.357Z" }, + { url = "https://files.pythonhosted.org/packages/bf/71/9f053177a7709b8c90abb00a2375b916286f9f0d6cfb21a5cadd4ef811e8/pyarrow-25.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:0721332c30fdd453fdd1fc203b2ac1f4c9db5aea28fa38d41f2574c4b068b9ec", size = 37616197, upload-time = "2026-07-10T08:26:53.564Z" }, + { url = "https://files.pythonhosted.org/packages/95/1a/22bfb6597dcdc861fa83c39c06e1457cb56f698940eff42fbb25de30e8e5/pyarrow-25.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fa1482b3da10cac2d4db6e26b81da543e237616af2ef6d466018b31ca586496f", size = 46841966, upload-time = "2026-07-10T08:27:07.685Z" }, + { url = "https://files.pythonhosted.org/packages/55/0e/cd705c042bc4fe7022478db577fcab4abdcfabb9bc37ab7a75556b3fcb2b/pyarrow-25.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5d1dbf24e151042f2fa3c129563f65d66674128868496fb008c4272b16bdf778", size = 50088993, upload-time = "2026-07-10T08:27:14.268Z" }, + { url = "https://files.pythonhosted.org/packages/98/ee/d822e1ee31fe31ec5d057210e0605c950b975dcd8d9a332976cc859a9df8/pyarrow-25.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:20887a762dd61dcc530f93a140840ab1f6aa7836b33270e42d627ab3cf11e537", size = 49941005, upload-time = "2026-07-10T08:27:21.274Z" }, + { url = "https://files.pythonhosted.org/packages/33/1b/207a90cc64619a095eb75a263ae069735f2810056d43c667befd573ec083/pyarrow-25.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:58d1ab556b0cea1c93fdb799b24ad58adb2f2a2788dbce782a94f64ae1a5cc9b", size = 53112355, upload-time = "2026-07-10T08:27:27.911Z" }, + { url = "https://files.pythonhosted.org/packages/7e/fe/81d1e5f8beed15c01e98649d5c6e2167b67fd395884a2488f18bf1cf0dba/pyarrow-25.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:3f356afe61186395c861d5cd63dc21ff7d5fa335012a4668d979257df7fea0f5", size = 27945954, upload-time = "2026-07-10T08:27:32.903Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c8/098ce17d778fd9d29e40bb8c5f19a40cc90c3f0b46c9057b0d7993f42f54/pyarrow-25.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:8831a3ba52fa7cdb78d368d968b1dcd06171e6dff5461e16d90de91d371e47bc", size = 35844549, upload-time = "2026-07-10T08:27:37.956Z" }, + { url = "https://files.pythonhosted.org/packages/bc/66/24c28877219abf6263d909b1592c97ff82c59f13a59acbed11fc87c0654f/pyarrow-25.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:5f4bacb60f91dd2fca6c52f1b9a0012cd090e0294f1f781dc1881a247a352f8e", size = 37610397, upload-time = "2026-07-10T08:27:43.803Z" }, + { url = "https://files.pythonhosted.org/packages/53/55/6d1d5f5aff317ec5de9421594679ed51ed828fe7e2ce209327f819d801e4/pyarrow-25.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:59516c822d5fd8e544aaa0dfe72f36fed5d4c24ea8390aab1bcd31d7e959c6be", size = 46841701, upload-time = "2026-07-10T08:27:49.741Z" }, + { url = "https://files.pythonhosted.org/packages/b5/5d/f790fb6965ab54c9da0dda7856abc75fd0d7648d865f8d603c111d203a64/pyarrow-25.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f9dbd83e91c239a1f5ee7ce13f108b5f6c0efbe40a4375260d8f08b43ad05e9", size = 50090118, upload-time = "2026-07-10T08:27:56.051Z" }, + { url = "https://files.pythonhosted.org/packages/0c/8c/faf025357ebf31bc96777f234277aa31e2aeca6dd4ecaa391f29085473c2/pyarrow-25.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:18dcc8cc50b5e72eae6fcbfc6c8776c21a007176b27a3cdec5c2f5bcf126708d", size = 49945559, upload-time = "2026-07-10T08:28:01.927Z" }, + { url = "https://files.pythonhosted.org/packages/07/a1/bd051871708ea99a5e0fc711926c26c6f2c6d0130c7aaac8093e34998af6/pyarrow-25.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4ec1895a87aa834c3b99b7a1e758747eb8bb57f922b32c0e0fa04afb8d6998b1", size = 53114238, upload-time = "2026-07-10T08:28:08.594Z" }, + { url = "https://files.pythonhosted.org/packages/7c/31/737f0c3cffcd6af647849477d1dd68045deac2e3963c3f9f211bedc48540/pyarrow-25.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:77c8d1ae46a44b4006e8db1cc977bbcc6ce4873c92f74137d68e45503b97fb18", size = 27861162, upload-time = "2026-07-10T08:28:12.975Z" }, + { url = "https://files.pythonhosted.org/packages/55/c7/581ccbcdb3d897eb2893328d68db3d52eca373bf2a7e964d0a6276b8e85b/pyarrow-25.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:72132b9a8a0a1840197794d4dea26080069b6b0981c116bc078762dc9691b21b", size = 35878945, upload-time = "2026-07-10T08:28:18.222Z" }, + { url = "https://files.pythonhosted.org/packages/64/d1/ccb01db7329ea0411ef4fbd9b62a04d3268b36777d4e758d5e39b91ddeab/pyarrow-25.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e009ef945e498dca2f050ea10d2e9764cb44017254826fc4574fdb8d2530173b", size = 37630854, upload-time = "2026-07-10T08:28:23.452Z" }, + { url = "https://files.pythonhosted.org/packages/af/9f/2d81ba89d1e4198d0cb25fe7529de936830fdaec0db926bb52a1ef7080d4/pyarrow-25.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f57a39dbcb416345401c2e77a4373669b45fd111a1768e6cf267a7a0607ff0ec", size = 46905617, upload-time = "2026-07-10T08:28:29.376Z" }, + { url = "https://files.pythonhosted.org/packages/6a/29/0ed312ec800fb536f93783215126cee4b8977dcfeccba6f0f44df0cc87d7/pyarrow-25.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:447df764beb07c544f0178a5f6b70ef44b9ecf382b3cdfad4c2d7867353c3887", size = 50119765, upload-time = "2026-07-10T08:28:35.826Z" }, + { url = "https://files.pythonhosted.org/packages/ca/88/cab5063ba0c4d46a9f6b4b7eb1c9029dc0302d65cd5ab3510c949a386568/pyarrow-25.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ac5dfeee59f9ceb4d45ba76e83b026c38c24334135bb329d8274baa49cec3c62", size = 50027563, upload-time = "2026-07-10T08:28:43.848Z" }, + { url = "https://files.pythonhosted.org/packages/7b/fb/4d24f1b7fe2e042dc4ef315ef75e4e702d8e46fe10c37e63caff00502b03/pyarrow-25.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0f100dacf2c0f400601664a79d1a907ced4740514bb2b00917341038e2ce76f", size = 53162437, upload-time = "2026-07-10T08:28:52.819Z" }, + { url = "https://files.pythonhosted.org/packages/fa/65/da20806de93ca6ee91e72cb6a9b08b3ac890b46efc8d94a7326c651c4c81/pyarrow-25.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:2e093efbecb5317372f819228fa4b4e6157eee48d3f0a7b0303705ebf81a7104", size = 28613262, upload-time = "2026-07-10T08:29:47.544Z" }, + { url = "https://files.pythonhosted.org/packages/86/9f/c632afb1d3ef4a7814cee236718235f3a47eac46e97eb87df40f550b6b48/pyarrow-25.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:26be35b80780d2d21f4bae3d568b1666337c3a89722cc1794c956a77017cb24e", size = 36120702, upload-time = "2026-07-10T08:28:59.577Z" }, + { url = "https://files.pythonhosted.org/packages/36/0a/093d53a0e72ad06e45d6443e00651bbc2d21af4211295086cbf4d873d3b9/pyarrow-25.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:6f4812bfbf11ca7d8faf59eb8fff8bf4dd25ce3a38b62baa010cc17a0926d1b2", size = 37750674, upload-time = "2026-07-10T08:29:06.916Z" }, + { url = "https://files.pythonhosted.org/packages/8a/18/b37fc31a69cff4bdfb8842683def5612f551b93fff6f44375e4a4a6a5535/pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:b8af8ceedf0c9c160fd2b63440f2d205b9404db85866c1217bfea601de7cfb50", size = 46912304, upload-time = "2026-07-10T08:29:14.656Z" }, + { url = "https://files.pythonhosted.org/packages/32/35/5cae19ba72493e5598022468b56f6a5571f399f485bf412f157356476caa/pyarrow-25.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:c70a5fd9a82bd1a702fd482bdc62d38dcb672fb2b449b1d7c0d7d1f4be7b7bfe", size = 50073652, upload-time = "2026-07-10T08:29:22.467Z" }, + { url = "https://files.pythonhosted.org/packages/2e/a5/ddd508424bdfd5e6945765e9e2ffc687e2f6115972badc8ecf423076c407/pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0490a7f8b38ffe11cc26526b50c65d111cb54ddac3717cec781806793f1244dc", size = 50058654, upload-time = "2026-07-10T08:29:29.689Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a4/324d0db203ff5eebe8694ec2d6ec5a23f9aaa5d02e5b8c692914c518c33c/pyarrow-25.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e83916bbcf380866b4e14255850b33323ff678dc9758411d0409cdd2523880b0", size = 53140153, upload-time = "2026-07-10T08:29:36.041Z" }, + { url = "https://files.pythonhosted.org/packages/bd/8d/d236e9c82fe315f9128885c8be3ec719f41965a1eb6b6f4b42470904cd41/pyarrow-25.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:13240f0d3dc5932ccd0bfa90cd76d835680b9d94a7661c635df4b703d40ce849", size = 28743657, upload-time = "2026-07-10T08:29:42.742Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pytest-cov" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage", extra = ["toml"] }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "recurquant" +version = "0.2.0a1" +source = { editable = "." } +dependencies = [ + { name = "huggingface-hub" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "packaging" }, + { name = "safetensors" }, + { name = "torch" }, + { name = "transformers" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-cov" }, + { name = "ruff" }, +] +eval = [ + { name = "datasets" }, +] + +[package.metadata] +requires-dist = [ + { name = "datasets", marker = "extra == 'eval'", specifier = ">=4.3,<5" }, + { name = "huggingface-hub", specifier = ">=0.36" }, + { name = "numpy", specifier = ">=2.0" }, + { name = "packaging", specifier = ">=24" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.4" }, + { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=6.2" }, + { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.12" }, + { name = "safetensors", specifier = ">=0.5" }, + { name = "torch", specifier = ">=2.7" }, + { name = "transformers", specifier = "==5.14.1" }, +] +provides-extras = ["dev", "eval"] + +[[package]] +name = "regex" +version = "2026.7.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/98/04b13f1ddfb63158025291c02e03eb42fbb7acb51d091d541050eb4e35e8/regex-2026.7.19.tar.gz", hash = "sha256:7e77b324909c1617cbb4c668677e2c6ae13f44d7c1de0d4f15f2e3c10f3315b5", size = 416440, upload-time = "2026-07-19T00:19:48.923Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/e5/cef4de2bac939280b68d32adc659478845238a8274f2f79c465063f590ad/regex-2026.7.19-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ac777001cdfc28b72477d93c8564bb7583081ea8fb45cdca3d568e0a4f87183c", size = 494012, upload-time = "2026-07-19T00:16:39.927Z" }, + { url = "https://files.pythonhosted.org/packages/ff/87/e86f51eb117457bb7803132ffe5cb6e2841e2b5bea4cc85d397f3c6e257d/regex-2026.7.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:59787bd5f8c70aa339084e961d2996b53fbdeab4d5393bba5c1fe1fc32e02bae", size = 295281, upload-time = "2026-07-19T00:16:41.433Z" }, + { url = "https://files.pythonhosted.org/packages/41/2e/2360c41d8080a3d9ec7e5c90fad6eab3b50192869d10e9a5609e48c8177b/regex-2026.7.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:90c633e7e8d6bf4e992b8b36ce69e018f834b641dd6de8cea6d78c06ffa119c5", size = 290615, upload-time = "2026-07-19T00:16:43.058Z" }, + { url = "https://files.pythonhosted.org/packages/cf/69/b65ba4344efbc771b28fe5dde84cbbb6c8f9551165952fe78def5b9dde6a/regex-2026.7.19-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87ccab0db8d5f4fbb0272642113c1adb2ffc698c16d3a0944580222331fa7a20", size = 791804, upload-time = "2026-07-19T00:16:44.662Z" }, + { url = "https://files.pythonhosted.org/packages/81/b6/a40dfa0dc6224b36f620c00296eacc830489cbf8c2837b6750dfe6170375/regex-2026.7.19-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e50d748a32da622f256e8d505867f5d3c43a837c6a9f0efb149655fadd1042a", size = 861723, upload-time = "2026-07-19T00:16:46.412Z" }, + { url = "https://files.pythonhosted.org/packages/e3/02/735991dee71abd83196a7962f7ed8bf5aa05720ff06e2d3ff896a85e2bbb/regex-2026.7.19-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf1516fe58fc104f39b2d1dbe2d5e27d0cd45c4be2e42ba6ee0cc763701ec3c7", size = 905932, upload-time = "2026-07-19T00:16:47.956Z" }, + { url = "https://files.pythonhosted.org/packages/45/6c/e7098d8b846ccdbf431d8c081b61e496526a27a28094ed09e0dce21b3f54/regex-2026.7.19-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09f3e5287f94f17b709dc9a9e70865855feee835c861613be144218ce4ca82cc", size = 801407, upload-time = "2026-07-19T00:16:49.43Z" }, + { url = "https://files.pythonhosted.org/packages/8a/18/34b69274e2649bcc7d9b089c2b2983fb2632d8ecf667e359593be9072e79/regex-2026.7.19-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6383cd2ed53a646c659ba1fe65727db76437fdaa069e697a0b44a51d5843d864", size = 774448, upload-time = "2026-07-19T00:16:51.352Z" }, + { url = "https://files.pythonhosted.org/packages/bb/e6/0a72247d025585fd3800b98e040b84d562a88af6303347100484849f4f01/regex-2026.7.19-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:09d3007fc76249a83cdd33de160d50e6cb77f54e09d8fa9e7148e10607ce24af", size = 783297, upload-time = "2026-07-19T00:16:53.071Z" }, + { url = "https://files.pythonhosted.org/packages/b1/aa/c4f65ae7dd02a36b323a70c4cff326e1f3442361aaebc9311100a130d54f/regex-2026.7.19-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:6f8c6e7a1cfa3dc9d0ee2de0e65e834537fa29992cc3976ffec914afc35c5dd5", size = 854736, upload-time = "2026-07-19T00:16:54.607Z" }, + { url = "https://files.pythonhosted.org/packages/62/c3/668082bcc817b9e694189b84997aeba7385b7779faa6711788679c482e35/regex-2026.7.19-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:b2ea4a3e8357be8849e833beeae757ac3c7a6b3fc055c03c808a53c91ad30d82", size = 763298, upload-time = "2026-07-19T00:16:56.289Z" }, + { url = "https://files.pythonhosted.org/packages/4b/fb/2d07ad555e7af88aa5f867fdafa47a8d945ee237c20af3ebceb46a820835/regex-2026.7.19-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:80115dd39481fd3a4b4080220799dbcacb921a844de4b827264ececacbe17c78", size = 844430, upload-time = "2026-07-19T00:16:57.933Z" }, + { url = "https://files.pythonhosted.org/packages/51/15/c82a471fe3dce56f03745635b43aa456c40dc0db089e07ef148b331507d1/regex-2026.7.19-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d6ce43a0269d68cee79a7d1ade7def53c20f8f2a047b92d7b5d5bcc73ae88327", size = 789683, upload-time = "2026-07-19T00:16:59.583Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f4/7532a2c59d56f5398902c20de60f0c9a5d1cd364e42a051b48e1b210be7b/regex-2026.7.19-cp311-cp311-win32.whl", hash = "sha256:9be2a6647740dd3cca6acb24e87f03d7632cd280dbce9bbe40c26353a215a45d", size = 266778, upload-time = "2026-07-19T00:17:01.032Z" }, + { url = "https://files.pythonhosted.org/packages/83/2b/cf1bc631db154eb95520d9d5dbc2371ff77a0f014bbf7d748fed8496aa63/regex-2026.7.19-cp311-cp311-win_amd64.whl", hash = "sha256:8d3469c91dd92ee41b7c95280edbd975ef1ba9195086686623a1c6e8935ce965", size = 277983, upload-time = "2026-07-19T00:17:02.571Z" }, + { url = "https://files.pythonhosted.org/packages/8d/bd/56ceaf170e875d5a6761bf2bfd0d040f1cacc896850d5e40cb29b11bbd06/regex-2026.7.19-cp311-cp311-win_arm64.whl", hash = "sha256:36aacfb15faaff3ced55afbf35ec72f50d4aee22082c4f7fe0573a33e2fca92e", size = 276961, upload-time = "2026-07-19T00:17:04.135Z" }, + { url = "https://files.pythonhosted.org/packages/3b/b9/d11d7e501ac8fd7d617684423ebb9561e0b998481c1e4cbc0cb212c5d74a/regex-2026.7.19-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:2cc3460cedf7579948486eab03bc9ad7089df4d7281c0f47f4afe03e8d13f02d", size = 496778, upload-time = "2026-07-19T00:17:05.677Z" }, + { url = "https://files.pythonhosted.org/packages/3f/a9/a5ab6f312f24318019170dc485d5421fe4f89e43a98640da50d95a8a7041/regex-2026.7.19-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:0e9554c8785eac5cffe6300f69a91f58ba72bc88a5f8d661235ad7c6aa5b8ccd", size = 297122, upload-time = "2026-07-19T00:17:07.59Z" }, + { url = "https://files.pythonhosted.org/packages/b3/63/4cab4d7f2d384a144d420b763d97674cb70619c878ea6fcd7640d0e62143/regex-2026.7.19-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d7da47a0f248977f08e2cb659ff3c17ddc13a4d39b3a7baa0a81bf5b415430f6", size = 292009, upload-time = "2026-07-19T00:17:09.648Z" }, + { url = "https://files.pythonhosted.org/packages/22/85/102a81b218298957d4ea7d2f084fae537a71add9d6ff93c8e67284c5f45e/regex-2026.7.19-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93db40c8de0815baab96a06e08a984bac71f989d13bab789e382158c5d426797", size = 796708, upload-time = "2026-07-19T00:17:11.542Z" }, + { url = "https://files.pythonhosted.org/packages/78/b5/dc136af5629938a037cd2b304c12240e132ec92f38be8ff9cc89af2a1f2d/regex-2026.7.19-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:66bd62c59a5427746e8c44becae1d9b99d22fb13f30f492083dfb9ad7c45cc18", size = 865651, upload-time = "2026-07-19T00:17:13.312Z" }, + { url = "https://files.pythonhosted.org/packages/e0/75/67402ae3cd9c8c988a4c805d15ee3eef015e7ca4cb112cf3e640fc1f4153/regex-2026.7.19-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1649eb39fcc9ea80c4d2f110fde2b8ab2aef3877b98f02ab9b14e961f418c511", size = 911756, upload-time = "2026-07-19T00:17:15.015Z" }, + { url = "https://files.pythonhosted.org/packages/2a/8e/096d00c7c480ef2ff4265349b14e2261d4ab787ba1f74e2e80d1c58079c3/regex-2026.7.19-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9dce8ec9695f531a1b8a6f314fd4b393adcccf2ea861db480cdf97a301d01a68", size = 801798, upload-time = "2026-07-19T00:17:17.208Z" }, + { url = "https://files.pythonhosted.org/packages/f0/41/e7ecac6edb5722417f85cc67eaf386322fbe8acf6918ec2fdc37c20dd9d0/regex-2026.7.19-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3080a7fd38ef049bd489e01c970c97dd84ff446a885b0f1f6b26d9b1ad13ce11", size = 776933, upload-time = "2026-07-19T00:17:19.347Z" }, + { url = "https://files.pythonhosted.org/packages/6f/69/03c9b3f058d66403e0ca2c938696e81d51cd4c6d47ec5265f02f96948d9a/regex-2026.7.19-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1d793a7988e04fcb1e2e135567443d82173225d657419ec09414a9b5a145b986", size = 784338, upload-time = "2026-07-19T00:17:21.057Z" }, + { url = "https://files.pythonhosted.org/packages/f6/f7/b38ab3d43f284afbb618fcd15d0e77eb786ae461ce1f6bc7494619ddc0f2/regex-2026.7.19-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e8b0abe7d870f53ca5143895fef7d1041a0c831a140d3dc2c760dd7ba25d4a8b", size = 860452, upload-time = "2026-07-19T00:17:23.119Z" }, + { url = "https://files.pythonhosted.org/packages/15/5c/ff60ef0571121714f3cf9920bc183071e384a10b556d042e0fdb06cc07a5/regex-2026.7.19-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:4e5413bd5f13d3a4e3539ca98f70f75e7fca92518dd7f117f030ebedd10b60cb", size = 765958, upload-time = "2026-07-19T00:17:24.81Z" }, + { url = "https://files.pythonhosted.org/packages/aa/0f/bd34021162c0ab47f9a315bd56cd5642e920c8e5668a75ef6c6a6fca590d/regex-2026.7.19-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:73b133a9e6fb512858e7f065e96f1180aa46646bc74a83aea62f1d314f3dd035", size = 851765, upload-time = "2026-07-19T00:17:26.993Z" }, + { url = "https://files.pythonhosted.org/packages/2a/20/a2ca43edade0595cccfdc98636739f536d9e26898e7dbddc2b9e98898953/regex-2026.7.19-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dbe6493fbd27321b1d1f2dd4f5c7e5bd4d8b1d7cab7f32fd67db3d0b2ed8248a", size = 789714, upload-time = "2026-07-19T00:17:28.699Z" }, + { url = "https://files.pythonhosted.org/packages/5d/47/e02db4015d424fc83c00ea0ac8c5e5ec14397943de9abf909d5ce3a25931/regex-2026.7.19-cp312-cp312-win32.whl", hash = "sha256:ddd67571c10869f65a5d7dde536d1e066e306cc90de57d7de4d5f34802428bb5", size = 267157, upload-time = "2026-07-19T00:17:31.051Z" }, + { url = "https://files.pythonhosted.org/packages/08/8e/c780c131f79b42ed22d1bd7da4096c2c35f813e835acd02ef0f018bd892c/regex-2026.7.19-cp312-cp312-win_amd64.whl", hash = "sha256:e30d40268a28d54ce0437031750497004c22602b8e3ab891f759b795a003b312", size = 277777, upload-time = "2026-07-19T00:17:32.848Z" }, + { url = "https://files.pythonhosted.org/packages/3e/4c/e4d7e086449bdf379d89774bf1f89dc4a41943f3c5a6125a03905b34b5fb/regex-2026.7.19-cp312-cp312-win_arm64.whl", hash = "sha256:de9208bb427130c82a5dbfd104f92c8876fc9559278c880b3002755bbbe9c83d", size = 277136, upload-time = "2026-07-19T00:17:34.803Z" }, + { url = "https://files.pythonhosted.org/packages/5d/3d/84165e4299ff76f3a40fe1f2abf939e976f693383a08d2beea6af62bd2c1/regex-2026.7.19-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f035d9dc1d25eff9d361456572231c7d27b5ccd473ca7dc0adfce732bd006d40", size = 496552, upload-time = "2026-07-19T00:17:36.808Z" }, + { url = "https://files.pythonhosted.org/packages/02/a2/a65293e6e4cf28eb7ee1be5335a5386c40d6742e9f47fafc8fec785e16c7/regex-2026.7.19-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c42572142ed0b9d5d261ba727157c426510da78e20828b66bbb855098b8a4e38", size = 296983, upload-time = "2026-07-19T00:17:38.816Z" }, + { url = "https://files.pythonhosted.org/packages/95/47/2d0564e93d87bc48618360ddca232a2ca612bbdf53ce8465d45ca5ce14ee/regex-2026.7.19-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40b34dd88658e4fedd2fddbf0275ac970d00614b731357f425722a3ed1983d11", size = 291832, upload-time = "2026-07-19T00:17:40.726Z" }, + { url = "https://files.pythonhosted.org/packages/07/cd/42dfbabff3dfc9603c501c0e2e2c5adbb09d127b267bf5348de0af338c15/regex-2026.7.19-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c41c63992bf1874cebb6e7f56fd7d3c007924659a604ae3d90e427d40d4fd13", size = 796775, upload-time = "2026-07-19T00:17:42.382Z" }, + { url = "https://files.pythonhosted.org/packages/df/5d/f6a4839f2b934e3eed5973fd07f5929ee97d4c98939fb275ea23c274ee16/regex-2026.7.19-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d3372064506b94dd2c67c845f2db8062e9e9ba84d04e33cb96d7d33c11fe1ae", size = 865687, upload-time = "2026-07-19T00:17:44.185Z" }, + { url = "https://files.pythonhosted.org/packages/14/b0/b47d6c36049bc59806a50bd4c86ced70bbe058d787f80281b1d7a9b0e024/regex-2026.7.19-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fce7760bf283405b2c7999cab3da4e72f7deca6396013115e3f7a955db9760da", size = 911962, upload-time = "2026-07-19T00:17:46.442Z" }, + { url = "https://files.pythonhosted.org/packages/2a/be/ff61f28f9273658cfe23acbbac5217221f6519960ed401e61dfdab12bc35/regex-2026.7.19-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0d702548d89d572b2929879bc883bb7a4c4709efafe4512cadee56c55c9bd15", size = 801817, upload-time = "2026-07-19T00:17:48.25Z" }, + { url = "https://files.pythonhosted.org/packages/c3/bb/8b4f7f26b333f9f79e1b453613c39bb4776f51d38ae66dd0ba31d6b354ca/regex-2026.7.19-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d446c6ac40bb6e05025ccee55b84d80fe9bf8e93010ffc4bb9484f13d498835f", size = 776908, upload-time = "2026-07-19T00:17:50.183Z" }, + { url = "https://files.pythonhosted.org/packages/09/13/610110fc5921d380516d03c26b652555f08aa0d23ea78a771231873c3638/regex-2026.7.19-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4c3501bfa814ab07b5580741f9bf78dfdfe146a04057f82df9e2402d2a975939", size = 784426, upload-time = "2026-07-19T00:17:52.454Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f5/1ef9e2a83a5947c57ebff0b377cb5727c3d5ec1992317a320d035cd0dbb6/regex-2026.7.19-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c4585c3e64b4f9e583b4d2683f18f5d5d872b3d71dcf24594b74ecc23602fa96", size = 860600, upload-time = "2026-07-19T00:17:54.229Z" }, + { url = "https://files.pythonhosted.org/packages/a0/02/073af33a3ec149241d11c80acea91e722aa0adbf05addd50f251c4fe89c3/regex-2026.7.19-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:571fde9741eb0ccde23dd4e0c1d50fbae910e901fa7e629faf39b2dda740d220", size = 765950, upload-time = "2026-07-19T00:17:56.041Z" }, + { url = "https://files.pythonhosted.org/packages/81/a9/d1e9f819dc394a568ef370cd56cf25394e957a2235f8370f23b576e5a475/regex-2026.7.19-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:15b364b9b98d6d2fe1a85034c23a3180ff913f46caddc3895f6fd65186255ccc", size = 851794, upload-time = "2026-07-19T00:17:57.897Z" }, + { url = "https://files.pythonhosted.org/packages/03/3a/8ae83eda7579feacdf984e71fb9e70635fb6f832eeddca58427ec4fca926/regex-2026.7.19-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffd8893ccc1c2fce6e0d6ca402d716fe1b29db70c7132609a05955e31b2aa8f2", size = 789845, upload-time = "2026-07-19T00:17:59.97Z" }, + { url = "https://files.pythonhosted.org/packages/4b/23/c195cbfe5a75fdec64d8f6554fd15237b837919d2c61bdc141d7c807b08b/regex-2026.7.19-cp313-cp313-win32.whl", hash = "sha256:f0fa4fa9c3632d708742baf2282f2055c11d888a790362670a403cbf48a2c404", size = 267135, upload-time = "2026-07-19T00:18:01.958Z" }, + { url = "https://files.pythonhosted.org/packages/b2/80/a11de8404b7272b70acb45c1c05987cce60b45d5693da2e176f0e390d564/regex-2026.7.19-cp313-cp313-win_amd64.whl", hash = "sha256:d51ffd3427640fa2da6ade574ceba932f210ad095f65fcc450a2b0a0d454868e", size = 277747, upload-time = "2026-07-19T00:18:04.121Z" }, + { url = "https://files.pythonhosted.org/packages/d1/29/0f5c8eff1b4f1f3d83276d365fccecf666afcc7d947420943bf394d07adb/regex-2026.7.19-cp313-cp313-win_arm64.whl", hash = "sha256:c670fe7be5b6020b76bc6e8d2196074657e1327595bca93a389e1a76ab130ad8", size = 277129, upload-time = "2026-07-19T00:18:05.821Z" }, + { url = "https://files.pythonhosted.org/packages/dc/4c/44b74742052cedda40f9ae469532a037112f7311a36669a891fba8984bb0/regex-2026.7.19-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db47b561c9afd884baa1f96f797c9ca369872c4b65912bc691cfa99e68340af2", size = 501134, upload-time = "2026-07-19T00:18:07.567Z" }, + { url = "https://files.pythonhosted.org/packages/f0/45/bbd038b5e39ee5613a5a689290145b40058cc152c41de9cc23639d2b9734/regex-2026.7.19-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:65dcd28d3eba2ab7c2fd906485cc301392b47cc2234790d27d4e4814e02cdfda", size = 299418, upload-time = "2026-07-19T00:18:09.38Z" }, + { url = "https://files.pythonhosted.org/packages/65/38/c5bde94b4cedfd5850d64c3f08222d8e1600e84f6ee71d9b44b4b8163f74/regex-2026.7.19-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:f2e7f8e2ab6c2922be02c7ec45185aa5bd771e2e57b95455ee343a44d8130dff", size = 294486, upload-time = "2026-07-19T00:18:11.188Z" }, + { url = "https://files.pythonhosted.org/packages/d7/6a/2f5e107cb26c960b781967178899daf2787a7ab151844ed3c01d6fc95474/regex-2026.7.19-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe31f28c94402043161876a258a9c6f757cb485905c7614ce8d6cd40e6b7bdc1", size = 811643, upload-time = "2026-07-19T00:18:12.975Z" }, + { url = "https://files.pythonhosted.org/packages/37/d4/a2f963406d7d73a62eed84ba05a258afb6cad1b21aa4517443ce40506b78/regex-2026.7.19-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8f6fa298bb4f7f58a33334406218ba74716e68feddf5e4e54cd5d8082705abf", size = 871081, upload-time = "2026-07-19T00:18:14.733Z" }, + { url = "https://files.pythonhosted.org/packages/45/a3/44be546340bedb15f13063f5e7fe16793ea4d9ea2e805d09bd174ac27724/regex-2026.7.19-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cc1b2440423a851fad781309dd87843868f4f66a6bcd1ddb9225cf4ec2c84732", size = 917372, upload-time = "2026-07-19T00:18:16.724Z" }, + { url = "https://files.pythonhosted.org/packages/f8/f6/e0870b0fd2a40dba0074e4b76e514b21313d37946c9248453e34ec43923e/regex-2026.7.19-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8ac59a0900474a52b7c04af8196affc22bd9842acb0950df12f7b813e983609a", size = 816089, upload-time = "2026-07-19T00:18:18.617Z" }, + { url = "https://files.pythonhosted.org/packages/ae/27/957e8e22690ad6634572b39b71f130a6105f4d0718bb16849eac00fff147/regex-2026.7.19-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4896db1f4ce0576765b8272aa922df324e0f5b9bb2c3d03044ff32a7234a9aba", size = 785206, upload-time = "2026-07-19T00:18:20.464Z" }, + { url = "https://files.pythonhosted.org/packages/76/a4/186e410941e731037c01166069ab86da9f65e8f8110c18009ccf4bd623ee/regex-2026.7.19-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4e6883a021db30511d9fb8cfb0f222ce1f2c369f7d4d8b0448f449a93ba0bdfc", size = 800431, upload-time = "2026-07-19T00:18:22.716Z" }, + { url = "https://files.pythonhosted.org/packages/73/9f/e4e10e023d291d64a33e246610b724493bf1ce98e0e59c9b7c837e5acfb7/regex-2026.7.19-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:09523a592938aa9f587fb74467c63ff0cf88fc3df14c82ab0f0517dcf76aaa62", size = 864906, upload-time = "2026-07-19T00:18:24.772Z" }, + { url = "https://files.pythonhosted.org/packages/24/57/ccb20b6be5f1f52a053d1ba2a8f7a077edb9d918248b8490d7506c6832b3/regex-2026.7.19-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:1ebac3474b8589fce2f9b225b650afd61448f7c73a5d0255a10cc6366471aed1", size = 773559, upload-time = "2026-07-19T00:18:27.008Z" }, + { url = "https://files.pythonhosted.org/packages/a3/82/f3b263cf8fad927dc102891da8502e718b7ff9d19af7a2a07c03865d7188/regex-2026.7.19-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:4a0530bb1b8c1c985e7e2122e2b4d3aedd8a3c21c6bfddae6767c4405668b56e", size = 857739, upload-time = "2026-07-19T00:18:29.107Z" }, + { url = "https://files.pythonhosted.org/packages/47/2e/1687bd1b6c2aed5e672ccf845fc11557821fe7366d921b50889ea5ce57bf/regex-2026.7.19-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2ef7eeb108c47ce7bcc9513e51bcb1bf57e8f483d52fce68a8642e3527141ae0", size = 804522, upload-time = "2026-07-19T00:18:31.362Z" }, + { url = "https://files.pythonhosted.org/packages/76/7c/cc4e7655181b2d9235b704f2c5e19d8eff002bbc437bae59baee0e381aca/regex-2026.7.19-cp313-cp313t-win32.whl", hash = "sha256:64b6ca7391a1395c2638dd5c7456d67bea44fc6c5e8e92c5dc8aa6a8f23292b4", size = 269141, upload-time = "2026-07-19T00:18:33.479Z" }, + { url = "https://files.pythonhosted.org/packages/bb/14/961b4c7b05a2391c32dbc85e27773076671ef8f97f36cec70fe414734c02/regex-2026.7.19-cp313-cp313t-win_amd64.whl", hash = "sha256:f04b9f56b0e0614c0126be12c2c2d9f8850c1e57af302bd0a63bed379d4af974", size = 280036, upload-time = "2026-07-19T00:18:35.419Z" }, + { url = "https://files.pythonhosted.org/packages/ce/67/795644550d788ddbb6dc458c95895f8009978ea6d6ea76b005eb3f45e8c9/regex-2026.7.19-cp313-cp313t-win_arm64.whl", hash = "sha256:fcee38cd8e5089d6d4f048ba1233b3ad76e5954f545382180889112ff5cb712d", size = 279394, upload-time = "2026-07-19T00:18:37.454Z" }, + { url = "https://files.pythonhosted.org/packages/d2/25/0c4c452f8ef3efe456745b2f33195f5904b573fb4c2ff3f0cb9ec188461e/regex-2026.7.19-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:a81758ed242b861b72e778ba34d41366441a2e10b16b472784c88da2dea7e2dd", size = 496750, upload-time = "2026-07-19T00:18:39.633Z" }, + { url = "https://files.pythonhosted.org/packages/24/9e/b70ca6c1704f6c7cd32a9e143c86cc5968d10981eca284bad670c245ea7d/regex-2026.7.19-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4aa5435cdb3eb6f55fe98a171b05e3fbcd95fadaa4aa32acf62afd9b0cfdbcac", size = 297093, upload-time = "2026-07-19T00:18:41.583Z" }, + { url = "https://files.pythonhosted.org/packages/87/74/0b692da2520d51fbff19c88b83d97e4c702909dd02386c585998b7e2dbed/regex-2026.7.19-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:60be8693a1dadc210bbcbc0db3e26da5f7d01d1d5a3da594e99b4fa42df404f5", size = 292043, upload-time = "2026-07-19T00:18:43.347Z" }, + { url = "https://files.pythonhosted.org/packages/e3/a7/1d478e614016045a33feae57446215f9fd65b665a5ceb2f891fb3183bc52/regex-2026.7.19-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19662dbedbe783d323196312d38f5ba53cf56296378252171985da6899887d3", size = 797214, upload-time = "2026-07-19T00:18:45.362Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ae/11b9c9411d92c30e3d2db32df5a31133e4a99a8fc397a604fd08f6c4bffb/regex-2026.7.19-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d15df07081d91b76ff20d43f94592ee110330152d617b730fdbe5ef9fb680053", size = 866433, upload-time = "2026-07-19T00:18:47.315Z" }, + { url = "https://files.pythonhosted.org/packages/b1/62/2b2efc4992f91d6d204b24c647c9f9412e85379d92b7c0ab9fdae622327e/regex-2026.7.19-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:56ad4d9f77df871a99e25c37091052a02528ec0eb059de928ee33956b854b45b", size = 911360, upload-time = "2026-07-19T00:18:49.588Z" }, + { url = "https://files.pythonhosted.org/packages/14/71/986ceea9aa3da548bf1357cad89b63915ec6d21ec957c8113b29ece567df/regex-2026.7.19-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7322ec6cc9fba9d49ab888bb82d67ac5625627aa168f0165139b17018df3fb8a", size = 801275, upload-time = "2026-07-19T00:18:51.767Z" }, + { url = "https://files.pythonhosted.org/packages/15/be/ce9d9534b2cda96eab32c548261224b9b4e220a4126f098f60f42ae7b4cd/regex-2026.7.19-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9c7472192ebfad53a6be7c4a8bfb2d64b81c0e93a1fc8c57e1dd0b638297b5d1", size = 777131, upload-time = "2026-07-19T00:18:54.053Z" }, + { url = "https://files.pythonhosted.org/packages/61/2b/58b5c710f2c3929515a25f3a1ca0dad0dcd4518d4fff3cf23bc7adb8dcd2/regex-2026.7.19-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c10b82c2634df08dfb13b1f04e38fe310d086ee092f4f69c0c8da234251e556e", size = 785020, upload-time = "2026-07-19T00:18:56.579Z" }, + { url = "https://files.pythonhosted.org/packages/84/03/5fe091935b74f15fe0f97998c215cae418d1c0413f6258c7d4d2e83aa37f/regex-2026.7.19-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:17ed5692f6acc4183e98331101a5f9e4f64d72fe58b753da4d444a2c77d05b12", size = 861263, upload-time = "2026-07-19T00:18:58.64Z" }, + { url = "https://files.pythonhosted.org/packages/d8/fa/d60bf82e10841eef62a9e32aac401468f05fddfbcb2942e342b1ba3d2433/regex-2026.7.19-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:22a992de9a0d91bda927bf02b94351d737a0302905432c88a53de7c4b9ce62e2", size = 766199, upload-time = "2026-07-19T00:19:00.705Z" }, + { url = "https://files.pythonhosted.org/packages/bf/5d/11e64d151b0662b81d6bf644c74dc118d461df85bdf2577fadbbf751788a/regex-2026.7.19-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:618a0aed532be87294c4477b0481f3aa0f1520f4014a4374dd4cf789b4cd2c97", size = 851317, upload-time = "2026-07-19T00:19:03.015Z" }, + { url = "https://files.pythonhosted.org/packages/7c/34/532efb87488d90807bae6a443d357ee5e2728a478c597619c8aaa17cc0bd/regex-2026.7.19-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ce9e679f776649746729b6c86382da519ef649c8e34cc41df0d2e5e0f6c36d4", size = 789557, upload-time = "2026-07-19T00:19:05.338Z" }, + { url = "https://files.pythonhosted.org/packages/d6/90/3a8d5ca977171ec3ae21a71207d2228b2663bde14d7f7ef0e6363ecf9290/regex-2026.7.19-cp314-cp314-win32.whl", hash = "sha256:73f272fba87b8ccfe70a137d02a54af386f6d27aa509fbffdd978f5947aae1aa", size = 272531, upload-time = "2026-07-19T00:19:07.487Z" }, + { url = "https://files.pythonhosted.org/packages/96/e1/8862885e70409de70e8c005f57fb2e7be8d9ef0317250d60f4c9660a300d/regex-2026.7.19-cp314-cp314-win_amd64.whl", hash = "sha256:d721e53758b2cca74990185eb0671dd466d7a388a1a45d0c6f4c13cef41a68ac", size = 280831, upload-time = "2026-07-19T00:19:09.46Z" }, + { url = "https://files.pythonhosted.org/packages/08/82/2693e53e29f9104d9de95d37ce4dd826bd32d5f9c0085d3aa6ac042675c4/regex-2026.7.19-cp314-cp314-win_arm64.whl", hash = "sha256:65fa6cb38ed5e9c3637e68e544f598b39c3b86b808ed0627a67b68320384b459", size = 281099, upload-time = "2026-07-19T00:19:11.398Z" }, + { url = "https://files.pythonhosted.org/packages/92/b7/9a01aa16461a18cde9d7b9c3ab21e501db2ce33725f53014342b91df2b0a/regex-2026.7.19-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:5a2721c8720e2cb3c209925dfb9200199b4b07361c9e01d321719404b21458b3", size = 501121, upload-time = "2026-07-19T00:19:13.425Z" }, + { url = "https://files.pythonhosted.org/packages/f3/5e/bbaeca815dc9191c424c94a4fdc5c87c75748a64a6271821212ebdd4e1a3/regex-2026.7.19-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:199535629f25caf89698039af3d1ad5fcae7f933e2112c73f1cdf49165c99518", size = 299415, upload-time = "2026-07-19T00:19:15.43Z" }, + { url = "https://files.pythonhosted.org/packages/cd/d6/0dd1a321afaab95eb7ff44aa0f637301786f1dc71c6b797b9ed236ed8890/regex-2026.7.19-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9b60d7814174f059e5de4ab98271cc5ba9259cfea55273a81544dceea32dc8d9", size = 294483, upload-time = "2026-07-19T00:19:17.879Z" }, + { url = "https://files.pythonhosted.org/packages/92/5f/40bacf91d0904f812e13bbbab3864604c463eced8afdc54aeaa50492ea95/regex-2026.7.19-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dbece16025afda5e3031af0c4059207e61dcf73ef13af844964f57f387d1c435", size = 811833, upload-time = "2026-07-19T00:19:20.102Z" }, + { url = "https://files.pythonhosted.org/packages/94/7c/4902744261f775aeede8b5627314b38482da29cf49a57b66a6fb753246c5/regex-2026.7.19-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d24ecb4f5e009ea0bd275ee37ad9953b32005e2e5e60f8bbae16da0dbbf0d3a0", size = 871270, upload-time = "2026-07-19T00:19:22.365Z" }, + { url = "https://files.pythonhosted.org/packages/16/70/6980c9be6bf21c0a60ed3e0aea39cf419ecf3b08d1d9947bc56e196ef186/regex-2026.7.19-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8cae6fd77a5b72dae505084b1a2ee0360139faf72fedbab667cd7cc65aae7a6a", size = 917534, upload-time = "2026-07-19T00:19:24.529Z" }, + { url = "https://files.pythonhosted.org/packages/52/92/8b2bd872782ce8c42691e39acb38eb8efe014e5ddb78ad7d943d6f197ce9/regex-2026.7.19-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9724e6cb5e478cd7d8cabf027826178739cb18cf0e117d0e32814d479fa02276", size = 816135, upload-time = "2026-07-19T00:19:26.919Z" }, + { url = "https://files.pythonhosted.org/packages/de/2d/33a602f657bdc4041f17d79f92ab18261d255d91a06117a6e29df023e5e2/regex-2026.7.19-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:572fc57b0009c735ee56c175ea021b637a15551a312f56734277f923d6fd0f6c", size = 785492, upload-time = "2026-07-19T00:19:29.192Z" }, + { url = "https://files.pythonhosted.org/packages/9e/36/0987cf4cb271680064a70d24a475873775a151d0b7058698a006cb0cae4a/regex-2026.7.19-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:20568e182eb82d39a6bf7cff3fd58566f14c75c6f74b2c8c96537eecf9010e3a", size = 800658, upload-time = "2026-07-19T00:19:31.392Z" }, + { url = "https://files.pythonhosted.org/packages/a8/24/c14f31c135e1ba55fa4f9a58ca98d0842512bf6188230763c31c8f449e3b/regex-2026.7.19-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:1d58561843f0ff7dc78b4c28b5e2dc388f3eff94ebc8a232a3adba961fc00009", size = 865073, upload-time = "2026-07-19T00:19:33.485Z" }, + { url = "https://files.pythonhosted.org/packages/14/85/181a12211f22469f24d2de1ebddfe397d2396e2c29013b9a58134a91069a/regex-2026.7.19-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:61bb1bd45520aacd56dd80943bd34991fb5350afdd1f36f2282230fd5154a218", size = 773684, upload-time = "2026-07-19T00:19:35.599Z" }, + { url = "https://files.pythonhosted.org/packages/23/58/bd1a0c1a62251366f8d21f41b1ea3c76994962071b8b6ea42f72d505c0f0/regex-2026.7.19-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:cd3584591ea4429026cdb931b054342c2bcf189b44ff367f8d5c15bc092a2966", size = 857769, upload-time = "2026-07-19T00:19:37.738Z" }, + { url = "https://files.pythonhosted.org/packages/e4/4f/f7e2dad6756b2fe1fe75dd90a628c3b45f249d39f948dd90cd2476325417/regex-2026.7.19-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5cc26a66e212fa5d6c6170c3a40d99d888db3020c6fdab1523250d4341382e44", size = 804546, upload-time = "2026-07-19T00:19:40.229Z" }, + { url = "https://files.pythonhosted.org/packages/2b/d7/01d31d5bdb09bc026fab77f59a371fdf8f9b292e4810546c56182ca70498/regex-2026.7.19-cp314-cp314t-win32.whl", hash = "sha256:2c4e61e2e1be56f63ec3cc618aa9e0de81ef6f43d177205451840022e24f5b78", size = 274526, upload-time = "2026-07-19T00:19:42.398Z" }, + { url = "https://files.pythonhosted.org/packages/52/0e/cea4ce73bc0a8247a0748228ae6669984c7e1f8134b6fa66e59c0572e0ea/regex-2026.7.19-cp314-cp314t-win_amd64.whl", hash = "sha256:c639ea314df70a7b2811e8020448c75af8c9445f5a60f8a4ced81c306a9380c2", size = 283763, upload-time = "2026-07-19T00:19:44.644Z" }, + { url = "https://files.pythonhosted.org/packages/6f/b6/26e41975febae63b7a6e3e02f32cff6cff2e4f10d19c929082f56aebf7c6/regex-2026.7.19-cp314-cp314t-win_arm64.whl", hash = "sha256:9a15e785f244f3e07847b984ce8773fc3da10a9f3c131cc49a4c5b4d672b4547", size = 283451, upload-time = "2026-07-19T00:19:46.639Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + +[[package]] +name = "ruff" +version = "0.16.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/70/25/7113f6d5498888c5fb7db34081cba7d5971c4cb1bfb26819966eee68f003/ruff-0.16.1.tar.gz", hash = "sha256:fedad7c801dabd3fb9741d76aca39246e6ddd9ca446a015875207bf19f1e6bc7", size = 4877500, upload-time = "2026-07-30T19:37:01.379Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/bd/694da69368e0973de65df2ddc73ab18d43c469d5963d9b150911de6bc513/ruff-0.16.1-py3-none-linux_armv6l.whl", hash = "sha256:58edb313b88f0c5460a26adf5f39a37a3be789494a15e3e411e35fa78b89f9a0", size = 10839126, upload-time = "2026-07-30T19:36:13.697Z" }, + { url = "https://files.pythonhosted.org/packages/3f/f0/b626e5d5bd0dd9576263658ef12885e2288afd1029a48e26ffed65ec1ac1/ruff-0.16.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fde5a99e2f97479af66edd6622c6d5a2a7592c77cf4153d9e4428f5eeb55b60c", size = 11070253, upload-time = "2026-07-30T19:36:17.14Z" }, + { url = "https://files.pythonhosted.org/packages/83/63/f40acfb6b35b88623e71684942b552c3edd96035f5d98f313815f7b277de/ruff-0.16.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e0d4c20532fca4f7fa609369161d968dd28f65d83dabbd61d8e9c7edbf7001f6", size = 10561425, upload-time = "2026-07-30T19:36:20.04Z" }, + { url = "https://files.pythonhosted.org/packages/aa/dd/14ec0e9c2b4d315547dd38765004b4863e354e1b52cb308272215d9f6f6d/ruff-0.16.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30affbcedf59ad5703d9c91f82266e02b47739f797e1a7b6e158e5526a6dae38", size = 10948879, upload-time = "2026-07-30T19:36:22.476Z" }, + { url = "https://files.pythonhosted.org/packages/33/e9/9d870cbae575030fdef595f04b4b97573c525b5497cce4f4498cf2f85446/ruff-0.16.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:24e9c631573cbca9d20f1283f8f479b2afa4a8503504822bd71a293889f16743", size = 10643691, upload-time = "2026-07-30T19:36:24.914Z" }, + { url = "https://files.pythonhosted.org/packages/c4/09/12743d544e2173f53ecd27217c65f90d2bc0f8424a66a60339e56bbc0457/ruff-0.16.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b41bdd48fb420987a9b5212e4957c26ad4abce401fa9ea9d4d85843727945f4f", size = 11435354, upload-time = "2026-07-30T19:36:28.447Z" }, + { url = "https://files.pythonhosted.org/packages/7f/89/a1652b2daee52083c9554a6333b678a8b01d0400f976827bb87857f9449a/ruff-0.16.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b0d1e1393b7648079e13669de1c1f4fde06d4583e84d8fd5c1551e0a77a2aa75", size = 12259033, upload-time = "2026-07-30T19:36:31.326Z" }, + { url = "https://files.pythonhosted.org/packages/16/96/ecdcb8c54ee7b123b487f807eb014e6e019155a0b81dfb669acd52f28ce3/ruff-0.16.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:07bf434b1c95f4e093be4532068ef4fcf00924eb2ade8796075980902d6fd54a", size = 11667981, upload-time = "2026-07-30T19:36:34.394Z" }, + { url = "https://files.pythonhosted.org/packages/cd/90/c52e12e0d862e9572f2a33aa227409143520abe53111e9a6babbac7b4af8/ruff-0.16.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39897739f112253ee4fdd2e8aa9a4f9ded99fb2be367d5f31dfa4ded6025584c", size = 11468183, upload-time = "2026-07-30T19:36:37.339Z" }, + { url = "https://files.pythonhosted.org/packages/2c/6b/4ffb7ad1d83eb16cf8cbb3c8815d3f11c88460fd162d4b372a2059be1c2a/ruff-0.16.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:82ae3c0c0d74daf17b968a10b7b3bb3ef297ab7de0c1f749646b25e690ccb150", size = 11470071, upload-time = "2026-07-30T19:36:39.91Z" }, + { url = "https://files.pythonhosted.org/packages/9c/72/32ae7db4c0b5e32ab611787caa19d1546800676d79f7483b7100a3561bf4/ruff-0.16.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4d5f2ed10f8242d83fc08d521301089364e3375375705356f20c0e31606ef3ef", size = 10919503, upload-time = "2026-07-30T19:36:42.65Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ca/3d901ba6ad6fc38da39c3448fc6c59ac945679293a17c3ceb6d6c1cba13e/ruff-0.16.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a4665b309891f83f3e3c25447935f1213e9abbd4b5640af7a1f2def9f8d413c1", size = 10649861, upload-time = "2026-07-30T19:36:45.18Z" }, + { url = "https://files.pythonhosted.org/packages/92/79/894ef1ced26552d5f8c9cf6d85b0687840e1128c55aeab7b9c2d54a0d880/ruff-0.16.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:26e9ca5c9bc3971f20d3cf18a957f52ffd6a5f6564ff15c4912a144dcac22494", size = 11148137, upload-time = "2026-07-30T19:36:47.936Z" }, + { url = "https://files.pythonhosted.org/packages/2d/69/3609a09fa1cb46cc28b762363e440a354204e5dff01bd0c8d7437874d6b9/ruff-0.16.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:67e1e1e3fa4f0c82f0e36d4cd61e661f6e7a6196cb1aa92fe0828fa7b8f257cd", size = 11559211, upload-time = "2026-07-30T19:36:50.448Z" }, + { url = "https://files.pythonhosted.org/packages/fc/8a/fb22af2fd78a736e241fabf67e30ce1799a64244026377a49e133af90762/ruff-0.16.1-py3-none-win32.whl", hash = "sha256:d31765e131295b8445caf301e3e8a85b34d1b9b211b4109b7ba457888b051806", size = 10838258, upload-time = "2026-07-30T19:36:53.298Z" }, + { url = "https://files.pythonhosted.org/packages/d4/35/e57fd9fb5d423961df087a00b12d42c0a830288dc2f3b45ecca299158b4f/ruff-0.16.1-py3-none-win_amd64.whl", hash = "sha256:09b05e8b90c2cb06ad63464350e7a45e8e44a2dfe52072ebfba6666ca8d3f596", size = 11961111, upload-time = "2026-07-30T19:36:56.107Z" }, + { url = "https://files.pythonhosted.org/packages/cb/46/240ea004bf6dc4feb40e9832f2205a476a47dd5b8a3f8211a5fc5f95e20e/ruff-0.16.1-py3-none-win_arm64.whl", hash = "sha256:dbaadaac38c70239f056d306b7476f246b0bf000fa6b3876402acbf5b227eaf8", size = 11309414, upload-time = "2026-07-30T19:36:58.79Z" }, +] + +[[package]] +name = "safetensors" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/06/f955dbbb1859e3bd23c8ac6141af5106e7ad5fedec4a3a6e3d60f94b7001/safetensors-0.8.0.tar.gz", hash = "sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d", size = 325846, upload-time = "2026-06-09T07:52:25.563Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/a0/f718cda65b05407d228f97602cf60dca269c979867aa5beb25410de26cd3/safetensors-0.8.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0", size = 473568, upload-time = "2026-06-09T07:52:18.829Z" }, + { url = "https://files.pythonhosted.org/packages/f5/b1/fa7c600e7dceae12e9606c7578cbc9ff1e1ed55844883ee5c92205e86226/safetensors-0.8.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25", size = 484562, upload-time = "2026-06-09T07:52:17.518Z" }, + { url = "https://files.pythonhosted.org/packages/09/7d/65a7de0af421317bb36a067241e4235fff194eed60b961ed6d3f59a3fc60/safetensors-0.8.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235", size = 502844, upload-time = "2026-06-09T07:52:07.624Z" }, + { url = "https://files.pythonhosted.org/packages/91/4f/3175c9d75634e0e0dda0082794193521035edd7c70a6f212bf33ca06ddf4/safetensors-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0", size = 511823, upload-time = "2026-06-09T07:52:09.565Z" }, + { url = "https://files.pythonhosted.org/packages/20/87/846c289e7aa2299eff406335717cf43ce8777194ece8aad75772e0411615/safetensors-0.8.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98", size = 633461, upload-time = "2026-06-09T07:52:11.128Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/8d64d9df2c45d5ded401df889d0ad90882804ca172d79ec4f0df8f727fe0/safetensors-0.8.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358", size = 545148, upload-time = "2026-06-09T07:52:13.603Z" }, + { url = "https://files.pythonhosted.org/packages/28/50/f203ff3a3ddfe19308efc83c5a3a29ed02bf786732ec35e68bf9162f3365/safetensors-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774", size = 516040, upload-time = "2026-06-09T07:52:16.29Z" }, + { url = "https://files.pythonhosted.org/packages/46/fb/cdaed17ceb2948784fd9c36b6fd3e951b608547cea81a48e8ee6f8cfdfcb/safetensors-0.8.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78", size = 513832, upload-time = "2026-06-09T07:52:12.37Z" }, + { url = "https://files.pythonhosted.org/packages/0d/49/1e15de264dcc3b77943d2d0c56a95809956883b1c2d6d585c792523f180b/safetensors-0.8.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca", size = 559930, upload-time = "2026-06-09T07:52:14.743Z" }, + { url = "https://files.pythonhosted.org/packages/2a/43/bf38443278eab4b1be1fce2931e2b012ad9cb7df52ada751d0aab8f7659a/safetensors-0.8.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4", size = 678670, upload-time = "2026-06-09T07:52:20.032Z" }, + { url = "https://files.pythonhosted.org/packages/72/e3/68cd3fa5b48488e84add63e04cb12f3bc28ae4638c06d4508c6e88823d0e/safetensors-0.8.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc", size = 786679, upload-time = "2026-06-09T07:52:21.322Z" }, + { url = "https://files.pythonhosted.org/packages/29/4b/1c19c509d56e01f4fbb3d0a2e597450f6cc04d1d56cf52defb0a62dfd715/safetensors-0.8.0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d", size = 765683, upload-time = "2026-06-09T07:52:22.594Z" }, + { url = "https://files.pythonhosted.org/packages/27/43/41c1621732edd934d868a00d1b891584c892a7b62a9aab82ea5a0a5623ee/safetensors-0.8.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846", size = 722361, upload-time = "2026-06-09T07:52:23.924Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3f/73ccf82579412b4a71c4ca673f10b5f1f888d7cf5af7fe24f27d30307be4/safetensors-0.8.0-cp310-abi3-win32.whl", hash = "sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d", size = 342401, upload-time = "2026-06-09T07:52:28.895Z" }, + { url = "https://files.pythonhosted.org/packages/1b/6d/3fba214c1e5e0f69991677ec3bc17023f0421776975e1de0c682dca475e2/safetensors-0.8.0-cp310-abi3-win_amd64.whl", hash = "sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f", size = 355540, upload-time = "2026-06-09T07:52:27.832Z" }, + { url = "https://files.pythonhosted.org/packages/8d/fc/7eedc3510d97878876e32774eebbeb61c43f148a96e915c84229a3e967aa/safetensors-0.8.0-cp310-abi3-win_arm64.whl", hash = "sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452", size = 340500, upload-time = "2026-06-09T07:52:26.745Z" }, +] + +[[package]] +name = "setuptools" +version = "83.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, +] + +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "sympy" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mpmath" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, +] + +[[package]] +name = "tokenizers" +version = "0.22.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/6f/f80cfef4a312e1fb34baf7d85c72d4411afde10978d4657f8cdd811d3ccc/tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917", size = 372115, upload-time = "2026-01-05T10:45:15.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/97/5dbfabf04c7e348e655e907ed27913e03db0923abb5dfdd120d7b25630e1/tokenizers-0.22.2-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:544dd704ae7238755d790de45ba8da072e9af3eea688f698b137915ae959281c", size = 3100275, upload-time = "2026-01-05T10:41:02.158Z" }, + { url = "https://files.pythonhosted.org/packages/2e/47/174dca0502ef88b28f1c9e06b73ce33500eedfac7a7692108aec220464e7/tokenizers-0.22.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:1e418a55456beedca4621dbab65a318981467a2b188e982a23e117f115ce5001", size = 2981472, upload-time = "2026-01-05T10:41:00.276Z" }, + { url = "https://files.pythonhosted.org/packages/d6/84/7990e799f1309a8b87af6b948f31edaa12a3ed22d11b352eaf4f4b2e5753/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7", size = 3290736, upload-time = "2026-01-05T10:40:32.165Z" }, + { url = "https://files.pythonhosted.org/packages/78/59/09d0d9ba94dcd5f4f1368d4858d24546b4bdc0231c2354aa31d6199f0399/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd", size = 3168835, upload-time = "2026-01-05T10:40:38.847Z" }, + { url = "https://files.pythonhosted.org/packages/47/50/b3ebb4243e7160bda8d34b731e54dd8ab8b133e50775872e7a434e524c28/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5", size = 3521673, upload-time = "2026-01-05T10:40:56.614Z" }, + { url = "https://files.pythonhosted.org/packages/e0/fa/89f4cb9e08df770b57adb96f8cbb7e22695a4cb6c2bd5f0c4f0ebcf33b66/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e", size = 3724818, upload-time = "2026-01-05T10:40:44.507Z" }, + { url = "https://files.pythonhosted.org/packages/64/04/ca2363f0bfbe3b3d36e95bf67e56a4c88c8e3362b658e616d1ac185d47f2/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b", size = 3379195, upload-time = "2026-01-05T10:40:51.139Z" }, + { url = "https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67", size = 3274982, upload-time = "2026-01-05T10:40:58.331Z" }, + { url = "https://files.pythonhosted.org/packages/1d/28/5f9f5a4cc211b69e89420980e483831bcc29dade307955cc9dc858a40f01/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4", size = 9478245, upload-time = "2026-01-05T10:41:04.053Z" }, + { url = "https://files.pythonhosted.org/packages/6c/fb/66e2da4704d6aadebf8cb39f1d6d1957df667ab24cff2326b77cda0dcb85/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a", size = 9560069, upload-time = "2026-01-05T10:45:10.673Z" }, + { url = "https://files.pythonhosted.org/packages/16/04/fed398b05caa87ce9b1a1bb5166645e38196081b225059a6edaff6440fac/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a", size = 9899263, upload-time = "2026-01-05T10:45:12.559Z" }, + { url = "https://files.pythonhosted.org/packages/05/a1/d62dfe7376beaaf1394917e0f8e93ee5f67fea8fcf4107501db35996586b/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5", size = 10033429, upload-time = "2026-01-05T10:45:14.333Z" }, + { url = "https://files.pythonhosted.org/packages/fd/18/a545c4ea42af3df6effd7d13d250ba77a0a86fb20393143bbb9a92e434d4/tokenizers-0.22.2-cp39-abi3-win32.whl", hash = "sha256:a6bf3f88c554a2b653af81f3204491c818ae2ac6fbc09e76ef4773351292bc92", size = 2502363, upload-time = "2026-01-05T10:45:20.593Z" }, + { url = "https://files.pythonhosted.org/packages/65/71/0670843133a43d43070abeb1949abfdef12a86d490bea9cd9e18e37c5ff7/tokenizers-0.22.2-cp39-abi3-win_amd64.whl", hash = "sha256:c9ea31edff2968b44a88f97d784c2f16dc0729b8b143ed004699ebca91f05c48", size = 2747786, upload-time = "2026-01-05T10:45:18.411Z" }, + { url = "https://files.pythonhosted.org/packages/72/f4/0de46cfa12cdcbcd464cc59fde36912af405696f687e53a091fb432f694c/tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc", size = 2612133, upload-time = "2026-01-05T10:45:17.232Z" }, +] + +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] + +[[package]] +name = "torch" +version = "2.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-bindings", marker = "python_full_version < '3.15' and sys_platform == 'linux'" }, + { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx" }, + { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, + { name = "setuptools" }, + { name = "sympy" }, + { name = "triton", marker = "python_full_version < '3.15' and sys_platform == 'linux'" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/fe/cba54dc58523434919b66f13a667e36e436deddd77ca519e96553617d4ec/torch-2.13.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:e76f9bcecc52b8ff711239a2f7547d5353df95878ab232f0773c1d95928b92f8", size = 111187938, upload-time = "2026-07-08T16:05:17.065Z" }, + { url = "https://files.pythonhosted.org/packages/c2/59/1e3160e18e12aa3038390efab3ce02b36a9d4d6a527ecdd8520dca2e68d8/torch-2.13.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:092790c696a760c729fd5722835f50b9d81fd7c8f141571f3f3cf4081a8f664c", size = 427199369, upload-time = "2026-07-08T16:04:51.054Z" }, + { url = "https://files.pythonhosted.org/packages/01/79/1f2d34ad7034ee1c7ffc1cf8bf0f8213af2a81df6ecdb3997ecec107c09d/torch-2.13.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:60fcdcb2f3876e21146cb4524ef06397d727ca9ad5f020818547e25075fe3cb7", size = 526574961, upload-time = "2026-07-08T16:04:07.075Z" }, + { url = "https://files.pythonhosted.org/packages/6c/fd/0f2ce40f58aefbdb3392f9acce3c8171940943ae2d661f70558bfa73befb/torch-2.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:a0d8b11f16a48d60e2015d8213aa0390744cbebb98e58b62b3514dddc656e330", size = 122015870, upload-time = "2026-07-08T16:05:27.59Z" }, + { url = "https://files.pythonhosted.org/packages/c4/3a/ed0f4d4d1dcde03bced7aac9a28e800abcdc0cbd06b6775044c9fbd877b7/torch-2.13.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2fe228aba290d14b9f31b049be550dbd469c3fd3013d7a19705b30454da97027", size = 111213045, upload-time = "2026-07-08T16:05:22.997Z" }, + { url = "https://files.pythonhosted.org/packages/df/a9/f6a2a4d763ff1df02e9a64c477029db614295bc9367f4131223791ccc243/torch-2.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:572df8be8ffb4599c88cbd6a0726f1f854f4da65d2e3c09f0e2c2283333cd6d4", size = 427210998, upload-time = "2026-07-08T16:04:37.708Z" }, + { url = "https://files.pythonhosted.org/packages/f3/82/fea946351658e6534db52d2cc12bc53087cbf87f9440c5f180f367c1950b/torch-2.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:796633c4cdf0fe2cdced72d8f88f22e73dbcfce83132763162f6d4bff13b820b", size = 526605292, upload-time = "2026-07-08T16:04:22.81Z" }, + { url = "https://files.pythonhosted.org/packages/21/d6/e8f3c6f7e01f626f77259de9860d2a78bc84c40539e28e79b7e98b0bb659/torch-2.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:024c6cc0c1b085f2f91f20a3dc27b0471d021c31ce84b81be3afdc39f791fd9d", size = 122057313, upload-time = "2026-07-08T16:03:53.43Z" }, + { url = "https://files.pythonhosted.org/packages/0d/fa/c1c10b7aff4a9a3e8956d4f0a5f468fa6db7abc3208805719076772b4833/torch-2.13.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:33449899ce5496c1b84b4853179d94fd102028ae1407314d9fb956bb79e70d09", size = 111213743, upload-time = "2026-07-08T16:03:28.579Z" }, + { url = "https://files.pythonhosted.org/packages/11/18/9ecb37b56293a0be8d80f810bf672a72fe7e02f8b475d5ef1b9bf8a0d748/torch-2.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:1e09d6a722504957c694faceca843acde562786df1144ebcc5a74075ec7f6005", size = 427213008, upload-time = "2026-07-08T16:03:44.106Z" }, + { url = "https://files.pythonhosted.org/packages/d4/5a/7c50ba1b7b713d71d34669c6d13dab0a11531a3eceb0307a5162dbfec0f7/torch-2.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:a3a9a21312872af8a26950b2c15680335a386a1f56ed03e780653d78b9607e9e", size = 526602329, upload-time = "2026-07-08T16:03:12.649Z" }, + { url = "https://files.pythonhosted.org/packages/91/3d/e7adcc6aaf36961cd18f56cf8ad0f3058c3a5c84ccf391762176c94581b8/torch-2.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:49b58f1e2c52440abb6f17c28f0335fe6c6d01ad1a7f55b0183b81e4b34d64e6", size = 122057920, upload-time = "2026-07-08T16:03:01.808Z" }, + { url = "https://files.pythonhosted.org/packages/36/76/6dcc7f0c07052102dd36f83cbc5800842a909c8c3fbf1a7f8a5844954de9/torch-2.13.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d849b390e07d8d333ce8ecaf91b273c656c598379a19c9acf1318a883f6b391c", size = 111227066, upload-time = "2026-07-08T16:03:33.6Z" }, + { url = "https://files.pythonhosted.org/packages/e9/09/2c10e8cd0e00fa5d23c052df6ce467eaa7182399f5e0f824f1e4ff42ccae/torch-2.13.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:a3893dc2da0a972a8ca5d698c85a9f967559ac5f8ee1797b77408aa8734d073c", size = 427226309, upload-time = "2026-07-08T16:02:53.127Z" }, + { url = "https://files.pythonhosted.org/packages/76/c6/22c2102bbef14ca6a6cb4c20e42f088e49c5f812be4e160ae57502e325f9/torch-2.13.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:49f1ea385c754e54919408a9bb3b5a72b0b755bbe2c916c1d6f70afbec4908a2", size = 526614507, upload-time = "2026-07-08T16:02:16.441Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0c/7d1deb6bce5bc3e6042caf39100ac768eba3b9a098e1dddd16f75bd6489b/torch-2.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:4f8573e3ce9ebcd53fe922f01077a6085ccdfbe5f12fd215883a9d87d7a744fd", size = 122051871, upload-time = "2026-07-08T16:03:23.521Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ce/aa8b7f9949d32e0f2f624f342bc3b48112c1b8a130288465938bc83bcbf9/torch-2.13.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:c28def70706c2f9ecc752574766e8ae4da9b810ab6676b611166761a78a9f1e1", size = 111537025, upload-time = "2026-07-08T16:02:44.28Z" }, + { url = "https://files.pythonhosted.org/packages/69/d1/491e3a0389430946145888b0203f2b6a759ce2a61481b96a85c2da4f2ced/torch-2.13.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:31061ff56ed8fbf26c749806905aeb749ebeb819810fd5d52508aa5afd90dddc", size = 427219769, upload-time = "2026-07-08T16:02:31.18Z" }, + { url = "https://files.pythonhosted.org/packages/9a/1d/38006e045bf0a1fc28ef01e757c554e59e59a8770c284bc4f47b14e60441/torch-2.13.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:cc26eead4cf51d0b544e31e364dcf000846549c273bd148936fe9d24d29acb92", size = 526571320, upload-time = "2026-07-08T16:01:59.348Z" }, + { url = "https://files.pythonhosted.org/packages/56/94/655c91992a882bd5071aa0b5d22a07dbb130d801e872be97c0b627a7c693/torch-2.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:a7de8a313090dc5c7d7ba4bfe5c3be222528f9a4dba1acc83bddb1157360c4b8", size = 122306773, upload-time = "2026-07-08T16:02:39.832Z" }, +] + +[[package]] +name = "tqdm" +version = "4.70.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/21/3b/6c24bec5be5e743ffd99576daa5cc077722fc7d5bbc00bd133fa0c698dc6/tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220", size = 795438, upload-time = "2026-07-27T11:33:15.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/1c/01bfd571a64e7f270e6bab5e33777debe0edc56759233ce84f27dec92d14/tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953", size = 80184, upload-time = "2026-07-27T11:33:13.167Z" }, +] + +[[package]] +name = "transformers" +version = "5.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "regex" }, + { name = "safetensors" }, + { name = "tokenizers" }, + { name = "tqdm" }, + { name = "typer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/fb/2a2ba88f325e68a921d8b69ff63b477830b2e73ade9a3c8c8cab2f06d741/transformers-5.14.1.tar.gz", hash = "sha256:60d196c27781eacf8637e2b533f517582907ad6f9ae142046d6b69431a5b2173", size = 9295927, upload-time = "2026-07-16T09:41:57.773Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6f/67/8d85ca2323233ae3c0365a659c4e52ee1f587b440e4bc577e7d8e4416d0f/transformers-5.14.1-py3-none-any.whl", hash = "sha256:9db974c4079ede2d1a3ea7ca5a240df33f2cc26fc2b36ba64c5f2a4f43b6e725", size = 11625234, upload-time = "2026-07-16T09:41:54.143Z" }, +] + +[[package]] +name = "triton" +version = "3.7.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/f9/19d842d06a08559534fa1eaab6ca551b1bcf40f06620bddec1babaa2772d/triton-3.7.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4a0e1cd4c4a76370ed74a8432a53cea28716827d19e40ffc732233e35ceb3f6", size = 184664887, upload-time = "2026-06-17T20:03:42.913Z" }, + { url = "https://files.pythonhosted.org/packages/cd/5e/fce69606f7f240297f163e25539906732b199530d486ce67ae319877e821/triton-3.7.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6744957e9fd610a29680ec2346057d0c86948ed3812468670719f391e94b44a5", size = 197701306, upload-time = "2026-06-17T19:53:13.673Z" }, + { url = "https://files.pythonhosted.org/packages/94/fa/f856e24deb462d5f18bd4b5a746957862ab9b6ee5834bda60605ec348366/triton-3.7.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9497f2e696ee368862a181a90b2dcc03ca978cc4f602abd67c7d81022a6988e1", size = 184692359, upload-time = "2026-06-17T20:03:48.288Z" }, + { url = "https://files.pythonhosted.org/packages/c4/6f/fb96d15db6f36d6eae4cafb998c2e0353bf59d7c4ea1662d7497f269134a/triton-3.7.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e40869937a68206ec70d7f25bb7ec6433cb083f9135e1f36dbd318dc449a728", size = 197719725, upload-time = "2026-06-17T19:53:20.419Z" }, + { url = "https://files.pythonhosted.org/packages/00/42/c5089d4d9327fcd1e862c599cc2927f39418f84dd11a84cb2ccff9d4787a/triton-3.7.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cdbfc09d9ec58bc5e68321525653220de7515c199e7a8097a97c85e62b52cd0a", size = 184694629, upload-time = "2026-06-17T20:03:53.444Z" }, + { url = "https://files.pythonhosted.org/packages/07/42/2c3ac59253ae8892b6f307875263dd23dc875cdf732d3aea40d6d41fb7cb/triton-3.7.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58c0e131da05134a2a4788ccbcc0c1105cf0f54c8e98f19e34cd465396dc15eb", size = 197729241, upload-time = "2026-06-17T19:53:27.801Z" }, + { url = "https://files.pythonhosted.org/packages/40/71/e01aa7ad573883ed9456f130226babdec70b005e098c4d6226a6238e761b/triton-3.7.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe4ea396a06171f1f1f58cbd39c70b09294398f7dd7c620939bab54ad6f934fa", size = 184705764, upload-time = "2026-06-17T20:03:59.064Z" }, + { url = "https://files.pythonhosted.org/packages/a4/09/5683146fda6a2b569deb78ccfd8fbfea8bfe55f726b081c0a6bb18dd6f28/triton-3.7.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2020153b08280415ec0da6607834e79166442147e78e144df06b508c75b186d2", size = 197729537, upload-time = "2026-06-17T19:53:35.516Z" }, + { url = "https://files.pythonhosted.org/packages/e9/f8/448220c3092019f9fdfab39ec47985968181d67da34b44f6a7f6280a5cbb/triton-3.7.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c58e4c61f0c73b5dba3b5d19b4a7093c32f90dc18b2a7f121a7c16ccd31107b7", size = 184814760, upload-time = "2026-06-17T20:04:04.984Z" }, + { url = "https://files.pythonhosted.org/packages/f0/ac/229b7d4589d2e5937310e72c6d46e89599d16a4a12b479ffa1499fee8eb8/triton-3.7.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10ba85fa2cca4a2fbdeb36bf1cb082f2c252bda55bf9fccd74f65ec5bc647e68", size = 197824404, upload-time = "2026-06-17T19:53:42.772Z" }, +] + +[[package]] +name = "typer" +version = "0.27.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "rich" }, + { name = "shellingham" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/37/78/fda3361b56efc27944f24225f6ecd13d96d6fcfe37bd0eb34e2f4c63f9fc/typer-0.27.0.tar.gz", hash = "sha256:629bd12ea5d13a17148125d9a264f949eb171fb3f120f9b04d85873cab054fa5", size = 203430, upload-time = "2026-07-15T19:21:07.007Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/03/26a383c9e58c213199d1aad1c3d353cfc22d4444ec6d2c0bf8ad02523843/typer-0.27.0-py3-none-any.whl", hash = "sha256:6f4b27631e47f077871b7dc30e933ec0131c1390fbe0e387ea5574b5bac9ccf1", size = 122716, upload-time = "2026-07-15T19:21:05.553Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "tzdata" +version = "2026.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/92/ff/5a28bdfd8c3ebec42564ac7d0e54ca3db65044a9314a97f9564fa7a1e926/tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415", size = 198674, upload-time = "2026-07-10T08:50:37.887Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/6d/b53b99a9f2766d095985947a5782f1702cabb129a34f7a802d7197af832f/tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931", size = 348168, upload-time = "2026-07-10T08:50:36.46Z" }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] + +[[package]] +name = "xxhash" +version = "3.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/63/71aa56b151a1b28770037a61bd4e461c2619cfc8866a4fcaf1548605e325/xxhash-3.8.1.tar.gz", hash = "sha256:b0de4bf3aa66363552d52c6a89003c479911f12098cd48a53d44a0f7a25f7c46", size = 86223, upload-time = "2026-07-06T10:49:58.937Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/5a/05eaa129555f85476a3e16ff869e95f81a78bbe4647eef9d0229f515a317/xxhash-3.8.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602efcad4a42c184e81d43a2b7e6e4f524d619878f2b6ee2ba469011f47c8147", size = 34699, upload-time = "2026-07-06T10:44:10.14Z" }, + { url = "https://files.pythonhosted.org/packages/80/59/0df1133958b2228929355e022aab1e958c7b2c43e27bf7f59bc9edfa8a54/xxhash-3.8.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:131324f719957b988861714de7d6ddf57b47abec3b0cc691302ffeaba0e05e10", size = 32373, upload-time = "2026-07-06T10:44:11.353Z" }, + { url = "https://files.pythonhosted.org/packages/3e/bf/1cfda5b5e6bf26617812b4a31662ef2220d2ad04e0a55b8ff9eb36e56a5c/xxhash-3.8.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:db77278a6eddadbf44ce5aae2fee5ebb4d061f026b1ce2130d058cd4d7a7b670", size = 220284, upload-time = "2026-07-06T10:44:12.683Z" }, + { url = "https://files.pythonhosted.org/packages/70/93/45dc0ad7913b69e5b08bd039236cf628380e4c9cc76a8a4c6625a328e058/xxhash-3.8.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c332dd48b8cb050da2bb2a3c96d72b1664168650a250ef9718e423df7989e05", size = 240980, upload-time = "2026-07-06T10:44:14.297Z" }, + { url = "https://files.pythonhosted.org/packages/e9/02/f28ba7d17f2c1410ee397982c817ab1bd5b2701070c2d2c373539aad000a/xxhash-3.8.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a5cd96f6dcdf4fa657b2d95668d71d58455248f98712ecffaa9c528edf40ccae", size = 264526, upload-time = "2026-07-06T10:44:16.017Z" }, + { url = "https://files.pythonhosted.org/packages/5c/d0/f10651cec2c7981b20d693deae6bdfc438427d92be2db4ccabb6181f0021/xxhash-3.8.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c959f88160b13b4e730b0d75b459b7929fc0d2225c284c9683ac95d6feeeac6a", size = 241369, upload-time = "2026-07-06T10:44:17.698Z" }, + { url = "https://files.pythonhosted.org/packages/ff/40/136e0cbaf5db51e191423b1c98643593189f02b6cd90837bf64b19113d70/xxhash-3.8.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:027dee4355f3fcc41481650d846cf6cfc895c85a1ab7acd063063821a0df5b4c", size = 473186, upload-time = "2026-07-06T10:44:19.354Z" }, + { url = "https://files.pythonhosted.org/packages/4b/3f/6aa808a96bdc43dba9a740dec56c744526ee3c0019e32c75e810fa90ae4d/xxhash-3.8.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ad52a0e4bcc0ba956a953a169d1feec2734a64981d689e4fc8f490f7bf91af60", size = 220092, upload-time = "2026-07-06T10:44:20.956Z" }, + { url = "https://files.pythonhosted.org/packages/47/28/a8675e78a9ced96dab853416162268e10e05b452e95db7888cf69f58ac5f/xxhash-3.8.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d3dfb1f0ff146da7952867a9414f0c7a29762f8825a84879592612fd6139342", size = 309846, upload-time = "2026-07-06T10:44:22.543Z" }, + { url = "https://files.pythonhosted.org/packages/89/0f/7fe4d4ef4e69f0033e012396ee2a115886bca7b10b7e45ce398626436bfc/xxhash-3.8.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4482380b462ca9e59994d072a877ecadd1cf51102daeeab2db696f96ab763723", size = 237659, upload-time = "2026-07-06T10:44:24.135Z" }, + { url = "https://files.pythonhosted.org/packages/38/8f/83e9e31d4ed57fe963b99cb5b13a23e3e0f0dad1885aa0ebd2a7819dd423/xxhash-3.8.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:950ac754d16daea42038f38e7465eb84cda4d08d7343c1c915771b29470f065a", size = 268737, upload-time = "2026-07-06T10:44:25.875Z" }, + { url = "https://files.pythonhosted.org/packages/57/79/7e7de46dbe5d1f49afc96a0bc42e6b8df24eae3d6bad6007b99e42f48430/xxhash-3.8.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0418ec8b2331b9d4d575fc9284427e8e69449d7172e99e1a86fcdd1f51a0a937", size = 224955, upload-time = "2026-07-06T10:44:27.777Z" }, + { url = "https://files.pythonhosted.org/packages/ec/34/b8540839e958d5ef5c6101af6f16032109e7099698ae8edbc8dcefe4d8f4/xxhash-3.8.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:32a94ad2763e0263d9102037d349002c3d3c401e42770542c3eeb4801f311661", size = 239653, upload-time = "2026-07-06T10:44:29.422Z" }, + { url = "https://files.pythonhosted.org/packages/ce/87/a735d05f7f859354acadabe470ff40e2c46672275f96dcf096a761904def/xxhash-3.8.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:89b11a5cdd441aa463f6d34ca0241602bc09b001a76994b6059828494108c673", size = 300213, upload-time = "2026-07-06T10:44:31.401Z" }, + { url = "https://files.pythonhosted.org/packages/98/31/3e1cb020237b68117fc212dc5f9753b87f865b4dfee7c1ce62d0836955b5/xxhash-3.8.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:09a204dd4bb0823daf938cdd0dc8057d5f1e14fe3cbde929424255f23f9de872", size = 442508, upload-time = "2026-07-06T10:44:33.023Z" }, + { url = "https://files.pythonhosted.org/packages/23/bf/f80090622141cc734b039ce1d15ce3ff6dced375e9680249bf5b9b8c6bf9/xxhash-3.8.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e710ad822c493fb80a4fbc1e3d0a807b1422cb90adbe64378f98291b7fa48fef", size = 216853, upload-time = "2026-07-06T10:44:34.983Z" }, + { url = "https://files.pythonhosted.org/packages/a6/a3/60157acecc307b238d3651c2483168e224b48b23a36ae6d6903588341d80/xxhash-3.8.1-cp311-cp311-win32.whl", hash = "sha256:5013be3bea7612852c62a7437f3302c1cfb91ca7e703b194459db0b2b2e0d792", size = 31936, upload-time = "2026-07-06T10:44:36.542Z" }, + { url = "https://files.pythonhosted.org/packages/59/5c/ef70c418d878d187b8da56d4cdc06aea6cf5e456b301e96e51e1d2cc8625/xxhash-3.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:f377012b86c0a23a1df0cf5a1b05aa7187649e472f71c7892e5f2c2815bbe74f", size = 32724, upload-time = "2026-07-06T10:44:38.177Z" }, + { url = "https://files.pythonhosted.org/packages/2c/25/f008db952cec6b2a26445b456eeed2ebebd65e08e848ebe09ed6ac0634e6/xxhash-3.8.1-cp311-cp311-win_arm64.whl", hash = "sha256:836f11d4474d3228e9909d97216faa4f7505df41cfaf3927eb29809de785a78d", size = 29212, upload-time = "2026-07-06T10:44:39.577Z" }, + { url = "https://files.pythonhosted.org/packages/42/91/f65c34a7aa7b4e7cf4854f8e6ef3f7ee32ceac41d4f008da0780db0612f6/xxhash-3.8.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e6e49370822c1f4d8d90e678b06dbcb08b51a026a7c4b55479e7d467f2e813bc", size = 34680, upload-time = "2026-07-06T10:44:40.932Z" }, + { url = "https://files.pythonhosted.org/packages/57/04/b10a245a4c09a9cfa88f8e9ae755029413ad1ac17047f9a61906e5ae0799/xxhash-3.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:220d68130f83f7cc86d6edfdeab176adc73d7200bf3a8ec10c629e8cf605c215", size = 32397, upload-time = "2026-07-06T10:44:42.196Z" }, + { url = "https://files.pythonhosted.org/packages/3a/75/45ab795b5945b6388583bd75202106af505537935566c15a1577797a0e08/xxhash-3.8.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4d365ee1892c1fa803536f8c6ce21d24b29c9718ec75eb856095c07830f8c478", size = 220549, upload-time = "2026-07-06T10:44:43.603Z" }, + { url = "https://files.pythonhosted.org/packages/13/44/5ba2bd0a14ddf4193fc7d8ec29625f659f22c06d60b28f04bf46305d8330/xxhash-3.8.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:852bfe059720632e2f16a6a4745e41d20937b2bf2a42a401e2412046bb6971cc", size = 241186, upload-time = "2026-07-06T10:44:45.534Z" }, + { url = "https://files.pythonhosted.org/packages/23/32/c4147def4d1e4538b906f82731e0ba23424377fc50a7cddd03cd284c8f63/xxhash-3.8.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f8c25a7061d952de589bd0ea0eaadee32378ff83dd6a677b267f9cd86f401f8", size = 264852, upload-time = "2026-07-06T10:44:47.199Z" }, + { url = "https://files.pythonhosted.org/packages/6c/bd/71ed14f4f0318bb7fd7b2ec51999413487fa8da8d41208e84d50d1ef0f98/xxhash-3.8.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:868a8dcaff1a84ba78038e1cef14fc88ccf84d9b4d12ea604696e0693296aa56", size = 242663, upload-time = "2026-07-06T10:44:48.846Z" }, + { url = "https://files.pythonhosted.org/packages/91/09/70af22c565a8473b3f2ae73f88e7721af281bc4a575236dbd1970c9f76f6/xxhash-3.8.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6536d8677d2fff7e64cd0b98b976df9de7aee0e69590044c2af5f51b76b7a170", size = 473510, upload-time = "2026-07-06T10:44:50.695Z" }, + { url = "https://files.pythonhosted.org/packages/18/96/34db781c8f0cf99c544ca1f2bc2e5bf55426e1eb4ca6de8ea5da56a9f352/xxhash-3.8.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:82c0cedd280eab2e8291270e6c04894dbc096f8159a39dcf1807429f026ca3cc", size = 220469, upload-time = "2026-07-06T10:44:52.422Z" }, + { url = "https://files.pythonhosted.org/packages/93/5f/9a184f615fa5a4dce30c01534f62946ce5a11ce40f73785cbd356ccabaa9/xxhash-3.8.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daa86e4b68221d38e669bb236ba112d0335353829fb627c82e5909e4bbe8694c", size = 310290, upload-time = "2026-07-06T10:44:54.142Z" }, + { url = "https://files.pythonhosted.org/packages/a9/dc/9b9a9789011ee153723a5eb9e7dd7fcbae2ba9b3fe7a729249ca7c252056/xxhash-3.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2bc7113e6f2b6b3922dd61796ca9f36af09da3773898e7003038dc992fc83b8d", size = 238173, upload-time = "2026-07-06T10:44:55.693Z" }, + { url = "https://files.pythonhosted.org/packages/ec/4d/71c6005ada9dcb608a4e1902e8475ecadb5f3fbfa04e1e244d276a2d0c43/xxhash-3.8.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5eed32dad81d6ba8e62dc7b9ffa0500199385d7810a8dd9d4eafaceb8c6e20bb", size = 269026, upload-time = "2026-07-06T10:44:57.424Z" }, + { url = "https://files.pythonhosted.org/packages/2f/87/d6c036ba25dfbd9c8633be5aa86fc9474bbb9e2c68212a841d090abe7344/xxhash-3.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:83697b0ea1f10e7f5d8b26a4906fa851393c61546c63839643a2b7fe2d868061", size = 224970, upload-time = "2026-07-06T10:44:59.085Z" }, + { url = "https://files.pythonhosted.org/packages/48/62/4c1f035a41c5752aa05e195b6c904c07b94fe9061a16de61e72a6e6b135f/xxhash-3.8.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:36fc69160465ae75c6ec4ac9f781bb2aa16ae7ff869e73c26fee85fbb11b9887", size = 240820, upload-time = "2026-07-06T10:45:00.746Z" }, + { url = "https://files.pythonhosted.org/packages/da/14/d39d565069b87e86d21a2af2a31d04db79249d25aa8d5b62959056a89857/xxhash-3.8.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:445e0f5a31f2f3546ae0895d4811e159518cdc9d824c11419898d40cfadb677e", size = 300619, upload-time = "2026-07-06T10:45:02.716Z" }, + { url = "https://files.pythonhosted.org/packages/13/22/75467acc887edc8cf71c97ab1708feb3df7a88bda589b9f399765c6387d2/xxhash-3.8.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:dfe0580fbfd5e4af87d0cc52d2044f155d55ebd8c8a93568758a2ea7d8e15975", size = 443267, upload-time = "2026-07-06T10:45:04.653Z" }, + { url = "https://files.pythonhosted.org/packages/a4/b6/1da3baa5fa6ef705e3425fddd382be7dfc4dfba2686df90a20f16e9c7b1b/xxhash-3.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:095e1323fa108be1292c54c86da3ef3c7a7dc015b105a52133973bc07a6ad11a", size = 217338, upload-time = "2026-07-06T10:45:06.304Z" }, + { url = "https://files.pythonhosted.org/packages/78/dd/b5295a9f97484e7a1c2b283a742ca45e3104991c55a1ef670dde161829ba/xxhash-3.8.1-cp312-cp312-win32.whl", hash = "sha256:bf28f55e427e0483acb1f666bd0d869b6d5e5a716680c216ad7befe3d4cfba2e", size = 31970, upload-time = "2026-07-06T10:45:07.823Z" }, + { url = "https://files.pythonhosted.org/packages/ec/31/3fa0b807d7e21515cd975e7fe5c039d52ac3e9401a96d6ad68dae6305215/xxhash-3.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:2256e80e4960ee282f63428adb349cb7f8bd8efe4db770d88eb815f4b9860724", size = 32741, upload-time = "2026-07-06T10:45:09.42Z" }, + { url = "https://files.pythonhosted.org/packages/b8/05/86feada74e239600e6875aa507afb40482a89b92700aa74a92da83bdcb77/xxhash-3.8.1-cp312-cp312-win_arm64.whl", hash = "sha256:9df56e6df96a60590935e22373041cccc91fd55858763dcffb55bf63b3a2b396", size = 29234, upload-time = "2026-07-06T10:45:10.809Z" }, + { url = "https://files.pythonhosted.org/packages/6b/8c/446bb782cd0d27007a917b5569a08dd73219c3e8d6e459014db104b27bdb/xxhash-3.8.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:3c682fcd96eb4bf64be32a4d95f96107e1588005831bd8a741b324fdda01b913", size = 38562, upload-time = "2026-07-06T10:45:12.425Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ec/c0c45627eaa6be7a5d6117423adf8f7a15b17ee74b4b17072cca5959a225/xxhash-3.8.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:036a024d8b9c01f70782e09ed98d532e76fd23f950ae7154bd950fe94e90ebec", size = 36656, upload-time = "2026-07-06T10:45:13.932Z" }, + { url = "https://files.pythonhosted.org/packages/f6/94/8324c04cc7597154caaeba6c094e01fbd2e7601d01e7a13eea9f5420e77b/xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d6a5c0bce213b23b0166fe0d35bcbbe23ce4b968f257cc7eb6fd57cb8e1e6297", size = 31169, upload-time = "2026-07-06T10:45:15.687Z" }, + { url = "https://files.pythonhosted.org/packages/40/a4/beb6bb26e1184e126dbe7a5682330214ef54dcfbf882078aa9f4b5428d42/xxhash-3.8.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5177aa44eddaa97c6ef0cc00c6d540edb64d51781d2f8fb941612ec61a92c9ed", size = 32177, upload-time = "2026-07-06T10:45:17.035Z" }, + { url = "https://files.pythonhosted.org/packages/56/0f/fc4c92a5a528f839b34b6419b2e53c8597f2a629d5a1f5d721f65bfa1fd6/xxhash-3.8.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7801b7223db017b9c0c9ccf37e44524edb35a1544a1c032add22c061c6af0276", size = 34642, upload-time = "2026-07-06T10:45:18.39Z" }, + { url = "https://files.pythonhosted.org/packages/d4/58/edbfb141d4000767ac6a9694f8ac0763e2c2e983e65c9e31620ba56e2667/xxhash-3.8.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:9e80238259655bf69d7bcd08226a970d7f42605f3157786bfa76dd13472d7fa0", size = 34684, upload-time = "2026-07-06T10:45:20.033Z" }, + { url = "https://files.pythonhosted.org/packages/07/3f/5072f1f0f5714186f0ac2a0b5a4929ce30d4b845e94886b6c01b6ebda0be/xxhash-3.8.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bcab50a389cc04d87f90092af78a6adba2ab3deca63175a3344ca83514045315", size = 32401, upload-time = "2026-07-06T10:45:21.414Z" }, + { url = "https://files.pythonhosted.org/packages/49/c7/802ea2f9c2ed59219934d6d65c470d502b1788043eae277a52af8658bda6/xxhash-3.8.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a2489d3a776fa380cb8e71f54c7fda268a9baf3de9b1395093fd280f95735907", size = 220617, upload-time = "2026-07-06T10:45:23.234Z" }, + { url = "https://files.pythonhosted.org/packages/99/a8/e10488efd31fcb13fcd6acbc6e788f10c6f8e3a0cc4ae3eb89dc19c55a12/xxhash-3.8.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:32ab1e5432690276e71192be7401b55f96db2d0eedea5d44eb1f164505669cc0", size = 241295, upload-time = "2026-07-06T10:45:25.364Z" }, + { url = "https://files.pythonhosted.org/packages/18/cc/14180b17d44892a631f8ae7323c30bfbb1328efc8209e528a480293528ac/xxhash-3.8.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b30e01a0b97a4bc3f519a4d7a82da3dc53251fb0de5eeea8660dcd4ff094c0c2", size = 264688, upload-time = "2026-07-06T10:45:27.09Z" }, + { url = "https://files.pythonhosted.org/packages/a9/72/a14019d0c5f6c41ee407a503036ae32787c91325ca218a96a9b5627be651/xxhash-3.8.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1f44275ddb0978b67a58a951501903f04d49335a91f7681c9ce122ecb8ccb329", size = 242740, upload-time = "2026-07-06T10:45:28.753Z" }, + { url = "https://files.pythonhosted.org/packages/68/08/92550e556c6fcfcb96c6a336945eb53a431ed43120ed749636debb16c5cf/xxhash-3.8.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e3b87cbd974512c0c5fc7b469c36b2cdc9ee6d76e4ec78bccb2c7184611c49b0", size = 473599, upload-time = "2026-07-06T10:45:30.524Z" }, + { url = "https://files.pythonhosted.org/packages/29/83/e361d3c1acd1b21e1d489616de6fa4aaf843365d8179f612e3743eac20a9/xxhash-3.8.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98ee81b4b7f3023c9cb04a78cc67610baffcb5812d92f2096cb5a5efc6f19437", size = 220559, upload-time = "2026-07-06T10:45:32.979Z" }, + { url = "https://files.pythonhosted.org/packages/05/01/006a4243c2c2a6831827f9999f6d1c23feeef100eb023c1f886022a00bf3/xxhash-3.8.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2666f059a1588a99267e33605365ed89cea92f424b3522806a9f4bd8ad2e3d62", size = 310383, upload-time = "2026-07-06T10:45:35.875Z" }, + { url = "https://files.pythonhosted.org/packages/d8/20/af388e8bf9f9a0f89eeef7d2a1935d176ee1c20bc6adeda05035879379cf/xxhash-3.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b0093cf7eeb91b84776e8742113afa4bdf47533d36cf719179aaaf1f56f6f8bf", size = 238228, upload-time = "2026-07-06T10:45:38.02Z" }, + { url = "https://files.pythonhosted.org/packages/63/6b/4666579a87eebd1744663c404297355fa0658617b015cedfa58810ee7036/xxhash-3.8.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3a800912a2e5e975d4128969d645c4a2a80aa886ccd6c9b1c6f44529e327e8cf", size = 269137, upload-time = "2026-07-06T10:45:39.954Z" }, + { url = "https://files.pythonhosted.org/packages/de/d3/e963a8a46f900a137d91b02144d8ea07a8f812971b138204a3b2f8b8e55c/xxhash-3.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:0fe37f72a207223d22a4eddc3149d4298993385aa9daef25c039246ca5a309f3", size = 225068, upload-time = "2026-07-06T10:45:41.718Z" }, + { url = "https://files.pythonhosted.org/packages/aa/80/9d181dbcde4b0fe48375f48833a5832d4b8cd2b349b15110c92ee472d874/xxhash-3.8.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5db43f249b4be9f99ef4b967863f37094fb40e67effafb78ba4f0356b6396104", size = 240874, upload-time = "2026-07-06T10:45:43.414Z" }, + { url = "https://files.pythonhosted.org/packages/39/15/ce3ab5a1cd27ead25a5196e55a7284220f6ad6e316da494ffd900b2b600f/xxhash-3.8.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c4ed42965c2cd9081f011be22f69d0e65d3b6165fe7734072fd0c232840bbd4e", size = 300702, upload-time = "2026-07-06T10:45:45.135Z" }, + { url = "https://files.pythonhosted.org/packages/96/c0/2281a8ab5f2a62dbf57a23c58a01ccc1d98abf40f71193c8a81f59e759b5/xxhash-3.8.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3557bec8fcb11738a8920eeb68974bc76b75262f6947998d3147954ce0a4b893", size = 443351, upload-time = "2026-07-06T10:45:47.188Z" }, + { url = "https://files.pythonhosted.org/packages/81/2e/071a58c1a53a52d4f7a3aa0987be0c396dffd40da8204805fe1b130a81f4/xxhash-3.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:00de40f3b42240db23a82a5c682b55d7263d84a26a953240c1aee463409660e3", size = 217396, upload-time = "2026-07-06T10:45:48.925Z" }, + { url = "https://files.pythonhosted.org/packages/68/44/36ab58134badd9d3433fc7b53c4ca8d113d8e807782885628640f8297a4d/xxhash-3.8.1-cp313-cp313-win32.whl", hash = "sha256:b5196cc2574cfec572a5f3fb7cfa5ade27305ae3d06516a082132441aff4c83a", size = 31974, upload-time = "2026-07-06T10:45:50.591Z" }, + { url = "https://files.pythonhosted.org/packages/96/2a/2a0b84798448e766f7b89ceed073cb0cb5a43fc9ebbacbdea74a38de18e3/xxhash-3.8.1-cp313-cp313-win_amd64.whl", hash = "sha256:538f5f865df6cd8c32dd63158a0e5b4f5dd08d732a7da8b7228a5a0776c8ce55", size = 32739, upload-time = "2026-07-06T10:45:52.221Z" }, + { url = "https://files.pythonhosted.org/packages/d4/60/bb51dbf7c363ff88a7cbd50b7959718219577ef44d7cf255929ffc4a2194/xxhash-3.8.1-cp313-cp313-win_arm64.whl", hash = "sha256:a6617f30641ba0d8baa1635fbefb1dffc5165ec36d26921bd5cee13497cd937a", size = 29239, upload-time = "2026-07-06T10:45:53.714Z" }, + { url = "https://files.pythonhosted.org/packages/56/d3/827ca123c2ee5443a6aaed3c5dd199237dc2f010e2bebd7ec09ef36f3a5f/xxhash-3.8.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:bfcd82852c62a60e314670a9602de354c4460f8adad916e2e42a20860c7870bc", size = 34964, upload-time = "2026-07-06T10:45:55.535Z" }, + { url = "https://files.pythonhosted.org/packages/05/67/67ae2a3ccdeb8b8ef025d35aee9edd1d26c3abe5051d47da9286232afbf8/xxhash-3.8.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:08ea2081f5e88615fec8622a9f87fbe21b8ea58d88cfc02163ca11026ee62a92", size = 32697, upload-time = "2026-07-06T10:45:57.288Z" }, + { url = "https://files.pythonhosted.org/packages/38/5a/3d3994346e1f45493679cb5c1ffc2bf454e410e9d1e8a662d253becee91e/xxhash-3.8.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2e32855b6f9e5b18f449e59d45e3d5778bdeb660632ef2693cca267a11246c75", size = 225954, upload-time = "2026-07-06T10:45:58.897Z" }, + { url = "https://files.pythonhosted.org/packages/3f/2c/53169270309b7cd8e05504e07fe123bac053b89d00ac63617faacf0a2ec0/xxhash-3.8.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6e088bd7870775624256a0d84c2a6714afd223b2eeb56b0ca58398e52a32fda", size = 249776, upload-time = "2026-07-06T10:46:00.977Z" }, + { url = "https://files.pythonhosted.org/packages/70/e0/5c551d8d592f944506f7c5185e210255c15e672a3c6008c156a1bd9b775e/xxhash-3.8.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:72eb5ae575cc7ae2b23f6f8064a8b10f638c7149819ae9cc6d20ebd4d37a1629", size = 274776, upload-time = "2026-07-06T10:46:02.869Z" }, + { url = "https://files.pythonhosted.org/packages/a0/2a/d3a762270cee2d7bcd0e25e28c623e5f3f5c0dc637b66e3e47dd5b0bb3f0/xxhash-3.8.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d0b48cdf690a64cedf7258c3dc9506cc41fc86edd7739c40e3098952265dc068", size = 252056, upload-time = "2026-07-06T10:46:04.688Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8f/b78e4373b2cb6d1c42af60ea2d7e9146ad0710b239ac7f706d5d31d5bb98/xxhash-3.8.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fb9e256a357dfcede7818c6d34e70db2d6b664394803d1de4b6984d2de76c0f1", size = 482108, upload-time = "2026-07-06T10:46:06.498Z" }, + { url = "https://files.pythonhosted.org/packages/e6/0d/642d923336ea61a15f8ce64fc7e078729e6e06c3a026e517fa79b2c23b7a/xxhash-3.8.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51f71a6e2ad071e70c937e41fcb6c19f82c3f9f49831eba850ed4a106ffbb647", size = 226739, upload-time = "2026-07-06T10:46:08.598Z" }, + { url = "https://files.pythonhosted.org/packages/a6/0a/a37d6da6427d45a8d23e3ee3a0ca9c9d4a90364849c6637fe2963a755f9b/xxhash-3.8.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e4a6443968c4e8dc69967e12776776a5952c119cc1bd94168ad1c5ad667c2be1", size = 319658, upload-time = "2026-07-06T10:46:10.504Z" }, + { url = "https://files.pythonhosted.org/packages/4a/51/ebbd40da8a3f1bc53b4b7a9a87f8e28bd95c5f21bc14b8a57860cf367d1b/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:714503083a1f2065c9ad15340dd49ac8a8e948a505a705ffa1750cb951519113", size = 246059, upload-time = "2026-07-06T10:46:12.634Z" }, + { url = "https://files.pythonhosted.org/packages/24/4c/d9014030147e1f0bb26e7da47aa240dd9ec61c763c573e558111d869f8e1/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:77f74e45a1e5574bbbf80181c8027b3a4c65c2248fffbd557bd596fff13102f9", size = 275535, upload-time = "2026-07-06T10:46:14.614Z" }, + { url = "https://files.pythonhosted.org/packages/84/86/caee2db41fadcd5a25aa4323213f9afec5a8586d4e419241e3d659362bd7/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:4e0e1b0fb0259c1b75d1251ac0bb4d7ab675d36f7a6bf4ba6aa630dae94f9ffa", size = 231292, upload-time = "2026-07-06T10:46:16.452Z" }, + { url = "https://files.pythonhosted.org/packages/0b/60/f52f08bcdc904c4514ea5c25caa19e9f3214144434a6ff96dc82dc1cbddd/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:10e4393ec33633c2f05ad01869e546ad080b1a18f2650503731f153774608b31", size = 250490, upload-time = "2026-07-06T10:46:18.318Z" }, + { url = "https://files.pythonhosted.org/packages/24/a0/94dc7ae310838f250669c6ad7168e6d6fca17d49dac1053f06dc232c4a56/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:b3ba794c3d885803db6c3116686923f1ec13bc86e621e169a375282b63ea1cc6", size = 309861, upload-time = "2026-07-06T10:46:20.503Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f9/adeead7d0eb28cdfc2832544ea639ffbc6749ccde47a8e228d667459182e/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:57189a69c0891e4818853feaa521c972d22c880a001453addea015f48e3c3398", size = 448739, upload-time = "2026-07-06T10:46:22.79Z" }, + { url = "https://files.pythonhosted.org/packages/04/a4/22ec0e07db57d901c9298ae98aa3cf2be45bafded6f07c13131e85b89032/xxhash-3.8.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d59e71153fe9ff85648d00e18649b07e9b22c797291abb7e27274fa06df8b838", size = 223657, upload-time = "2026-07-06T10:46:24.831Z" }, + { url = "https://files.pythonhosted.org/packages/94/32/8a9531f37b59e5a013003db7cb7414baf4ce7e0e1268e0d5947cd3d6a2df/xxhash-3.8.1-cp313-cp313t-win32.whl", hash = "sha256:5b96f0024e9840f449bd91b2d005c921a4b666055a0d1b6492463799f32aae22", size = 32377, upload-time = "2026-07-06T10:46:26.86Z" }, + { url = "https://files.pythonhosted.org/packages/e7/ab/2ca45fd7f671de5f81fc297ef1c95080b40c86ec6be0cc6034b8f7707ac8/xxhash-3.8.1-cp313-cp313t-win_amd64.whl", hash = "sha256:37d5a56c36dcc0b9a87b814cd992598d33863ff683749de6c86081f278d5e629", size = 33274, upload-time = "2026-07-06T10:46:28.39Z" }, + { url = "https://files.pythonhosted.org/packages/5a/54/20d7163463ddb6438b73a427d1655a77a502cf9b9b0c3ada3599629d9c0a/xxhash-3.8.1-cp313-cp313t-win_arm64.whl", hash = "sha256:6696c8752aded28ff3b16f33ef28ce28fb5d209b80c206746f943199fcf5fd65", size = 29375, upload-time = "2026-07-06T10:46:29.962Z" }, + { url = "https://files.pythonhosted.org/packages/c2/8b/df2ba04f22a6cd6b39f96a6577329a8471a55c90ef8d8e2f7c102363613f/xxhash-3.8.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:9db455cb649dcfe4504d6d68a6d83a7315a99a3ca59871dc3ff840671f99adba", size = 38430, upload-time = "2026-07-06T10:46:31.496Z" }, + { url = "https://files.pythonhosted.org/packages/b2/4f/6a059e8ad3ca8deedc91dfe335b211204900895152212c03ebbe721de68b/xxhash-3.8.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:affb37f152e55b5e4494bb9d0107f7bb08515c6704fbed82d9f61214d74adc17", size = 36558, upload-time = "2026-07-06T10:46:33.078Z" }, + { url = "https://files.pythonhosted.org/packages/cb/95/40be178205acce092ae418feb20ac737b32a02c7b864926ed0717354c9f8/xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:460261045936975193bfd20549a0de1cd52a33b405cbb972f0d80940c42266cd", size = 31181, upload-time = "2026-07-06T10:46:34.793Z" }, + { url = "https://files.pythonhosted.org/packages/3f/89/2da4dbf051bafa156c0e3f12012db2b0ac3b84ff37ca1f021f6bfffcdfbb/xxhash-3.8.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:38c887aedb696ef8bca19983206d270848558cfae4a91afa6a2fb05dde58ffc5", size = 32192, upload-time = "2026-07-06T10:46:36.393Z" }, + { url = "https://files.pythonhosted.org/packages/7c/4e/e000bbae3566bc8e0be771a8a0f294aa99075e3f0bc4ef43922ebffdebc8/xxhash-3.8.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:594131ce1aad18db3689781f806db1b065cdaa04f4df36b4c038d2013aefd0bf", size = 34691, upload-time = "2026-07-06T10:46:38.1Z" }, + { url = "https://files.pythonhosted.org/packages/b4/4a/ea954aacc7d1c8711880ac2b55da94429a9b4296b151c4fc0966549ca1ee/xxhash-3.8.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:78c794b643d214f1522e7a288bcf5a2de120d26cd170516749a4009dc92722c9", size = 34807, upload-time = "2026-07-06T10:46:39.647Z" }, + { url = "https://files.pythonhosted.org/packages/ca/29/df598e738ff37558ac627264deb2e560902d9bf7f46d3bd5175c9eee593e/xxhash-3.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:af0c9fedc4a2c24e8664953882fe8185f3790b8338c9c700f76f5ad660817711", size = 32410, upload-time = "2026-07-06T10:46:41.359Z" }, + { url = "https://files.pythonhosted.org/packages/59/9c/81ab40e7d33ada0b3df5d1bc884894d15dbf4f805cd645b685e4606bb8e0/xxhash-3.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:115772daeb71b2f3b9381177017f53e6cf3f3439c840737fdabd21aba6e54920", size = 220564, upload-time = "2026-07-06T10:46:43.463Z" }, + { url = "https://files.pythonhosted.org/packages/fd/6f/62ae6f5c8606320a0e2a41c2dc8c6d91cc5d63d0f84dd9582e9543779dd8/xxhash-3.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:000435984a0469b0f822fe76f35bddea0f96a4d6521b3339a60a6428cdee1edc", size = 241462, upload-time = "2026-07-06T10:46:45.509Z" }, + { url = "https://files.pythonhosted.org/packages/15/a1/9c3a0ec6cb524396f551eddd102a76690a795494eb9784fc67542b0daa37/xxhash-3.8.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2f1c68394818e0595569c2ff3cbc1e6d5a36a434e796f5c526b987b80c8a8c62", size = 264491, upload-time = "2026-07-06T10:46:47.655Z" }, + { url = "https://files.pythonhosted.org/packages/64/f2/700a4674e4308eb59d2fdb973977e82eae231bea5044753fee5c9eec0e0c/xxhash-3.8.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:46b39976d008e2a845758650f0ff7136bca004f40da0c8798bd37ac37860154f", size = 242905, upload-time = "2026-07-06T10:46:49.857Z" }, + { url = "https://files.pythonhosted.org/packages/f3/8a/72d9874375c8d4cbc64a8cd1d659d5695a8765c3db82efa82dc5bd9f14d0/xxhash-3.8.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d5006c65ec507a333479e76e00e2c368781f16c24ededa764763956b32a0e93e", size = 473873, upload-time = "2026-07-06T10:46:51.953Z" }, + { url = "https://files.pythonhosted.org/packages/03/f0/6db07590ed7e0a77f186ef0bcea8d52553bf1ba57833e09467a2411f0f2d/xxhash-3.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31a2649bcf1fe97cf11c79848d761df33ac46b3896942d31b640557b486ff6b", size = 220765, upload-time = "2026-07-06T10:46:55.41Z" }, + { url = "https://files.pythonhosted.org/packages/8f/10/00d12d8b8beabbf49a8bbc626fb9f40445145a8887eb41a6acfb69149ac4/xxhash-3.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8f759eed402448c2bdbb492e4fba1f20668ffe29688605ea61f0f67f9e4e386d", size = 310478, upload-time = "2026-07-06T10:46:57.729Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f9/12a82394eefb0f185d15a7f7b9f627c61c475a72dd83718436a5b84b42ac/xxhash-3.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7b5f97ecfede10d5b2870383620e2d25c8561e217c7bf9081073802b54248d2b", size = 238393, upload-time = "2026-07-06T10:46:59.87Z" }, + { url = "https://files.pythonhosted.org/packages/20/f3/53f963e320b9ce678337aa7273f39ce692ded8b99e3d22a866ec722159ab/xxhash-3.8.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1da930bbcac3e8fbe2191850e2abb57977a99348c12c4b385e1058ac1b0a9ecc", size = 268704, upload-time = "2026-07-06T10:47:01.806Z" }, + { url = "https://files.pythonhosted.org/packages/0a/50/5b5badbd87c82d9f9b5f58ac74a3f29ef08f6fc387b324b8fd482450b862/xxhash-3.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:747476436f6891b9773374ce8d48edcc8b12cb5b61b67c6fb6289633747d088f", size = 225015, upload-time = "2026-07-06T10:47:03.784Z" }, + { url = "https://files.pythonhosted.org/packages/30/93/3ca68265afe7b4e69435e08a7b6a1d9d0f2a071e889da1f8041ed00fe878/xxhash-3.8.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:4ef09bbc2519a93cd0f95f2ceb5f7b85919dffea643278e02362bf40e3c4bed1", size = 240951, upload-time = "2026-07-06T10:47:05.816Z" }, + { url = "https://files.pythonhosted.org/packages/dd/a6/27e19670c40f46b5e76e11f2f4713d21054804568425d870670e757172ad/xxhash-3.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a5eed9d41995a83f3332b4e3396abb7f433cac584222bd7e305b606d8353861e", size = 300751, upload-time = "2026-07-06T10:47:07.95Z" }, + { url = "https://files.pythonhosted.org/packages/bc/fb/b33e27689959fe7ed2ae0b830af41560d65213943983afa9db3a8d481bce/xxhash-3.8.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:53f3ed9118397074ff63a79b66b7fec1c84c782eecde35c5bc94e420a971c231", size = 443480, upload-time = "2026-07-06T10:47:10Z" }, + { url = "https://files.pythonhosted.org/packages/26/60/0e0d973be5fe280753ef02fbc89349492ad6e903bf1dcb870b668f94b662/xxhash-3.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d247b34bf433c92b41689318fd25d246313cab2275a6a47e2efac178b80d6efe", size = 217657, upload-time = "2026-07-06T10:47:12.196Z" }, + { url = "https://files.pythonhosted.org/packages/ad/68/c9e3ecef4a9a417d464cb5bd200aa12f73192dee677901b9e08e0ad0d1bb/xxhash-3.8.1-cp314-cp314-win32.whl", hash = "sha256:d58ce8b6cfa9c4d2f230557f69caf7c06369e318015d0b19485095bc2c5963ab", size = 32690, upload-time = "2026-07-06T10:47:14.204Z" }, + { url = "https://files.pythonhosted.org/packages/d7/99/e9e44588c0b62837bbec5ba7927816de0afa03406b1a0b6c7a7e1d1a30a0/xxhash-3.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:6cee733fe4ccb1737e0997135283c82341e5cfa9cf214b165f9087fb663aaf4f", size = 33460, upload-time = "2026-07-06T10:47:16.021Z" }, + { url = "https://files.pythonhosted.org/packages/45/2b/64f36d86380b3657ad9031967ab814f3ef31307174650853f69c18932ebc/xxhash-3.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:58346024d47e84f7d8b3e7f5d6faa1d58acbbe49a8771497872059f58c1d8ea5", size = 30092, upload-time = "2026-07-06T10:47:17.81Z" }, + { url = "https://files.pythonhosted.org/packages/92/cb/18b64bff88c58a0ca209dc533e63cf02d7ae5aa6b1b9a9fd14e81b5dbd60/xxhash-3.8.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:01cab782f8a0a05ecad2c63d7ef10f7ab475f660e0d6419d069418c14d88de7c", size = 35024, upload-time = "2026-07-06T10:47:19.821Z" }, + { url = "https://files.pythonhosted.org/packages/af/1d/72d8a70520e5dcddb472ea0486d299da3240745a10658290cd7b5690ede2/xxhash-3.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:717b12fdc51819833704e85e6926d76981ffa3f780ef92e33ebb8b26d46bb230", size = 32697, upload-time = "2026-07-06T10:47:21.649Z" }, + { url = "https://files.pythonhosted.org/packages/9c/b8/e041f555903c56db3d0a731b3d72a6575d75e0ed868b1bd2e5176111ca44/xxhash-3.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ec55d80e9b8a519d742669e0b49e8ce9e6747be42bf3c138158b6543a9c8e489", size = 226044, upload-time = "2026-07-06T10:47:23.612Z" }, + { url = "https://files.pythonhosted.org/packages/3a/7e/5cdcf06bf6ec4b5d2ac073feb23432ec1d603fd438864cbd2c09c7cb45e1/xxhash-3.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98d8ac1129b4dd39098cffed94d1284aceb61c3aa396757ccc736ac392e4cee5", size = 249899, upload-time = "2026-07-06T10:47:25.812Z" }, + { url = "https://files.pythonhosted.org/packages/c0/c0/eb7e059cb5e1dba11fd30d2fdf882f56e5a417a3eaa43669d43623767f45/xxhash-3.8.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3bc0fa90830df1e1277f33cc6e55de9990b83c0319fd8c7412866cfde38b025e", size = 274892, upload-time = "2026-07-06T10:47:27.931Z" }, + { url = "https://files.pythonhosted.org/packages/66/74/a600aaf7cd39957fd1510adeedb1749c1e7eb82bd632a1153d9c664c3135/xxhash-3.8.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c73b6f652f0745425aa6378319c331293b5341756262e9408ed3d45f183375e6", size = 252243, upload-time = "2026-07-06T10:47:30.288Z" }, + { url = "https://files.pythonhosted.org/packages/ad/04/78d88fa75a6763e5d09bf1b947a392a27988903381b219006f92f3c68fc8/xxhash-3.8.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6114692261eff4266386cdec0f7d87eee24e317ab397c218b7ae6a76b4c6339", size = 482191, upload-time = "2026-07-06T10:47:32.45Z" }, + { url = "https://files.pythonhosted.org/packages/7f/06/07a8aea1108d682de8791ce608cdf367d75ff4e7e57cd3c154bdc6f47b23/xxhash-3.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df57c0b161ec1b3ed0526a67b0db0914b557e86ee8aae51887aec941b261542", size = 226877, upload-time = "2026-07-06T10:47:34.705Z" }, + { url = "https://files.pythonhosted.org/packages/ed/b5/86bade5618a524d2c06c4041aa2fe8e5749ce16e88afba60d67c1684a21f/xxhash-3.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9043877a917be88ccf230aa5667c1bd059bce80f4c2727e4defa1b29b7f48b08", size = 319794, upload-time = "2026-07-06T10:47:37.08Z" }, + { url = "https://files.pythonhosted.org/packages/23/69/9b1a2b89b1621bb740fbcb7beb512f60f99480c1bdc680c0c90e1f56ff75/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:559e3cabe522231909f9de98ef06929edbd53782046bd21aae0c72db6f2a0775", size = 246202, upload-time = "2026-07-06T10:47:39.676Z" }, + { url = "https://files.pythonhosted.org/packages/08/ea/662ed6cb49f1d34078b6a3a3e0f3d29ff93fd7b5a03c0bc9ecfd9b2159c3/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:264710bd335016f303763ce1275c6486df30bb57c2245c91b224c983d7ac39b8", size = 275628, upload-time = "2026-07-06T10:47:41.99Z" }, + { url = "https://files.pythonhosted.org/packages/13/f5/49fc9e4c6728a5a3bd8fe639199d2fa67609b3a84f938aff6e8568dd3e4f/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e14800b9b10bb39d7a60ad4a310e403164d7b8988a27ae933d4e40618a44088e", size = 231390, upload-time = "2026-07-06T10:47:44.233Z" }, + { url = "https://files.pythonhosted.org/packages/64/9d/3acaf8f599c0e0b30e910a3a11ba32929da53c86dc73c7c55fe6a010b4e9/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:ea6a3e734b0fd41b82784a400be946821900daebe610c050a5e0760838a34f99", size = 250600, upload-time = "2026-07-06T10:47:47.611Z" }, + { url = "https://files.pythonhosted.org/packages/23/64/8acab4c5ec60dbe664b5b9858fd44c2413b07e535b09556a0a5022e78aa6/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cf399fac542a1c7a4734a435b93df2c55e858c7d31abf6c1bdf46f9ae67fbfd0", size = 310032, upload-time = "2026-07-06T10:47:49.88Z" }, + { url = "https://files.pythonhosted.org/packages/56/47/a0288d7329b1fe63e2734a32d19d444a96ae2b4810f545bc61e561224917/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:44c89d915a75c11d2547eaee9098fcd80398987c4bff2974a0497a925bf92c07", size = 448882, upload-time = "2026-07-06T10:47:52.631Z" }, + { url = "https://files.pythonhosted.org/packages/01/e7/3071dfd3beb5c38204ce1cf56bf7749fce08de900fa92714b81d1d8ca1f2/xxhash-3.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:358650d5bda9c635da699c53adf4e8134af492ecc79c960f917eebf088bb6799", size = 223728, upload-time = "2026-07-06T10:47:55.093Z" }, + { url = "https://files.pythonhosted.org/packages/12/11/b99949f0ba2b07e9f9ffe83b9c86faa685f9080725dc21a916a607313be5/xxhash-3.8.1-cp314-cp314t-win32.whl", hash = "sha256:c240939e963653054fc7e4a17c382829cda4aa88a7daf0af841715dbded1b497", size = 33150, upload-time = "2026-07-06T10:47:57.274Z" }, + { url = "https://files.pythonhosted.org/packages/54/1c/09703eb341f8416e74e58d6c6732d4b5c46de59c942363203cb237cc95b0/xxhash-3.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:7258ee276e8772599bc19e14b36f6260306e21b637190cd7cb489a2449d48684", size = 34005, upload-time = "2026-07-06T10:47:59.434Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f9/6ed7251bb6a8af10ac73b1821c60583d2826e5b2064e45a979c935287c98/xxhash-3.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:8f454166c2ffed45636c8d501741e649851ba2f346c4eb73a64c07ac00428f20", size = 30239, upload-time = "2026-07-06T10:48:01.874Z" }, + { url = "https://files.pythonhosted.org/packages/99/e4/4d8040435aeac814fc69ba63621565fbeb19229a138e2568324a26b2a45c/xxhash-3.8.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:39c9d5b61508b0bb68f29e54546de0ed2a74943c6a18585535a7e37356f1dd12", size = 32687, upload-time = "2026-07-06T10:49:42.803Z" }, + { url = "https://files.pythonhosted.org/packages/da/6a/975f1f2318c760e5bcec109ed379713ae645d8d856c2a3b9ec5d26857087/xxhash-3.8.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:83b9130b80b216d56fdf9e87131946b353c9627930c061955a101ea82b09fed9", size = 29879, upload-time = "2026-07-06T10:49:45.172Z" }, + { url = "https://files.pythonhosted.org/packages/08/0b/40a2a55ff52cf635bfdc5eae67a772bec85b4f44c6c737f73f6f528d51d1/xxhash-3.8.1-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8304be0982130954b7fd3aad18e2c6f8ee40254bc3d2e635991c16d77c91e2bd", size = 43246, upload-time = "2026-07-06T10:49:47.905Z" }, + { url = "https://files.pythonhosted.org/packages/9c/6d/56ed2b6b200f26fb474f3fd387d95d0601efcd5bb33430c90c68924bdd77/xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b512261801b1e5fde7b6ebf2fef7977339c620cbbca88a0040ad9ad134f4d02", size = 38202, upload-time = "2026-07-06T10:49:50.59Z" }, + { url = "https://files.pythonhosted.org/packages/0d/a3/56864d895d1161a9f17502088e9c1fb7c06bde2c2efdde620d22bb7a9c43/xxhash-3.8.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49aa8692507835dcc1e8ad8021f20c74c2dc13d83b5112e87877faa2a0035b20", size = 34448, upload-time = "2026-07-06T10:49:53.242Z" }, + { url = "https://files.pythonhosted.org/packages/6b/57/5c6e0908a47f61dca96d01c8ee6fce01ed1050611eb779083ba8758fed81/xxhash-3.8.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:345b07b78e2bf583d71682aa34ae5b5fab575f7a1cb31e10263ebbc6f89f8c42", size = 32869, upload-time = "2026-07-06T10:49:55.972Z" }, +] + +[[package]] +name = "yarl" +version = "1.24.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "multidict" }, + { name = "propcache" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/31/33/ebe9e3d1f86c7a0b51094c0a146392045ca1631d2664889539dec8088a33/yarl-1.24.5.tar.gz", hash = "sha256:e81b83143bee16329c23db3c1b2d82b29892fcbcb849186d2f6e98a5abe9a57f", size = 228679, upload-time = "2026-07-20T02:07:45.435Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/db/3cb5df059756a45761cc3dee8fd25ec82b83a6585ea3542b969fda850f99/yarl-1.24.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2c1fe720934a16ea8e7146175cba2126f87f54912c8c5435e7f7c7a51ef808d3", size = 135043, upload-time = "2026-07-20T02:04:52.39Z" }, + { url = "https://files.pythonhosted.org/packages/44/f8/767d6bd5a03db63bc467df2fb56d6fafeae9667d74aea92cd6af399f828b/yarl-1.24.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c687ed078e145f5fd53a14854beff320e1d2ab76df03e2009c98f39a0f68f39a", size = 96942, upload-time = "2026-07-20T02:04:54.26Z" }, + { url = "https://files.pythonhosted.org/packages/ce/97/10b939c44d7b28d1dbc389cfc7012306d1ea8dba01eaef44b39fffaee52a/yarl-1.24.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:709f1efed56c4a145793c046cd4939f9959bcd818979a787b77d8e09c57a0840", size = 97046, upload-time = "2026-07-20T02:04:56.638Z" }, + { url = "https://files.pythonhosted.org/packages/5b/7a/b410dbe39b6255c55fb2a2bcee96eb844d0789235ddc381a889a90dc72d6/yarl-1.24.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:874019bd513008b009f58657134e5d0c5e030b3559bd0553976837adf52fe966", size = 110512, upload-time = "2026-07-20T02:04:58.955Z" }, + { url = "https://files.pythonhosted.org/packages/83/c7/da591971f78a5617e1f21f5699858ebccd836fe181a6493788ffc91ba69b/yarl-1.24.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a4582acf7ef76482f6f511ebaf1946dae7f2e85ec4728b81a678c01df63bd723", size = 102454, upload-time = "2026-07-20T02:05:00.623Z" }, + { url = "https://files.pythonhosted.org/packages/c4/8e/73b0ed4de47289a78a96045d76d1cfe5e41848bf0da59ce25b2ec87ee05d/yarl-1.24.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2cabe6546e41dabe439999a23fcb5246e0c3b595b4315b96ef755252be90caeb", size = 117617, upload-time = "2026-07-20T02:05:02.325Z" }, + { url = "https://files.pythonhosted.org/packages/cf/14/b744747bc4f57a8d55bd744df463457524583e1e9f7538b5ace0346ab92e/yarl-1.24.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:17f57620f5475b3c69109376cc87e42a7af5db13c9398e4292772a706ff10780", size = 116135, upload-time = "2026-07-20T02:05:04.05Z" }, + { url = "https://files.pythonhosted.org/packages/66/ca/95aa4d0e5b7ea4f20e4d577c42d001ed9df207569fdb063cc5ed4ebb496b/yarl-1.24.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:570fec8fbd22b032733625f03f10b7ff023bc399213db15e72a7acaef28c2f4e", size = 111935, upload-time = "2026-07-20T02:05:05.738Z" }, + { url = "https://files.pythonhosted.org/packages/72/0d/d2ad8d6b147832d177a4e720ba1962fe686eb0913b74503b3eca094b8bba/yarl-1.24.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5fede79c6f73ff2c3ef822864cb1ada23196e62756df53bc6231d351a49516a2", size = 110010, upload-time = "2026-07-20T02:05:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/50/18/eb335e4120903903f4865041355ae46256a2406eb2865bc24827f4f27b61/yarl-1.24.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8ccf9aca873b767977c73df497a85dbedee4ee086ae9ae49dc461333b9b79f58", size = 110058, upload-time = "2026-07-20T02:05:09.246Z" }, + { url = "https://files.pythonhosted.org/packages/44/70/97353add32c62ad6f206d948ac5a5ee84398225e534dc6ed6433d1b335b6/yarl-1.24.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:ad5d8201d310b031e6cd839d9bac2d4e5a01533ce5d3d5b50b7de1ef3af1de61", size = 103308, upload-time = "2026-07-20T02:05:11.31Z" }, + { url = "https://files.pythonhosted.org/packages/68/39/5e7398d4b6f6b3c9062823ebc60802df5b272e3fe9e788f9734c6ee46c85/yarl-1.24.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:841f0852f48fefea3b12c9dfec00704dfa3aef5215d0e3ce564bb3d7cd8d57c6", size = 116898, upload-time = "2026-07-20T02:05:13.099Z" }, + { url = "https://files.pythonhosted.org/packages/e4/c9/09e52f2239e8b96357eccca05915382e4ba5405ebfb623b6036040d99654/yarl-1.24.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:9baafc71b04f8f4bb0703b21d6fc9f0c30b346c636a532ff16ec8491a5ea4b1f", size = 109400, upload-time = "2026-07-20T02:05:14.821Z" }, + { url = "https://files.pythonhosted.org/packages/4b/6a/e94133d4c2d1a14d2384310bf3e79d9cf32c9d1eae1c6f034fb80d098fa1/yarl-1.24.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d897129df1a22b12aeed2c2c98df0785a2e8e6e0bde87b389491d0025c187077", size = 115934, upload-time = "2026-07-20T02:05:17.78Z" }, + { url = "https://files.pythonhosted.org/packages/4e/3c/34955ed967b976fc38edcbb6d538dee79dbda4cb7fc7f72a0907a7c78e0f/yarl-1.24.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:dd625535328fd9882374356269227670189adfcc6a2d90284f323c05862eecbd", size = 112178, upload-time = "2026-07-20T02:05:19.675Z" }, + { url = "https://files.pythonhosted.org/packages/f5/46/d7bd3a8859d47dcfaffd7127af7076032a7da278a9a02e17b5f37bfb6712/yarl-1.24.5-cp311-cp311-win_amd64.whl", hash = "sha256:f4239bbec5a3577ddb49e4b50aeb32d8e5792098262ae2f63723f916a29b1a25", size = 97544, upload-time = "2026-07-20T02:05:21.523Z" }, + { url = "https://files.pythonhosted.org/packages/01/69/c1bfd21e32c638974ea2c542a0b8c53ef1fa9eff336020f5d014f9503ff2/yarl-1.24.5-cp311-cp311-win_arm64.whl", hash = "sha256:3ac6aff147deb9c09461b2d4bbdf6256831198f5d8a23f5d37138213090b6d8a", size = 93359, upload-time = "2026-07-20T02:05:23.493Z" }, + { url = "https://files.pythonhosted.org/packages/1b/84/71d051c850b5af41d168c679d9eb67eb7c55283ac4ee131673edf134bc4e/yarl-1.24.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d693396e5aea78db03decd60aec9ece16c9b40ba00a587f089615ff4e718a81d", size = 136035, upload-time = "2026-07-20T02:05:25.489Z" }, + { url = "https://files.pythonhosted.org/packages/03/4d/8ad27f9a1b7e69313cca5d695b925b48efe51208d3490e0844bae97cabc0/yarl-1.24.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3363fcc96e665878946ad7a106b9a13eac0541766a690ef287c0232ac768b6ec", size = 97642, upload-time = "2026-07-20T02:05:27.429Z" }, + { url = "https://files.pythonhosted.org/packages/ea/b4/05b4131c407006cd1e410e9c6539f16a0945724677e5364447313c15ea3e/yarl-1.24.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9d399bdcfb4a0f659b9b3788bbc89babe63d9a6a65aacdf4d4e7065ff2e6316c", size = 97323, upload-time = "2026-07-20T02:05:29.441Z" }, + { url = "https://files.pythonhosted.org/packages/20/16/e618c875c73e0e39611f20a581b3d5e8d59b8857bf001bee3263044c6deb/yarl-1.24.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:90333fd89b43c0d08ac85f3f1447593fc2c66de18c3d6378d7125ea118dc7a54", size = 107741, upload-time = "2026-07-20T02:05:31.367Z" }, + { url = "https://files.pythonhosted.org/packages/d9/9a/c4defeaf3ed33fcb346aacf9c6e971a8d4e2bde04a0310e79abb208e7965/yarl-1.24.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:665b0a2c463cc9423dd647e0bfd9f4ccc9b50f768c55304d5e9f80b177c1de12", size = 103570, upload-time = "2026-07-20T02:05:33.303Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e7/0e0e0de5865ebd5914537ef486f36c727a59865c3ac0cf5ff1b32aececbf/yarl-1.24.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e006d3a974c4ee19512e5f058abedb6eef36a5e553c14812bdeba1758d812e6d", size = 115815, upload-time = "2026-07-20T02:05:35.292Z" }, + { url = "https://files.pythonhosted.org/packages/2b/27/ca56b700cb170aba25a3893b75355b213935657dc5714d2383354a270e62/yarl-1.24.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e7d42c531243450ef0d4d9c172e7ed6ef052640f195629065041b5add4e058d1", size = 116025, upload-time = "2026-07-20T02:05:37.503Z" }, + { url = "https://files.pythonhosted.org/packages/d6/d0/d56c859b8222116f5d68459199f48359e0bf121b6f65a69bf329b3602ba0/yarl-1.24.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f08c7513ecef5aad65687bfdf6bc601ae9fccd04a42904501f8f7141abad9eb9", size = 109835, upload-time = "2026-07-20T02:05:39.506Z" }, + { url = "https://files.pythonhosted.org/packages/70/a2/3a35557e4d1a79425040eba202ccaf08bdc8717680fc77e2498a1ad2e0a5/yarl-1.24.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6c95b17fe34ed802f17e205112e6e10db92275c34fee290aa9bdc55a9c724027", size = 108884, upload-time = "2026-07-20T02:05:41.584Z" }, + { url = "https://files.pythonhosted.org/packages/e4/35/ef4c26356b7913c68983bac2d72a4212b3347af551cb8d250b99b5ed7b7f/yarl-1.24.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:56b149b22de33b23b0c6077ab9518c6dcb538ad462e1830e68d06591ccf6e38b", size = 107308, upload-time = "2026-07-20T02:05:43.697Z" }, + { url = "https://files.pythonhosted.org/packages/d5/91/ff0dc66c2ccf3e0153ab97ff61eabab4400e6a5264af427ab30cd69f1857/yarl-1.24.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:a8fe66b8f300da93798025a785a5b90b42f3810dc2b72283ff84a41aaaebc293", size = 103646, upload-time = "2026-07-20T02:05:45.895Z" }, + { url = "https://files.pythonhosted.org/packages/74/f0/33b9271c7f881766359d58266fa0811d2e5210ed860e28da7dc6d7786344/yarl-1.24.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:377fe3732edbaf78ee74efdf2c9f49f6e99f20e7f9d2649fda3eb4badd77d76e", size = 115305, upload-time = "2026-07-20T02:05:47.832Z" }, + { url = "https://files.pythonhosted.org/packages/ef/65/fd79fb1868c4a80db8661091de525bf430f63c3bea1b20e8b6a84fc7d359/yarl-1.24.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e8ffa78582120024f476a611d7befc123cee59e47e8309d470cf667d806e613b", size = 108404, upload-time = "2026-07-20T02:05:49.604Z" }, + { url = "https://files.pythonhosted.org/packages/ff/ba/dbabe6b262f17a816c70cfc09558dbf03ece3ec76684d02f911a3d3a189c/yarl-1.24.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:daba5e594f06114e37db186efd2dd916609071e59daca901a0a2e71f02b142ce", size = 115940, upload-time = "2026-07-20T02:05:51.741Z" }, + { url = "https://files.pythonhosted.org/packages/a5/43/fab2d1dad9d340a268cdde63756a123d069723efff6a372d123fa74a9517/yarl-1.24.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:65be18ec59496c13908f02a2472751d9ef840b4f3fb5726f129306bf6a2a7bba", size = 110006, upload-time = "2026-07-20T02:05:53.554Z" }, + { url = "https://files.pythonhosted.org/packages/c4/27/41eb51bbd1b8d89546b83897cfb0164f1e109304fd408dbb151b639eec0f/yarl-1.24.5-cp312-cp312-win_amd64.whl", hash = "sha256:a929d878fec099030c292803b31e5d5540a7b6a31e6a3cc76cb4685fc2a2f51b", size = 97618, upload-time = "2026-07-20T02:05:55.57Z" }, + { url = "https://files.pythonhosted.org/packages/3c/25/b2553764b3d65db711d8f45416351ec4f420847558eb669edcbcaadf5780/yarl-1.24.5-cp312-cp312-win_arm64.whl", hash = "sha256:7ce27823052e2013b597e0c738b13e7e36b8ccb9400df8959417b052ab0fd92c", size = 93018, upload-time = "2026-07-20T02:05:57.554Z" }, + { url = "https://files.pythonhosted.org/packages/e1/63/64ef361967cc983573149dc1515d531db5da8a4c92d22bb833d59e01b313/yarl-1.24.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:79af890482fc94648e8cde4c68620378f7fef60932710fa17a66abc039244da2", size = 135075, upload-time = "2026-07-20T02:05:59.671Z" }, + { url = "https://files.pythonhosted.org/packages/bb/89/55920fd853ce43e608adbc3962456f0d649d6bb15250dc2988321da0fe1c/yarl-1.24.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:46c2f213e23a04b93a392942d782eb9e413e6ef6bf7c8c53884e599a5c174dcb", size = 97225, upload-time = "2026-07-20T02:06:01.769Z" }, + { url = "https://files.pythonhosted.org/packages/15/f0/7688d3f2cfff7590df2af38ec46d969f4281a4dddb08a9ad2eafbcdddf98/yarl-1.24.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92ab3e11448f2ff7bf53c5a26eff0edc086898ec8b21fb154b85839ce1d88075", size = 96751, upload-time = "2026-07-20T02:06:03.676Z" }, + { url = "https://files.pythonhosted.org/packages/05/1a/a851a0f94aaaf379dd4f901bfc80f634280bec51eb260b47363e2a4cd62e/yarl-1.24.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ebb0ec7f17803063d5aeb982f3b1bd2b2f4e4fae6751226cbd6ba1fcfe9e63ff", size = 107960, upload-time = "2026-07-20T02:06:05.699Z" }, + { url = "https://files.pythonhosted.org/packages/6c/a8/faea066c12f9c77ca0de90641f1655f9dd7b412477bf28c76d692f3aecff/yarl-1.24.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:82632daed195dcc8ea664e8556dc9bdbd671960fb3776bd92806ce05792c2448", size = 103500, upload-time = "2026-07-20T02:06:07.556Z" }, + { url = "https://files.pythonhosted.org/packages/fb/9c/1e67084c2a6e2f2db0e3be798328cb3be42c0119b621d25461479a224d21/yarl-1.24.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:53e549287ef628fecba270045c9701b0c564563a9b0577d24a4ec75b8ab8040f", size = 115780, upload-time = "2026-07-20T02:06:09.599Z" }, + { url = "https://files.pythonhosted.org/packages/58/86/1f94664e147474337e3359f52012cf3d02f825f694317b178bfba1078c62/yarl-1.24.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fcd3b77e2f17bbe4ca56ec7bcb07992647d19d0b9c05d84886dcd6f9eb810afd", size = 115308, upload-time = "2026-07-20T02:06:11.352Z" }, + { url = "https://files.pythonhosted.org/packages/0a/43/8e55ae7538ba5f28ccb3c845c6dd4549cf7016d5992e5326512519107cdd/yarl-1.24.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d46b86567dd4e248c6c159fcbcdcce01e0a5c8a7cd2334a0fff759d0fa075b16", size = 110574, upload-time = "2026-07-20T02:06:13.129Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ba/a889ec8765cedcf2ac44dcb02d6a21e4861399b243b263c5f2dde27ee740/yarl-1.24.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7f72c74aa99359e27a2ee8d6613fefa28b5f76a983c083074dfc2aaa4ab46213", size = 109914, upload-time = "2026-07-20T02:06:15.243Z" }, + { url = "https://files.pythonhosted.org/packages/9c/c3/e45f821af67b791c2dbbe4a9f4137a1d33f8d386654a05a0c3f47bdfa25d/yarl-1.24.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f45789ce415a7ec0820dc4f82925f9b5f7732070be1dec1f5f23ec381435a24", size = 107712, upload-time = "2026-07-20T02:06:17.443Z" }, + { url = "https://files.pythonhosted.org/packages/02/00/2ab0f42c9857fcb490bfaa6647b14540b53d241ab209f23220b958cc5832/yarl-1.24.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:6e73e7fe93f17a7b191f52ec9da9dd8c06a8fe735a1ecbd13b97d1c723bff385", size = 104251, upload-time = "2026-07-20T02:06:19.259Z" }, + { url = "https://files.pythonhosted.org/packages/7a/70/709d9a286e98af2c7fd8e4e6cada658b5c0e30d87dd7e2a63c2fb5767217/yarl-1.24.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4a36f9becdd4c5c52a20c3e9484128b070b1dcfc8944c006f3a528295a359a9c", size = 115319, upload-time = "2026-07-20T02:06:21.207Z" }, + { url = "https://files.pythonhosted.org/packages/5c/6c/3eaa515142991fe84cfc483ff986492211f1978f90161ccefdbec919d09b/yarl-1.24.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:7bcbe0fcf850eae67b6b01749815a4f7161c560a844c769ad7b48fcd99f791c4", size = 109163, upload-time = "2026-07-20T02:06:23.006Z" }, + { url = "https://files.pythonhosted.org/packages/bb/64/711dafce66c323a3144d470547a71c5384c57623308ac8bb5e4b903ac148/yarl-1.24.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:24e861e9630e0daddcb9191fb187f60f034e17a4426f8101279f0c475cd74144", size = 115435, upload-time = "2026-07-20T02:06:24.923Z" }, + { url = "https://files.pythonhosted.org/packages/cf/f3/9b9d0e6d84bea851eb1ba99e4bdc755b86fd813e49ec86dfe42f26befdef/yarl-1.24.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9335a099ad87287c37fe5d1a982ff392fa5efe5d14b40a730b1ec1d6a41382b4", size = 110691, upload-time = "2026-07-20T02:06:26.973Z" }, + { url = "https://files.pythonhosted.org/packages/86/e4/62a06b7e87c4246ac76b7c2da136f972eb4a3a1fc94abb07e7022d6fdb0a/yarl-1.24.5-cp313-cp313-win_amd64.whl", hash = "sha256:2dbe06fc16bc91502bca713704022182e5729861ae00277c3a23354b40929740", size = 97454, upload-time = "2026-07-20T02:06:29.163Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c9/5fc8025b318ab10db413b61056bd0d95c557a70e8df4210c7511f866329c/yarl-1.24.5-cp313-cp313-win_arm64.whl", hash = "sha256:6b8536851f9f65e7f00c7a1d49ba7f2be0ffe2c11555367fc9f50d9f842410a1", size = 92813, upload-time = "2026-07-20T02:06:31.113Z" }, + { url = "https://files.pythonhosted.org/packages/a9/08/5f3085fef9564217074db9dd8573de1795bc82cde61a7ad10b6a7234a569/yarl-1.24.5-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2729fcfc4f6a596fb0c50f32090400aa9367774ac296a00387e65098c0befa76", size = 135680, upload-time = "2026-07-20T02:06:33.273Z" }, + { url = "https://files.pythonhosted.org/packages/98/35/ba9436e579bd48a8801f2021d842d9ab4994c26e4c7dd3a4c1f1bcb57a9e/yarl-1.24.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff330d3c30db4eb6b01d79e29d2d0b407a7ecad39cfd9ec993ece57396a2ec0d", size = 97395, upload-time = "2026-07-20T02:06:35.259Z" }, + { url = "https://files.pythonhosted.org/packages/18/a9/a07f76f3c44e02b25cc743af5ef93eef27f7013eadca770451b6a6ccb5db/yarl-1.24.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e42d75862735da90e7fc5a7b23db0c976f737113a54b3c9777a9b665e9cbff75", size = 97223, upload-time = "2026-07-20T02:06:37.216Z" }, + { url = "https://files.pythonhosted.org/packages/77/f7/a9a1d6fa7dd9e388f95b30f6ad3ec4e285f6c8f61f44ce16070c3fcfe414/yarl-1.24.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a3732e66413163e72508da9eff9ce9d2846fde51fae45d3605393d3e6cd303e9", size = 108777, upload-time = "2026-07-20T02:06:39.292Z" }, + { url = "https://files.pythonhosted.org/packages/2f/44/e0b86c302471fabd6f02808ecf2ac52b8412b624787849d4bf2cdb466f6f/yarl-1.24.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5b8ee53be440a0cffc991a27be3057e0530122548dbe7c0892df08822fce5ede", size = 103119, upload-time = "2026-07-20T02:06:41.456Z" }, + { url = "https://files.pythonhosted.org/packages/d1/16/9c16d180bf8faaf223225eb50e1245870ff1ae0e302a27153988e65c51fd/yarl-1.24.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:af3aefa655adb5869491fa907e652290386800ae99cc50095cba71e2c6aefdca", size = 116471, upload-time = "2026-07-20T02:06:43.696Z" }, + { url = "https://files.pythonhosted.org/packages/d2/8d/b219b9df28a02ce95cfbdd41d2f7caa5669d0ff979c1c9975697145e33c5/yarl-1.24.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2120b96872df4a117cde97d270bac96aea7cc52205d305cf4611df694a487027", size = 115974, upload-time = "2026-07-20T02:06:45.874Z" }, + { url = "https://files.pythonhosted.org/packages/9b/e8/f20557aca240d88e69850ad1ee91756821d094bb1310565c04d25c6682a2/yarl-1.24.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66410eb6345d467151934b49bfa70fb32f5b35a6140baa40ad97d6436abea2e9", size = 110830, upload-time = "2026-07-20T02:06:47.852Z" }, + { url = "https://files.pythonhosted.org/packages/db/18/199b85109a53eeca64ee19c9cca228287e8e4ab0cc1a09b28f530e65cce0/yarl-1.24.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4af7b7e1be0a69bee8210735fe6dcfc38879adfac6d62e789d53ba432d1ffa41", size = 110054, upload-time = "2026-07-20T02:06:49.84Z" }, + { url = "https://files.pythonhosted.org/packages/aa/2f/ed28147f8cd7f48c49367c90713b30a555284b6105a6a56f3a05568da795/yarl-1.24.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa139875ff98ab97da323cfadfaff08900d1ad42f1b5087b0b812a55c5a06373", size = 108312, upload-time = "2026-07-20T02:06:51.835Z" }, + { url = "https://files.pythonhosted.org/packages/c5/c5/55e16ae0a5c227cea8df1c6871ba57d614a34243146c05729caf2a1bd9c5/yarl-1.24.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:0055afc45e864b92729ac7600e2d102c17bef060647e74bca75fa84d66b9ff36", size = 103662, upload-time = "2026-07-20T02:06:54.061Z" }, + { url = "https://files.pythonhosted.org/packages/8d/ea/dbd7c2caec459c9a426f18b02688ecbfb58620d0f6a3422d24769fbaf8ab/yarl-1.24.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f0e466ed7511fe9d459a819edbc6c2585c0b6eabde9fa8a8947552468a7a6ef0", size = 116090, upload-time = "2026-07-20T02:06:56.015Z" }, + { url = "https://files.pythonhosted.org/packages/06/84/39ce4ce3059e07fece5fbdbee8c4053406af9aca911ce9fa5f8548aab6af/yarl-1.24.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f141474e85b7e54998ec5180530a7cda99ab29e282fa50e0756d89981a9b43c5", size = 109523, upload-time = "2026-07-20T02:06:57.926Z" }, + { url = "https://files.pythonhosted.org/packages/a9/8b/71ff44137b405c64a7788075669c24010019f57a7464b78c3a6cbee539d9/yarl-1.24.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:e2935f8c39e3b03e83519292d78f075189978f3f4adc15a78144c7c8e2a1cba5", size = 116084, upload-time = "2026-07-20T02:06:59.868Z" }, + { url = "https://files.pythonhosted.org/packages/62/c0/423078fdd4042e1862c11f0ffd977a0ffa393783c12bee94685923bc189e/yarl-1.24.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9d1216a7f6f77836617dba35687c5b78a4170afc3c3f18fc788f785ba26565c4", size = 111006, upload-time = "2026-07-20T02:07:01.907Z" }, + { url = "https://files.pythonhosted.org/packages/cf/52/6daa2ee9d95e5c98b8128f8df91eb692eb423ab274b8cf08db52152fad26/yarl-1.24.5-cp314-cp314-win_amd64.whl", hash = "sha256:5ba4f78df2bcc19f764a4b26a8a4f5049c110090ad5825993aacb052bf8003ad", size = 99215, upload-time = "2026-07-20T02:07:03.852Z" }, + { url = "https://files.pythonhosted.org/packages/ec/0e/464a847d7359e0da75dd9fc5c1d1aa35d0159ea31e5f8e66a3c1c29ff3d0/yarl-1.24.5-cp314-cp314-win_arm64.whl", hash = "sha256:9e4e16c73d717c5cf27626c524d0a2e261ad20e46932b2670f64ad5dde23e26f", size = 94566, upload-time = "2026-07-20T02:07:06.074Z" }, + { url = "https://files.pythonhosted.org/packages/e2/55/e03acc4446772660bc335e86e41ef31e4d0d838fd641531a11a5ee33b493/yarl-1.24.5-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:e1ae548a9d901adca07899a4147a7c826bbcc06239d3ce9a59f57886a28a4c88", size = 142533, upload-time = "2026-07-20T02:07:08.284Z" }, + { url = "https://files.pythonhosted.org/packages/ae/71/4acd3a1fc7cf14345cdb302665ecd2097f62c365b4f14ca17d4f37775cf9/yarl-1.24.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff405d91509d88e8d44129cd87b18d70acd1f0c1aeabd7bc3c46792b1fe2acba", size = 100776, upload-time = "2026-07-20T02:07:10.197Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0b/cfb76b7fe99686db264bff829779a539d923e7564ffd7ef18da6c54c3774/yarl-1.24.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:47e98aab9d8d82ff682e7b0b5dded33bf138a32b817fcf7fa3b27b2d7c412928", size = 100913, upload-time = "2026-07-20T02:07:12.357Z" }, + { url = "https://files.pythonhosted.org/packages/8b/3f/7116e782992abbd4fb6948488aec72078895e929a23078290739e8396fce/yarl-1.24.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0a658a6d3fafee5c6f63c58f3e785c8c43c93fbc02bf9f2b6663f8185e0971f", size = 106507, upload-time = "2026-07-20T02:07:14.173Z" }, + { url = "https://files.pythonhosted.org/packages/33/90/d4d2d73ee78229cc889872eb8e085d8f5c6f51abdb178409fd9b23cf74fd/yarl-1.24.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4377407001ca3c057773f44d8ddd6358fa5f691407c1ba92210bd3cf8d9e4c95", size = 99219, upload-time = "2026-07-20T02:07:16.019Z" }, + { url = "https://files.pythonhosted.org/packages/3e/fa/a6df1a9bccd644eec00abee0dff4277416222cec435330fd1f2858523ec1/yarl-1.24.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7c0494a31a1ac5461a226e7947a9c9b78c44e1dc7185164fa7e9651557a5d9bc", size = 111804, upload-time = "2026-07-20T02:07:18.141Z" }, + { url = "https://files.pythonhosted.org/packages/8a/9e/7b2a1f4bcc20e9447156dd2b1c4d01f70d9df0759025ee7d09a84ffae134/yarl-1.24.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a7cff474ab7cd149765bb784cf6d78b32e18e20473fb7bda860bce98ab58e9da", size = 110943, upload-time = "2026-07-20T02:07:20.06Z" }, + { url = "https://files.pythonhosted.org/packages/08/ff/22c92affb0f9b623ca753d27d968b5625b868f12c6378d049d55ae247643/yarl-1.24.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbb833ccacdb5519eff9b8b71ee618cc2801c878e77e288775d77c3a2ced858a", size = 108251, upload-time = "2026-07-20T02:07:22.217Z" }, + { url = "https://files.pythonhosted.org/packages/45/44/5769b96298c1e195fb412997b6090af2a84105cf59c17613558a2d011d1f/yarl-1.24.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:82f75e05912e84b7a0fe57075d9c59de3cb352b928330f2eb69b2e1f54c3e1f0", size = 106025, upload-time = "2026-07-20T02:07:24.083Z" }, + { url = "https://files.pythonhosted.org/packages/4c/40/009e8e791fd9762c0e1567e69248acb4f49064597e1680874c16dd8bb798/yarl-1.24.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:16a2f5010280020e90f5330257e6944bc33e73593b136cc5a241e6c1dc292498", size = 106573, upload-time = "2026-07-20T02:07:26.248Z" }, + { url = "https://files.pythonhosted.org/packages/20/c6/b7480578f8a0a80946f36ad6df547ecec704f9ba69d2de60f8aa6f1c1cbf/yarl-1.24.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:ffcd54362564dc1a30fb74d8b8a6e5a6b11ebd5e27266adc3b7427a21a6c9104", size = 100751, upload-time = "2026-07-20T02:07:28.098Z" }, + { url = "https://files.pythonhosted.org/packages/d4/27/4476f3360b91a48c5cf125e91f59a3bd35299d84a431a258d57f5977bb11/yarl-1.24.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0465ec8cedc2349b97a6b595ace64084a50c6e839eca40aa0626f38b8350e331", size = 111643, upload-time = "2026-07-20T02:07:30.88Z" }, + { url = "https://files.pythonhosted.org/packages/4c/4b/5cdd3e5ee944e8af31e52f6cd3d3af5fd7b937e036ccbbba2c9ffebede95/yarl-1.24.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4db9aecb141cb7a5447171b57aa1ed3a8fee06af40b992ffc31206c0b0121550", size = 106312, upload-time = "2026-07-20T02:07:33.06Z" }, + { url = "https://files.pythonhosted.org/packages/18/86/f406b0c2a6f99575de2da671ef47aa06f89a5be83a27a46971c3b86cecdb/yarl-1.24.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:f540c013589084679a6c7fac07096b10159737918174f5dfc5e11bf5bca4dfe6", size = 110379, upload-time = "2026-07-20T02:07:35.155Z" }, + { url = "https://files.pythonhosted.org/packages/f0/6c/9f3adfbd3b30b4fa0f7ccb3a83eba2c1152d3fff554d535e640ba0f7ba2b/yarl-1.24.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a61834fb15d81322d872eaafd333838ae7c9cea84067f232656f75965933d047", size = 108497, upload-time = "2026-07-20T02:07:37.35Z" }, + { url = "https://files.pythonhosted.org/packages/dd/37/91eb2e5ca883a529c1b390348a74cd9fc0512171727f547ce70bfe02be5c/yarl-1.24.5-cp314-cp314t-win_amd64.whl", hash = "sha256:5c88e5815a49d289e599f3513aa7fde0bc2092ff188f99c940f007f90f53d104", size = 102450, upload-time = "2026-07-20T02:07:39.578Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f4/ed5c402ac8fde4403ed3366c2716bfddc8a6677ebd59f3d62772cc7fe468/yarl-1.24.5-cp314-cp314t-win_arm64.whl", hash = "sha256:cf139c02f5f23ef6532040a30ff662c00a318c952334f211046b8e60b7f17688", size = 97222, upload-time = "2026-07-20T02:07:41.55Z" }, + { url = "https://files.pythonhosted.org/packages/61/02/962c1cbfc401a30c1d034dc67ff395f64b52302c6d62de556c1fca99acc0/yarl-1.24.5-py3-none-any.whl", hash = "sha256:a33700d13d9b7d84fd10947b09ff69fb9a792e519c8cb9764a3ca70baa6c23a7", size = 58612, upload-time = "2026-07-20T02:07:43.461Z" }, +] From 447295e5f705a74a85ad74a74b68985914096357 Mon Sep 17 00:00:00 2001 From: HiddenTrojan Date: Fri, 14 Aug 2026 23:58:12 +0800 Subject: [PATCH 16/24] Repair Experiment 013 RULER phase boundary --- ...EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md | 114 ++++++++++- scripts/capture_static_q468_identity_input.py | 141 ++++++++++---- .../generate_static_q468_ruler_receipts.py | 128 +++++++++++- scripts/resolve_static_q468_identity.py | 15 +- ...test_capture_static_q468_identity_input.py | 182 ++++++++++++++++-- ...est_generate_static_q468_ruler_receipts.py | 109 ++++++++++- tests/test_resolve_static_q468_identity.py | 18 +- 7 files changed, 627 insertions(+), 80 deletions(-) diff --git a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md index 12226c1..1ee621f 100644 --- a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md +++ b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md @@ -193,6 +193,80 @@ quality result was opened, and no evaluator was run for this amendment. This working copy remains unpreregistered until its exact bytes are committed and bound as H0 before protected materialization. +Twelfth pre-H0 execution-boundary amendment candidate: 2026-08-14. A final +pre-execution audit found that the calibration capture's complete-RULER verifier +iterated, parsed, base64-decoded, and semantically replayed all 20 receipt +records before selecting the 16 calibration records. That path therefore could +open the four held-out Stage-A RULER receipt bodies before the one-run seal. The +then-current calibration capture was started and manually terminated after +approximately seven minutes. It published no identity input, candidate, or +frozen identity, and it loaded no model weights or quality results. The CLI has +no progress receipt, so the exact last call cannot be proven; because complete +RULER verification precedes every phase-specific dataset capture, the four +Stage-A RULER bodies are conservatively treated as possibly decoded by the +automated process. No receipt body or derived content was printed or inspected +by a person. Source commit `19ef835a8ec2341c36657b7e010ad1ae6135de9a` +and tag `experiment013-h0-19ef835` are preserved as the first bound H0 attempt, +but that H0 is now retired and superseded. It authorizes no further identity +capture or execution and produced no identity, H1, model, or quality artifact. + +Calibration verification is now phase-scoped. It authenticates the canonical +complete generation-manifest bytes and the public identity, size, and SHA-256 +commitments for every result, but it reads, base64-decodes, and semantically +replays only the 16 calibration receipt files. Parsing the complete manifest as +JSON necessarily parses the four Stage-A result envelopes, their embedded +command objects, and their raw-validation strings as JSON values. During +calibration those embedded protected values are deliberately uninterpreted: the +command objects are not traversed or authenticated semantically, the payload +strings are not base64-decoded, the separate Stage-A receipt files are not read, +and no Stage-A row is replayed. They are next decoded and semantically verified +only by the authenticated Stage-A identity-capture/materialization phase, which +necessarily precedes the evaluator's one-run seal; the offline evaluator then +reauthenticates and rematerializes them after reserving that one run. Thus the +repair restores cross-phase isolation from calibration, not absolute preseal +semantic blindness. Paired regressions supply deliberately invalid protected +embedded values: calibration must accept them as uninterpreted while reading +exactly its 16 receipt files, whereas Stage-A identity capture must reject them +and must never read the 16 calibration receipt files. + +The conservative recovery is fixed before replacement generation and is not +result-adaptive. The original Stage-A seed 2,339 is incremented past the +already-frozen Stage-B seed grid 2,339 through 2,342; therefore 2,343, the first +seed outside that grid, is the sole replacement seed. The four old Stage-A +seed-2,339 cells are retired and are neither Stage-A nor Stage-B evidence. The +replacement inventory is exactly: + +| Category/config | Replacement receipt | Canonical command-manifest SHA-256 | +| --- | --- | --- | +| retrieval / `niah_multiquery` | `retrieval__niah_multiquery__l4096__s2343.json` | `4f33fdcdf1902c17988ecce5cc344d5feffa3e99d6a948a98e4a4a0ccce51252` | +| multi-hop tracing / `vt` | `multi_hop_tracing__vt__l4096__s2343.json` | `9d2038bbc19723b27af364b870a14ed4f516272db83df13dc0e8b524b9a44bff` | +| aggregation / `fwe` | `aggregation__fwe__l4096__s2343.json` | `613477e4edc91064f120c728683858bd79c4f7c3c7c359f0010215b5adb01f48` | +| question answering / `qa_1` | `question_answering__qa_1__l4096__s2343.json` | `c0b5591282d51a94280aedafecf3c31400e9fc3423011bc4cedd6ce26e719663` | + +A fresh complete 20-receipt batch and generation manifest must contain the 16 +unchanged calibration identities plus exactly these four replacements. A batch +or manifest containing any retired Stage-A seed-2,339 receipt is rejected. The +capture and resolver procedure versions advance together from 5 to 6, while the +identity input, candidate, and frozen schema strings remain v5 because their +field sets did not change and no v5 identity was published. The RULER launcher +advances from v6 to v7, so all 20 canonical command-manifest hashes are +recomputed from the authenticated v7 launcher source. Generation-manifest +schema v2 remains unchanged because its structure did not change. + +The next clean H0 must bind the seed-2,343 rule, exact replacement inventory, +complete v7 command-hash table, procedure versions, formatter, and regressions. +It must be committed and tagged before any seed-2,343 receipt is generated or +its raw body is accessed. Generated receipt-byte hashes do not exist at H0; +after generation they are bound by the canonical complete generation manifest +and then by the promoted identity chain. Seed 2,343 was chosen solely as the +first integer after the reserved 2,339-through-2,342 Stage-B grid, before +generation or content access. If a seed-2,343 receipt fails generation, length, +or semantic gates, stop: there is no alternate seed, config, or fallback. A +same-identity retry is allowed only after a documented infrastructure +interruption. This incident does not require new PG19 or HumanEval+ Stage-A +identities: the complete-RULER verifier ran before the phase-specific PG19 and +HumanEval+ reads. + ## Question Can a calibration-frozen, static Q4/Q6/Q8 recurrent-state layout satisfy the @@ -638,8 +712,8 @@ Stage A contains exactly 12 examples: - the first four SHA-ranked eligible PG19 validation books, each using 4,096 prefill tokens followed by 128 continuation tokens, of which 127 predictions are exposed to the committed quantized cache; -- four RULER category representatives at configured length 4,096 and seed - 2,339: `niah_multiquery`, `vt`, `fwe`, and `qa_1`, using each +- four RULER category representatives at configured length 4,096 and recovery + seed 2,343: `niah_multiquery`, `vt`, `fwe`, and `qa_1`, using each identity-bound teacher-forced target derived from the official references; and - the first four SHA-ranked HumanEval+ canonical IDs, using at most the first @@ -706,9 +780,10 @@ new experiment but may not replace the candidate post hoc. Stage B remains closed until Stage A and every identity gate pass. It contains: - the remaining 28 eligible PG19 validation books after Stage A; -- the remaining 48 configurations in the complete development grid of all 13 - exact RULER configs at configured length 4,096 and seeds 2,339 through - 2,342; and +- the original 48-cell RULER development grid at configured length 4,096: all + 52 combinations of the 13 exact configs and seeds 2,339 through 2,342 except + the four retired seed-2,339 cells for `niah_multiquery`, `vt`, `fwe`, and + `qa_1`; and - the remaining 28 HumanEval+ tasks under the Stage-A/B ranking domain. Stage-B identities and token spans are unresolved protected placeholders in @@ -1007,6 +1082,24 @@ training code but no tagged release or pretrained checkpoint for the reported Gated DeltaNet path plus end-to-end adoption benefit could be differentiated; this protocol makes no novelty claim. +[MixKVQ](https://arxiv.org/abs/2512.19206) already combines query relevance +with intrinsic quantization difficulty for mixed-precision KV-cache allocation, +and [Block-GTQ](https://arxiv.org/abs/2606.24033) already uses label-free query/key +energy, marginal-gain bit allocation, and a packed serving path. Experiment +013's query-energy times measured row-MSE selector is therefore a prespecified +diagonal read-error approximation to test, not a new general allocation +principle. [LightMamba](https://arxiv.org/abs/2502.15260v2) already combines +Hadamard-assisted low-bit Mamba inference with hardware co-design and reports +that its elementwise Mamba hidden-state recurrence is not rotation-equivariant. +The potentially differentiating RecurQuant hypothesis is narrower: prove that +a Gated DeltaNet matrix state admits an exact right/value-axis orthogonal +basis, keep the persistent state in that basis, and consume its physical-row +Q4/Q6/Q8 representation directly in the recurrent update without a persistent +FP copy. +[Nemotron 3 Super](https://arxiv.org/abs/2604.12374) further makes recurrent +rounding drift and stochastic rounding mandatory baselines rather than optional +ablations. + Even a complete pass would establish only that the frozen static packed layout was useful on the pinned 0.8B checkpoint, workloads, budgets, and hardware. It would not establish that RHT, mixed precision, loss sensitivity, Q4/Q6/Q8, @@ -1016,5 +1109,16 @@ justify "breakthrough," "state of the art," "lossless," or universal language. It also would not prove a closed-loop StateLease controller; Experiment 013's map is immutable after calibration. +A later breakthrough-level claim requires evidence beyond Experiment 013: +exact unquantized basis-equivalence tests; no-RHT and multi-seed RHT ablations; +round-to-nearest versus stochastic-rounding comparisons; uniform Q6 and static +INT8 baselines; long-horizon 4K, 32K, and 128K recurrence including free +generation; at least one larger and one independent Gated-DeltaNet-family +checkpoint; a packed-native kernel with no persistent FP mirror compared +against an optimized architecture-native baseline; fair batch-N byte and +throughput accounting; and independent reproduction on another GPU/software +stack. Until those pass, the public claim remains the exact frozen quality, +storage, and implementation result actually measured here. + Failure is a publishable result. Any change after a gate is observed creates a new experiment number with new protected data. diff --git a/scripts/capture_static_q468_identity_input.py b/scripts/capture_static_q468_identity_input.py index 7c3ec63..43e64f3 100644 --- a/scripts/capture_static_q468_identity_input.py +++ b/scripts/capture_static_q468_identity_input.py @@ -60,7 +60,8 @@ class or requests a weight file. sys.modules.pop(_RESOLVER_MODULE_NAME, None) raise -CAPTURE_VERSION: Final = 5 +# Procedure version. The resolver-compatible identity field sets remain v5. +CAPTURE_VERSION: Final = 6 RUNTIME_AUTHENTICATION_CONTEXT_FIELDS: Final = frozenset( { "base_runtime_root", @@ -74,7 +75,7 @@ class or requests a weight file. RULER_RECEIPT_SCHEMA: Final = "recurquant.experiment013.ruler-receipt.v1" RULER_GENERATION_MANIFEST_SCHEMA: Final = "recurquant.experiment013.ruler-generation-manifest.v2" RULER_RUNTIME_MANIFEST_SCHEMA: Final = "recurquant.experiment013.ruler-runtime-manifest.v3" -RULER_LAUNCHER_REVISION: Final = "experiment-013-ruler-argv-launcher-v6" +RULER_LAUNCHER_REVISION: Final = "experiment-013-ruler-argv-launcher-v7" RULER_RUNTIME_PYTHON_VERSION: Final = "3.11.15" RULER_SEALED_STARTUP_POLICY: Final = { "dont_write_bytecode": 1, @@ -363,64 +364,64 @@ class RulerTaskInvariant: # and task-specific flags, without trusting claims embedded in the receipt set. RULER_COMMAND_MANIFEST_SHA256_BY_FILENAME: Final = { "aggregation__cwe__l2048__s12340.json": ( - "2b0bf4952b9fd1f4447947ff0e6257da3a32552ec28bf850b7d992761a90c76c" + "2e51bae29377e1cc5d4f92d0b8479951e788e7fb27fd2f8fcc177c9e3cc2811a" ), "aggregation__cwe__l4096__s12340.json": ( - "47e06031fdeea0041e007526e18b0f68902b8e248fb6a4325ac0ca1f3e179071" + "a63f87abadca40d66a682584ee831122d3c837ecc44a1ddb57a9e0b676744aa7" ), "aggregation__fwe__l2048__s12339.json": ( - "39f939a6cd7041a7922f2a3f87c5c1af047e948040ae39969468934993dfca54" + "ec652c7d26916a1e988c02a88c9d856ff05ce926583e1a2fce953b919b3c7ad2" ), "aggregation__fwe__l4096__s12339.json": ( - "d4e47efd8b709a6e27d5969128d2869bea3a99e1093606e89f31720ccb650448" + "49f19c6045484e1c584657fb57412f1fbd1945d6cfcca6504a666b344765a0d0" ), - "aggregation__fwe__l4096__s2339.json": ( - "b889115e728fe5eb1e59c3a6f276d464969ae696011d8bfb99a1330bf5ced6f4" + "aggregation__fwe__l4096__s2343.json": ( + "613477e4edc91064f120c728683858bd79c4f7c3c7c359f0010215b5adb01f48" ), "multi_hop_tracing__vt__l2048__s12339.json": ( - "c362d4c629dd96a94ac01eb24d3eb92b8cae0adaffec49dc8de931a881a68335" + "9d253ced1f09e593fc1cffff0f035604685f54acb0f4dc0080d5f920f9a65993" ), "multi_hop_tracing__vt__l2048__s12340.json": ( - "82bd4e0157a8ac52ad1f7ab1c001bce38a73a584e7a557aeffb4d8672f75556a" + "25e0258140d82c80ae6d466c8ba3c22dc98aae29c6b8b87bbbf1817c1c4aa561" ), "multi_hop_tracing__vt__l4096__s12339.json": ( - "0b039028f6888c790979c521d679b334ea512b41fc0bbf021a7037afefddf0f4" + "570153e676619042f72a71c2ed2eb27829fcef292cebce7734d3523b10581271" ), "multi_hop_tracing__vt__l4096__s12340.json": ( - "d2a5b85b5bd563374a492f45f4578e6f6e04c6addf7c8c62d7f34fd5ab2e0035" + "93ee5ec2be6fe5ac4bde20d82f32a1e9f46ec4008aab2b4d9c035b813255dae5" ), - "multi_hop_tracing__vt__l4096__s2339.json": ( - "ed61a589b8c288a7499d8a26ff1dc91af4418b3ac5f61eef7e4f77c7c3c2741a" + "multi_hop_tracing__vt__l4096__s2343.json": ( + "9d2038bbc19723b27af364b870a14ed4f516272db83df13dc0e8b524b9a44bff" ), "question_answering__qa_1__l2048__s12339.json": ( - "c5d7533b7aa53ff7a6e0efc67af88947cfc5c1c76672024dd7daf0e6ad4a4018" + "12cc42446211811efe8e525e3fa8f68306338b7cb15901e4bc6c037563505bea" ), "question_answering__qa_1__l4096__s12339.json": ( - "e57351321e38e98129a7784915333ac08079dcc01a80724e1dc78d068f8101ff" + "ff9623b65bd1fa8285c9078bcc861a7ceefe9ca6b6bfe470dae6c2cc08a6de83" ), - "question_answering__qa_1__l4096__s2339.json": ( - "77b57f21537d318c7100055f1198823f68ebfc4c0deccbe528e881f952e08239" + "question_answering__qa_1__l4096__s2343.json": ( + "c0b5591282d51a94280aedafecf3c31400e9fc3423011bc4cedd6ce26e719663" ), "question_answering__qa_2__l2048__s12340.json": ( - "c210afae817638883b1b07da358fc726c7d12575a55772078ddb9091443b595c" + "402b92a2c76288d37b819683a1e5e83e8a5bd95c60143d958e0b027e74975997" ), "question_answering__qa_2__l4096__s12340.json": ( - "a25b93adc8b8f0e4bb6f9faeaa319f58942349ec9c7c8c163f376883353982d3" + "e841f60a9858d16da1490cee99c64db20435c8d992f27a71d196a1601b478e2a" ), "retrieval__niah_multikey_2__l2048__s12340.json": ( - "bc71b720edfa491e5fe0cad882cd0e361829f3c10ed4781b66a71c28b9075731" + "58a474ae63c52c5bf897de988ef952e85a87326096a5db2374a577e44c53aeed" ), "retrieval__niah_multiquery__l2048__s12339.json": ( - "2d57d83f619d8e462beb67ac629badde65b661bbbc771ff39e5cfdb9c9eaa97f" + "61b77e3475cdf72fed65e7d28d97151dcbb0a9b728cf40756e634371be99009d" ), - "retrieval__niah_multiquery__l4096__s2339.json": ( - "481c41baed008fc121049ca0b43308c79f3e85839c0e941ea8eb8d7294a0037c" + "retrieval__niah_multiquery__l4096__s2343.json": ( + "4f33fdcdf1902c17988ecce5cc344d5feffa3e99d6a948a98e4a4a0ccce51252" ), "retrieval__niah_multivalue__l4096__s12340.json": ( - "252be364c6fbc93f21e004f71b38468da195f9acd0aeaf2d0357e85ddfe6c1c8" + "d791dcf45a923a7d8aafc5cf2d9dda3b039878288556307bf75023a338c0b6f8" ), "retrieval__niah_single_1__l4096__s12339.json": ( - "aa7d34fb3a5176fccaa58a925635c667fcfb0fea92dbf6ba3d900316faa68e60" + "d08571d33bde611a4e8b7394ce0b6ec92742f6b4d15a6a3f044974ea99d42949" ), } @@ -1976,8 +1977,11 @@ def _verify_ruler_raw_row( def _verify_complete_ruler_bundle( source: CaptureSource, *, + phase: str, tokenizer_material: TokenizerMaterial, ) -> VerifiedRulerBundle: + if phase not in resolver.ALLOWED_PHASES: + raise ValueError(f"unsupported RULER verification phase: {phase!r}") raw_manifest = source.ruler_generation_manifest_bytes() manifest = _strict_json(raw_manifest, context="RULER generation manifest") if canonical_json_bytes(manifest) != raw_manifest: @@ -2078,7 +2082,6 @@ def _verify_complete_ruler_bundle( for field, minimum in ( ("configured_length", 1), ("seed", 0), - ("generator_reported_length", 1), ("size_bytes", 1), ): _require_int( @@ -2097,6 +2100,19 @@ def _verify_complete_ruler_bundle( if result[field] != expected[field]: raise ValueError(f"RULER receipt result {field} drifted") filename = str(expected["filename"]) + _require_sha256(result["sha256"], context=f"RULER receipt result {filename} SHA-256") + if expected["phase"] != phase: + # The complete generation-manifest hash commits to all 20 results, but a + # phase process must not open or semantically validate the other phase's + # receipt bodies. Matching bytes are authenticated and decoded in their + # own identity-capture phase; the offline evaluator reauthenticates them + # again after its one-run reservation. + continue + _require_int( + result["generator_reported_length"], + context="RULER receipt result generator_reported_length", + minimum=1, + ) receipt_bytes = source.ruler_receipt_bytes( category=str(expected["category"]), config=str(expected["config"]), @@ -2886,7 +2902,7 @@ def _capture_identity_input_with_tokens( """Run the sole capture flow, optionally retaining formatter token IDs.""" if phase in resolver.PROTECTED_STAGES: - raise PermissionError(f"{phase} is protected; capture v5 refuses it before source access") + raise PermissionError(f"{phase} is protected; capture v6 refuses it before source access") if phase not in resolver.ALLOWED_PHASES: raise ValueError(f"unsupported identity phase: {phase!r}") if phase == "stage_a" and calibration_binding is None: @@ -2923,7 +2939,11 @@ def _capture_identity_input_with_tokens( before = _validate_heads(source.source_heads(), context="pre-capture") material = source.tokenizer_material() tokenizer_contract, tokenizer_manifest_hash = _tokenizer_contract(material) - ruler_bundle = _verify_complete_ruler_bundle(source, tokenizer_material=material) + ruler_bundle = _verify_complete_ruler_bundle( + source, + phase=phase, + tokenizer_material=material, + ) token_sink: TokenCaptureSink | None = {} if collect_tokens else None mbpp_records, mbpp_manifest_hash = _capture_mbpp( source, @@ -3089,8 +3109,8 @@ def materialize_stage_a_identity_sequences( if not isinstance(calibration_binding_artifact, bytes): raise TypeError("Stage-A calibration binding artifact must be bytes") if ( - CAPTURE_VERSION != 5 - or resolver.RESOLVER_VERSION != 5 + CAPTURE_VERSION != 6 + or resolver.RESOLVER_VERSION != 6 or resolver.INPUT_SCHEMA != "recurquant.experiment013.identity-input.v5" or resolver.FROZEN_SCHEMA != "recurquant.experiment013.identity-frozen.v5" ): @@ -3195,12 +3215,50 @@ def atomic_write_no_overwrite(path: Path, payload: bytes) -> None: temporary.unlink() +def _verify_live_ruler_receipt_inventory(root: Path) -> Mapping[str, Path]: + """Return the exact complete non-redirected live receipt-file inventory.""" + + unresolved = Path(os.path.abspath(root)) + + def is_redirected(path: Path) -> bool: + try: + status = path.lstat() + except OSError as error: + raise ValueError(f"cannot authenticate RULER receipt path: {path}") from error + return path.is_symlink() or bool(getattr(status, "st_file_attributes", 0) & 0x400) + + if is_redirected(unresolved) or not unresolved.is_dir(): + raise ValueError("RULER receipt root must be a regular non-redirected directory") + entries = sorted(unresolved.iterdir(), key=lambda path: path.name) + names = [path.name for path in entries] + if len({name.casefold() for name in names}) != len(names): + raise ValueError("RULER receipt directory contains case-colliding names") + observed = set(names) + expected = {"generation-manifest.json"} | { + str(item["filename"]) for item in required_ruler_receipts() + } + if observed != expected: + missing = sorted(expected - observed) + unexpected = sorted(observed - expected) + raise ValueError( + f"RULER receipt directory inventory drifted: missing={missing}, unexpected={unexpected}" + ) + result: dict[str, Path] = {} + for path in entries: + if is_redirected(path) or not path.is_file(): + raise ValueError( + f"RULER receipt entry must be a regular non-redirected file: {path.name}" + ) + result[path.name] = path + return MappingProxyType(result) + + class LiveCaptureSource: """Pinned, read-only network source; RULER generated rows come from receipts.""" def __init__(self, *, cache_dir: Path, ruler_receipt_dir: Path) -> None: self.cache_dir = cache_dir.resolve() - self.ruler_receipt_dir = ruler_receipt_dir.resolve() + self.ruler_receipt_dir = Path(os.path.abspath(ruler_receipt_dir)) @staticmethod def _github_revision(repo_id: str, revision: str) -> str: @@ -3401,24 +3459,25 @@ def _receipt_filename(*, category: str, config: str, configured_length: int, see ) def ruler_generation_manifest_bytes(self) -> bytes: - path = self.ruler_receipt_dir / "generation-manifest.json" - if not path.is_file(): - raise FileNotFoundError(f"missing complete RULER generation manifest: {path}") - return path.read_bytes() + inventory = _verify_live_ruler_receipt_inventory(self.ruler_receipt_dir) + return inventory["generation-manifest.json"].read_bytes() def ruler_receipt_bytes( self, *, category: str, config: str, configured_length: int, seed: int ) -> bytes: - path = self.ruler_receipt_dir / self._receipt_filename( + filename = self._receipt_filename( category=category, config=config, configured_length=configured_length, seed=seed, ) - if not path.is_file(): + inventory = _verify_live_ruler_receipt_inventory(self.ruler_receipt_dir) + try: + path = inventory[filename] + except KeyError as error: raise FileNotFoundError( - f"missing audited RULER receipt; generation is intentionally separate: {path}" - ) + f"requested RULER receipt is outside the exact frozen inventory: {filename}" + ) from error return path.read_bytes() def ruler_receipt( @@ -4826,7 +4885,7 @@ def main(argv: Sequence[str] | None = None) -> int: args = parse_args(argv) if args.phase in resolver.PROTECTED_STAGES: raise PermissionError( - f"{args.phase} is protected; capture v5 refuses it before file or source access" + f"{args.phase} is protected; capture v6 refuses it before file or source access" ) if args.ruler_receipt_dir is None: raise ValueError("--ruler-receipt-dir is required") diff --git a/scripts/generate_static_q468_ruler_receipts.py b/scripts/generate_static_q468_ruler_receipts.py index 423302f..b981f69 100644 --- a/scripts/generate_static_q468_ruler_receipts.py +++ b/scripts/generate_static_q468_ruler_receipts.py @@ -32,7 +32,7 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[1] CAPTURE_PATH = REPOSITORY_ROOT / "scripts" / "capture_static_q468_identity_input.py" -LAUNCHER_REVISION: Final = "experiment-013-ruler-argv-launcher-v6" +LAUNCHER_REVISION: Final = "experiment-013-ruler-argv-launcher-v7" GENERATION_MANIFEST_SCHEMA: Final = "recurquant.experiment013.ruler-generation-manifest.v2" RUNTIME_MANIFEST_SCHEMA: Final = "recurquant.experiment013.ruler-runtime-manifest.v3" RUNTIME_PYTHON_VERSION: Final = "3.11.15" @@ -1969,6 +1969,114 @@ def _verify_owned_orphan_tree( raise ValueError("published RULER diagnostic orphan is incomplete") +def _verify_receipt_batch_inventory( + *, + required: Sequence[Mapping[str, object]], + output_dir: Path, + raw_root: Path, + require_complete: bool, +) -> None: + """Authenticate the exact resumable receipt/diagnostic top-level inventory.""" + + output_root = Path(os.path.abspath(output_dir)) + diagnostic_root = Path(os.path.abspath(raw_root)) + output_root.mkdir(parents=True, exist_ok=True) + diagnostic_root.mkdir(parents=True, exist_ok=True) + if ( + _is_reparse_point(output_root) + or _is_reparse_point(diagnostic_root) + or output_root.resolve() == diagnostic_root.resolve() + ): + raise ValueError("RULER output and raw roots must be distinct non-redirected directories") + + required_by_filename: dict[str, Mapping[str, object]] = {} + required_by_stem: dict[str, Mapping[str, object]] = {} + required_by_receipt_key: dict[str, Mapping[str, object]] = {} + for item in required: + filename = str(item["filename"]) + if Path(filename).name != filename or not filename.endswith(".json"): + raise ValueError("required RULER receipt filename is not a canonical JSON basename") + stem = filename.removesuffix(".json") + if filename in required_by_filename or stem in required_by_stem: + raise ValueError("required RULER receipt inventory is not unique") + receipt_key = _sha256_bytes(filename.encode("utf-8"))[:12] + if receipt_key in required_by_receipt_key: + raise ValueError("required RULER receipt staging keys collide") + required_by_filename[filename] = item + required_by_stem[stem] = item + required_by_receipt_key[receipt_key] = item + + def scan(root: Path, *, context: str) -> dict[str, Path]: + entries = sorted(root.iterdir(), key=lambda path: path.name) + names = [path.name for path in entries] + if len({name.casefold() for name in names}) != len(names): + raise ValueError(f"{context} contains case-colliding names") + return {path.name: path for path in entries} + + output_entries = scan(output_root, context="RULER output directory") + allowed_output = set(required_by_filename) | {"generation-manifest.json"} + unexpected_output = sorted(set(output_entries) - allowed_output) + if unexpected_output: + raise ValueError(f"RULER output directory contains unexpected entries: {unexpected_output}") + for name, path in output_entries.items(): + if _is_reparse_point(path) or not path.is_file(): + raise ValueError(f"RULER output entry must be a regular non-redirected file: {name}") + + raw_entries = scan(diagnostic_root, context="RULER raw directory") + staging_pattern = re.compile(r"^\.rq-([0-9a-f]{12})\.[A-Za-z0-9_-]+\.staging$") + published_raw: dict[str, Path] = {} + staging_raw: dict[str, tuple[Path, Mapping[str, object]]] = {} + unexpected_raw: list[str] = [] + for name, path in raw_entries.items(): + if name in required_by_stem: + published_raw[name] = path + continue + match = staging_pattern.fullmatch(name) + item = None if match is None else required_by_receipt_key.get(match.group(1)) + if item is None: + unexpected_raw.append(name) + else: + staging_raw[name] = (path, item) + unexpected_raw.sort() + if unexpected_raw: + raise ValueError(f"RULER raw directory contains unexpected entries: {unexpected_raw}") + for stem, path in published_raw.items(): + if _is_reparse_point(path) or not path.is_dir(): + raise ValueError(f"RULER raw entry must be a regular non-redirected directory: {stem}") + _verify_owned_orphan_tree( + path, + raw_root=diagnostic_root, + config=str(required_by_stem[stem]["config"]), + require_complete=True, + ) + for name, (path, item) in staging_raw.items(): + if _is_reparse_point(path) or not path.is_dir(): + raise ValueError( + f"RULER staging entry must be a regular non-redirected directory: {name}" + ) + filename = str(item["filename"]) + if filename in output_entries: + raise ValueError("RULER staging orphan exists beside its published receipt") + _verify_owned_orphan_tree( + path, + raw_root=diagnostic_root, + config=str(item["config"]), + require_complete=False, + ) + + receipt_names = set(output_entries) - {"generation-manifest.json"} + diagnostic_receipts = {f"{stem}.json" for stem in published_raw} + if not receipt_names <= diagnostic_receipts: + raise ValueError("a RULER receipt exists without its raw diagnostic inventory") + complete = receipt_names == set(required_by_filename) + if "generation-manifest.json" in output_entries and not complete: + raise ValueError("complete RULER generation manifest exists beside an incomplete set") + if require_complete and ( + not complete or diagnostic_receipts != set(required_by_filename) or staging_raw + ): + raise ValueError("RULER receipt or raw diagnostic batch is incomplete") + + def recover_owned_receipt_orphans( *, filename: str, @@ -2204,6 +2312,12 @@ def finalize_generation_manifest_if_complete( ) -> dict[str, object] | None: """Publish the sole manifest only after re-verifying the full 20-file set.""" + _verify_receipt_batch_inventory( + required=required, + output_dir=output_dir, + raw_root=raw_root, + require_complete=False, + ) manifest_path = output_dir / "generation-manifest.json" missing = [ str(item["filename"]) @@ -2249,6 +2363,12 @@ def finalize_generation_manifest_if_complete( if len(results) != 20: raise ValueError("complete RULER generation manifest must contain exactly 20 receipts") _atomic_publish_same(manifest_path, _canonical_json_bytes(manifest)) + _verify_receipt_batch_inventory( + required=required, + output_dir=output_dir, + raw_root=raw_root, + require_complete=True, + ) return manifest @@ -2304,6 +2424,12 @@ def main(argv: Sequence[str] | None = None) -> int: } if len(identities) != len(required): raise ValueError("required RULER receipt identities must be unique") + _verify_receipt_batch_inventory( + required=required, + output_dir=args.output_dir, + raw_root=args.raw_dir, + require_complete=False, + ) if args.receipt: if len(set(args.receipt)) != len(args.receipt): raise ValueError("--receipt values must be unique") diff --git a/scripts/resolve_static_q468_identity.py b/scripts/resolve_static_q468_identity.py index 7c5807f..dd57941 100644 --- a/scripts/resolve_static_q468_identity.py +++ b/scripts/resolve_static_q468_identity.py @@ -36,7 +36,8 @@ CANDIDATE_SCHEMA: Final = "recurquant.experiment013.identity-candidate.v5" FROZEN_SCHEMA: Final = "recurquant.experiment013.identity-frozen.v5" ARTIFACT_KIND: Final = "recurquant_static_rht_q468_identity" -RESOLVER_VERSION: Final = 5 +# Procedure version. The identity field sets remain the published v5 contract. +RESOLVER_VERSION: Final = 6 PARQUET_MATERIALIZATION_MANIFEST_FILE_SHA256: Final = ( "ee5628e50e5d3516fd79077542d355fd915455ac0e53128d372f4177ad63d39c" ) @@ -171,10 +172,10 @@ ("question_answering", "qa_2", 4_096, 12_340), ) RULER_STAGE_A_SCHEDULE: Final = ( - ("retrieval", "niah_multiquery", 4_096, 2_339), - ("multi_hop_tracing", "vt", 4_096, 2_339), - ("aggregation", "fwe", 4_096, 2_339), - ("question_answering", "qa_1", 4_096, 2_339), + ("retrieval", "niah_multiquery", 4_096, 2_343), + ("multi_hop_tracing", "vt", 4_096, 2_343), + ("aggregation", "fwe", 4_096, 2_343), + ("question_answering", "qa_1", 4_096, 2_343), ) CLAIM_BOUNDARY: Final = ( @@ -1319,7 +1320,9 @@ def build_candidate( raise ValueError("identity input schema drifted") if phase not in ALLOWED_PHASES: if phase in PROTECTED_STAGES: - raise PermissionError(f"{phase} is protected and unavailable in resolver v5") + raise PermissionError( + f"{phase} is protected and unavailable in resolver procedure v{RESOLVER_VERSION}" + ) raise ValueError(f"unsupported identity phase: {phase!r}") if source["model_weights_loaded"] is not False: raise ValueError("identity resolution must occur before model weights") diff --git a/tests/test_capture_static_q468_identity_input.py b/tests/test_capture_static_q468_identity_input.py index ba4f38f..21e62d9 100644 --- a/tests/test_capture_static_q468_identity_input.py +++ b/tests/test_capture_static_q468_identity_input.py @@ -790,7 +790,7 @@ def test_calibration_capture_is_deterministic_and_resolver_compatible() -> None: first, expected_revisions=resolver.FROZEN_DATASET_REVISIONS ) assert candidate["evidence"]["record_count"] == 160 - assert capture.CAPTURE_VERSION == resolver.RESOLVER_VERSION == 5 + assert capture.CAPTURE_VERSION == resolver.RESOLVER_VERSION == 6 assert first["schema"] == "recurquant.experiment013.identity-input.v5" assert candidate["evidence"]["identity_schema"] == ( "recurquant.experiment013.identity-candidate.v5" @@ -816,6 +816,144 @@ def test_calibration_capture_is_deterministic_and_resolver_compatible() -> None: ) +@pytest.mark.parametrize( + ("protected_field", "protected_value", "stage_a_error"), + [ + ( + "command_manifest", + {"protected_stage_a": "must-remain-uninterpreted"}, + "RULER command manifest fields drifted", + ), + ( + "raw_validation_base64", + "protected-stage-a-must-not-be-decoded", + "RULER raw validation is not canonical base64", + ), + ], +) +def test_calibration_leaves_stage_a_ruler_embedded_values_uninterpreted( + protected_field: str, + protected_value: object, + stage_a_error: str, +) -> None: + fixture = FakeSource() + manifest = json.loads(fixture.ruler_generation_manifest_bytes()) + stage_a_filenames = { + item["filename"] for item in capture.required_ruler_receipts() if item["phase"] == "stage_a" + } + calibration_filenames = { + item["filename"] + for item in capture.required_ruler_receipts() + if item["phase"] == "calibration" + } + for result in manifest["receipts"]: + if result["phase"] == "stage_a": + result[protected_field] = protected_value + manifest_bytes = capture.canonical_json_bytes(manifest) + + class ManifestSource(FakeSource): + def __init__(self) -> None: + super().__init__() + self.receipt_reads: set[str] = set() + + def ruler_generation_manifest_bytes(self) -> bytes: + return manifest_bytes + + def ruler_receipt_bytes( + self, *, category: str, config: str, configured_length: int, seed: int + ) -> bytes: + filename = capture.ruler_receipt_filename( + category=category, + config=config, + configured_length=configured_length, + seed=seed, + ) + self.receipt_reads.add(filename) + return super().ruler_receipt_bytes( + category=category, + config=config, + configured_length=configured_length, + seed=seed, + ) + + class OpaqueStageASource(ManifestSource): + def ruler_receipt_bytes( + self, *, category: str, config: str, configured_length: int, seed: int + ) -> bytes: + filename = capture.ruler_receipt_filename( + category=category, + config=config, + configured_length=configured_length, + seed=seed, + ) + if filename in stage_a_filenames: + raise AssertionError("calibration attempted to read a protected Stage-A receipt") + return super().ruler_receipt_bytes( + category=category, + config=config, + configured_length=configured_length, + seed=seed, + ) + + source = OpaqueStageASource() + captured = capture.capture_identity_input(phase="calibration", source=source) + + assert captured["phase"] == "calibration" + assert source.receipt_reads == calibration_filenames + + stage_a_source = ManifestSource() + with pytest.raises(ValueError, match=stage_a_error): + capture.capture_identity_input( + phase="stage_a", + source=stage_a_source, + calibration_binding=_binding(), + ) + assert stage_a_source.receipt_reads + assert stage_a_source.receipt_reads <= stage_a_filenames + + +def test_stage_a_capture_reads_only_stage_a_ruler_receipts() -> None: + manifest_bytes = FakeSource().ruler_generation_manifest_bytes() + + class CountingSource(FakeSource): + def __init__(self) -> None: + super().__init__() + self.receipt_reads: set[str] = set() + + def ruler_generation_manifest_bytes(self) -> bytes: + return manifest_bytes + + def ruler_receipt_bytes( + self, *, category: str, config: str, configured_length: int, seed: int + ) -> bytes: + filename = capture.ruler_receipt_filename( + category=category, + config=config, + configured_length=configured_length, + seed=seed, + ) + self.receipt_reads.add(filename) + return super().ruler_receipt_bytes( + category=category, + config=config, + configured_length=configured_length, + seed=seed, + ) + + expected = { + item["filename"] for item in capture.required_ruler_receipts() if item["phase"] == "stage_a" + } + source = CountingSource() + captured = capture.capture_identity_input( + phase="stage_a", + source=source, + calibration_binding=_binding(), + ) + + assert captured["phase"] == "stage_a" + assert source.receipt_reads == expected + + def test_execution_artifacts_are_authenticated_before_and_after_all_data_access( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -1405,7 +1543,7 @@ def test_stage_a_materialization_accepts_exact_two_token_target() -> None: source = FakeSource() source.receipt_mutator = lambda receipt: ( receipt.update({"outputs": ["xy"]}) - if receipt["config"] == "qa_1" and receipt["seed"] == 2_339 + if receipt["config"] == "qa_1" and receipt["seed"] == 2_343 else None ) frozen_bytes = _frozen_stage_a_identity(source) @@ -1570,7 +1708,7 @@ def test_ruler_receipt_required_output_cardinality_and_uniqueness_fail_closed() category="retrieval", config="niah_multiquery", configured_length=4_096, - seed=2_339, + seed=2_343, ) receipt["outputs"] = ["only-one"] with pytest.raises(ValueError, match="exactly 4 required outputs"): @@ -1579,7 +1717,7 @@ def test_ruler_receipt_required_output_cardinality_and_uniqueness_fail_closed() category="retrieval", config="niah_multiquery", configured_length=4_096, - seed=2_339, + seed=2_343, ) receipt["outputs"] = ["same"] * 4 @@ -1589,7 +1727,7 @@ def test_ruler_receipt_required_output_cardinality_and_uniqueness_fail_closed() category="retrieval", config="niah_multiquery", configured_length=4_096, - seed=2_339, + seed=2_343, ) @@ -1639,7 +1777,7 @@ def test_ruler_receipt_replays_frozen_task_semantics( category=category, config=config, configured_length=4_096, - seed=2_339, + seed=2_343, ) mutate(receipt) @@ -1649,7 +1787,7 @@ def test_ruler_receipt_replays_frozen_task_semantics( category=category, config=config, configured_length=4_096, - seed=2_339, + seed=2_343, ) @@ -1658,7 +1796,7 @@ def test_ruler_receipt_rejects_boolean_sample_index() -> None: category="retrieval", config="niah_multiquery", configured_length=4_096, - seed=2_339, + seed=2_343, ) receipt["sample_index"] = False @@ -1668,7 +1806,7 @@ def test_ruler_receipt_rejects_boolean_sample_index() -> None: category="retrieval", config="niah_multiquery", configured_length=4_096, - seed=2_339, + seed=2_343, ) @@ -1679,7 +1817,7 @@ def test_ruler_raw_row_rejects_boolean_numeric_fields(field: str) -> None: category="retrieval", config="niah_multiquery", configured_length=4_096, - seed=2_339, + seed=2_343, ) row = json.loads(source._raw_row(receipt)) row[field] = False @@ -1936,6 +2074,23 @@ def test_incomplete_ruler_generation_manifest_is_rejected() -> None: capture.capture_identity_input(phase="calibration", source=source) +def test_live_source_rejects_retired_stage_a_receipt_directory_extra(tmp_path: Path) -> None: + receipt_dir = tmp_path / "receipts" + receipt_dir.mkdir() + (receipt_dir / "generation-manifest.json").write_bytes(b"fixture") + for item in capture.required_ruler_receipts(): + (receipt_dir / item["filename"]).write_bytes(b"fixture") + retired = "retrieval__niah_multiquery__l4096__s2339.json" + (receipt_dir / retired).write_bytes(b"retired") + source = capture.LiveCaptureSource( + cache_dir=tmp_path / "cache", + ruler_receipt_dir=receipt_dir, + ) + + with pytest.raises(ValueError, match=r"inventory drifted: .*unexpected=.*s2339"): + source.ruler_generation_manifest_bytes() + + def test_ruler_command_manifest_argv_tamper_is_rejected() -> None: source = FakeSource() source.manifest_mutator = lambda manifest: manifest["receipts"][0]["command_manifest"][ @@ -2400,13 +2555,16 @@ def test_capture_output_contains_no_raw_model_or_weight_claim() -> None: def test_required_ruler_receipt_inventory_is_exact_and_unique() -> None: receipts = capture.required_ruler_receipts() + stage_a_receipts = [item for item in receipts if item["phase"] == "stage_a"] assert len(receipts) == 20 assert len({item["filename"] for item in receipts}) == 20 assert sum(item["phase"] == "calibration" for item in receipts) == 16 - assert sum(item["phase"] == "stage_a" for item in receipts) == 4 + assert len(stage_a_receipts) == 4 + assert {item["seed"] for item in stage_a_receipts} == {2343} + assert all("__s2339.json" not in item["filename"] for item in stage_a_receipts) assert receipts[0]["filename"] == ("retrieval__niah_multiquery__l2048__s12339.json") - assert receipts[-1]["filename"] == ("question_answering__qa_1__l4096__s2339.json") + assert receipts[-1]["filename"] == ("question_answering__qa_1__l4096__s2343.json") def _parquet_bytes(columns: dict[str, list[str]]) -> bytes: diff --git a/tests/test_generate_static_q468_ruler_receipts.py b/tests/test_generate_static_q468_ruler_receipts.py index 4af3151..61ddc30 100644 --- a/tests/test_generate_static_q468_ruler_receipts.py +++ b/tests/test_generate_static_q468_ruler_receipts.py @@ -388,6 +388,90 @@ def test_retry_never_cleans_diagnostics_beside_a_published_receipt(tmp_path) -> assert published.is_dir() +def test_batch_inventory_allows_recoverable_required_raw_only_orphan(tmp_path) -> None: + filename = "retrieval__niah_single_1__l4096__s12339.json" + config = "niah_single_1" + required = [{"filename": filename, "config": config}] + raw_root = tmp_path / "raw" + output_dir = tmp_path / "output" + published = raw_root / filename.removesuffix(".json") + (published / config).mkdir(parents=True) + output_dir.mkdir() + for relative in ( + "command-manifest.json", + "runtime-manifest.json", + "stdout.log", + "stderr.log", + f"{config}/validation.jsonl", + ): + (published / relative).write_bytes(b"fixture") + + ruler._verify_receipt_batch_inventory( + required=required, + output_dir=output_dir, + raw_root=raw_root, + require_complete=False, + ) + recovered = ruler.recover_owned_receipt_orphans( + filename=filename, + config=config, + raw_root=raw_root, + output_dir=output_dir, + ) + + assert recovered == (published.name,) + assert not published.exists() + + +def test_batch_inventory_allows_recoverable_owned_staging_orphan(tmp_path) -> None: + filename = "retrieval__niah_single_1__l4096__s12339.json" + config = "niah_single_1" + required = [{"filename": filename, "config": config}] + raw_root = tmp_path / "raw" + output_dir = tmp_path / "output" + raw_root.mkdir() + output_dir.mkdir() + receipt_key = ruler._sha256_bytes(filename.encode("utf-8"))[:12] + staging = raw_root / f".rq-{receipt_key}.fixture.staging" + staging.mkdir() + (staging / "command-manifest.json").write_bytes(b"fixture") + + ruler._verify_receipt_batch_inventory( + required=required, + output_dir=output_dir, + raw_root=raw_root, + require_complete=False, + ) + recovered = ruler.recover_owned_receipt_orphans( + filename=filename, + config=config, + raw_root=raw_root, + output_dir=output_dir, + ) + + assert recovered == (staging.name,) + assert not staging.exists() + + +def test_batch_inventory_rejects_retired_stage_a_receipt(tmp_path) -> None: + capture = ruler._load_capture_module() + required = capture.required_ruler_receipts() + output_dir = tmp_path / "output" + raw_root = tmp_path / "raw" + output_dir.mkdir() + raw_root.mkdir() + retired = "retrieval__niah_multiquery__l4096__s2339.json" + (output_dir / retired).write_bytes(b"retired") + + with pytest.raises(ValueError, match="unexpected entries"): + ruler._verify_receipt_batch_inventory( + required=required, + output_dir=output_dir, + raw_root=raw_root, + require_complete=False, + ) + + def test_two_partial_invocations_then_full_set_publish_one_complete_manifest( monkeypatch, tmp_path ) -> None: @@ -404,8 +488,25 @@ def test_two_partial_invocations_then_full_set_publish_one_complete_manifest( ] output_dir = tmp_path / "receipts" output_dir.mkdir() + raw_dir = tmp_path / "raw" + raw_dir.mkdir() verified: list[str] = [] + def publish_fixture(item: dict[str, object], payload: bytes) -> None: + filename = str(item["filename"]) + config = str(item["config"]) + (output_dir / filename).write_bytes(payload) + diagnostic = raw_dir / filename.removesuffix(".json") + (diagnostic / config).mkdir(parents=True) + for relative in ( + "command-manifest.json", + "runtime-manifest.json", + "stdout.log", + "stderr.log", + f"{config}/validation.jsonl", + ): + (diagnostic / relative).write_bytes(b"fixture") + def fake_verify(*, path: Path, receipt, **_kwargs): verified.append(path.name) return { @@ -441,7 +542,7 @@ def fake_verify(*, path: Path, receipt, **_kwargs): kwargs = { "required": required, "output_dir": output_dir, - "raw_root": tmp_path / "raw", + "raw_root": raw_dir, "capture": capture, "python": tmp_path / "python.exe", "package_root": tmp_path / "site-packages", @@ -459,15 +560,15 @@ def fake_verify(*, path: Path, receipt, **_kwargs): } for item in required[:7]: - (output_dir / item["filename"]).write_bytes(b"partial-one") + publish_fixture(item, b"partial-one") assert ruler.finalize_generation_manifest_if_complete(**kwargs) is None assert not (output_dir / "generation-manifest.json").exists() for item in required[7:13]: - (output_dir / item["filename"]).write_bytes(b"partial-two") + publish_fixture(item, b"partial-two") assert ruler.finalize_generation_manifest_if_complete(**kwargs) is None assert not (output_dir / "generation-manifest.json").exists() for item in required[13:]: - (output_dir / item["filename"]).write_bytes(b"complete") + publish_fixture(item, b"complete") manifest = ruler.finalize_generation_manifest_if_complete(**kwargs) diff --git a/tests/test_resolve_static_q468_identity.py b/tests/test_resolve_static_q468_identity.py index ba568e5..770b4df 100644 --- a/tests/test_resolve_static_q468_identity.py +++ b/tests/test_resolve_static_q468_identity.py @@ -525,29 +525,25 @@ def _stage_a_source() -> dict[str, Any]: scored_stop=4_224, ) ) - ruler_rows = ( - ("retrieval", "niah_multiquery"), - ("multi_hop_tracing", "vt"), - ("aggregation", "fwe"), - ("question_answering", "qa_1"), - ) - for rank, (category, config) in enumerate(ruler_rows): + for rank, (category, config, configured_length, seed) in enumerate( + resolver.RULER_STAGE_A_SCHEDULE + ): records.append( _record( family="ruler", canonical_id=resolver.ruler_canonical_id( category=category, config=config, - configured_length=4_096, - seed=2_339, + configured_length=configured_length, + seed=seed, ), config=config, rank=rank, - seed=2_339, + seed=seed, sequence_length=4_096, prefill_stop=4_092, scored_stop=4_096, - configured_length=4_096, + configured_length=configured_length, ruler_category=category, ) ) From de4b8d8b514a331bcc8f4ab5b039f4c2b12473ef Mon Sep 17 00:00:00 2001 From: HiddenTrojan Date: Sat, 15 Aug 2026 00:59:08 +0800 Subject: [PATCH 17/24] research: freeze Experiment 013 calibration identity v5 --- evidence/experiment013-calibration-identity-v5.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 evidence/experiment013-calibration-identity-v5.json diff --git a/evidence/experiment013-calibration-identity-v5.json b/evidence/experiment013-calibration-identity-v5.json new file mode 100644 index 0000000..34eb5db --- /dev/null +++ b/evidence/experiment013-calibration-identity-v5.json @@ -0,0 +1 @@ +{"canonical_evidence_sha256":"ffef2570940d2d97658af470e90a9df0a0153f0d17f3bbe63be8174ea6dc76e0","evidence":{"artifact_kind":"recurquant_static_rht_q468_identity","calibration_binding":null,"calibration_split_half":{"assignment":[{"canonical_id":"872","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":0,"rank_sha256":"01e970ad8f49c615f8994f4fc293d373d517f753cf5cef3ee60e04b0f70e91de","ruler_category":null,"seed":null,"sequence_length":205},{"canonical_id":"907","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":1,"rank_sha256":"0302f20255cec68e850f67a965f94ac9c6e975fdbeb3654b213c03aaf97749f8","ruler_category":null,"seed":null,"sequence_length":195},{"canonical_id":"602","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":2,"rank_sha256":"0533693c5eafac0f2bffd8ecb5f531f37e3220e7347ef75eab142ed8a013bcf5","ruler_category":null,"seed":null,"sequence_length":131},{"canonical_id":"884","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":3,"rank_sha256":"08317423d6ed9f655783d9a55d4773b1ff31c0a8355dfd9186fb319920e3d84b","ruler_category":null,"seed":null,"sequence_length":180},{"canonical_id":"771","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":4,"rank_sha256":"0aa65ef4aee19c821f354944ea63d3a5573594cbf98328585a2bf06c2de91309","ruler_category":null,"seed":null,"sequence_length":210},{"canonical_id":"885","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":5,"rank_sha256":"0b8df4c9f6062430c6c4ccd1cb93ed972689c2181d647b3425c8ac35779c5f9c","ruler_category":null,"seed":null,"sequence_length":193},{"canonical_id":"672","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":6,"rank_sha256":"0d45859c2052e7483b7caae8e52246e30d4af1be980cbcb175164817f5ed511d","ruler_category":null,"seed":null,"sequence_length":164},{"canonical_id":"955","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":7,"rank_sha256":"15743574d0995e07e15392053b3cafd1cef3998b668e87041851ac95861883ca","ruler_category":null,"seed":null,"sequence_length":116},{"canonical_id":"794","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":8,"rank_sha256":"19ee9c8929dc155f23f27a50530b329f876367ad3aea0bf4ad48ce4621328517","ruler_category":null,"seed":null,"sequence_length":154},{"canonical_id":"641","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":9,"rank_sha256":"19fd16656926305e85d6a90a8abfc8688cde21fff9383ac96aa4a507279d79f9","ruler_category":null,"seed":null,"sequence_length":104},{"canonical_id":"744","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":10,"rank_sha256":"1afe31e9cfc41a6df7f6609d1e2119ad55693730c75b4bd46f7c9235522eafa0","ruler_category":null,"seed":null,"sequence_length":137},{"canonical_id":"647","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":11,"rank_sha256":"1b76da1efe10f0a39d6b61952eb3fee319e653390381bd4a8efb060e10161d77","ruler_category":null,"seed":null,"sequence_length":107},{"canonical_id":"763","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":12,"rank_sha256":"1c7722a133b7999ee3e6ff0c4c7d9286371137589ac7b013d5d9a1615c2eca43","ruler_category":null,"seed":null,"sequence_length":181},{"canonical_id":"835","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":13,"rank_sha256":"27e79049077f4f70e44f882154a065bb8fe957aa57c51a5f95d025e7f6b69e7e","ruler_category":null,"seed":null,"sequence_length":113},{"canonical_id":"720","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":14,"rank_sha256":"2c2ba3d77f6374f2c035569f8be3b970f76c25bc587f0f2bd165939b49af1c27","ruler_category":null,"seed":null,"sequence_length":292},{"canonical_id":"677","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":15,"rank_sha256":"3015295069fd5535022e22aa7d447fdfd3b893623d25fbe8b2279d94c7d99ee5","ruler_category":null,"seed":null,"sequence_length":120},{"canonical_id":"819","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":16,"rank_sha256":"3355f161793b19251d4281f96e19b37a970a709a1b417d52e6dfbb2a4812d792","ruler_category":null,"seed":null,"sequence_length":385},{"canonical_id":"950","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":17,"rank_sha256":"339900553b726e1b1256a75b102fbeed3f547bfae8d96f80e8fec0f8dec6c09b","ruler_category":null,"seed":null,"sequence_length":399},{"canonical_id":"622","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":18,"rank_sha256":"363cce01da9a0aa22049e9b3a8e04a2dc931468e02d0086bb7af971621cfc711","ruler_category":null,"seed":null,"sequence_length":373},{"canonical_id":"708","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":19,"rank_sha256":"372acd4106f75fd22c1fdb5d8e13b74c64a9364db2f490f50daa482c0962112d","ruler_category":null,"seed":null,"sequence_length":97},{"canonical_id":"878","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":20,"rank_sha256":"38728fc433a299c9ff5e0f49471dda51ee67c5a980c966c00c715bd76e7f1c88","ruler_category":null,"seed":null,"sequence_length":168},{"canonical_id":"648","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":21,"rank_sha256":"388ca5f79823a878cd3ce1e033b61e5cab8bf668fe57d751138244feef74a2f4","ruler_category":null,"seed":null,"sequence_length":225},{"canonical_id":"937","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":22,"rank_sha256":"3ad5eee8852be5065365af9c25ff78ec2974ee89b2c6fa6759e27b7de255e202","ruler_category":null,"seed":null,"sequence_length":110},{"canonical_id":"894","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":23,"rank_sha256":"3c7238b06352863f75aab36214380eedfa02c7c388385142283f0980d21c1f7a","ruler_category":null,"seed":null,"sequence_length":226},{"canonical_id":"653","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":24,"rank_sha256":"3dd1d2a846dba10b3ffa7e554091ac65fe9284c15f2d83dc44c705155130a763","ruler_category":null,"seed":null,"sequence_length":287},{"canonical_id":"679","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":25,"rank_sha256":"3e59d101b564d5e3344a2e764584a194a5f663f3b20f7e939d7f719f84dacba4","ruler_category":null,"seed":null,"sequence_length":136},{"canonical_id":"657","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":26,"rank_sha256":"3e6b96b2be683d5d193895bbc05beb5a13bfc3b1db01872788e0ae8133c72b36","ruler_category":null,"seed":null,"sequence_length":171},{"canonical_id":"924","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":27,"rank_sha256":"45104189ef24ab437b12b531013325cf0a41819cf135255a4ca993f29f74dfd2","ruler_category":null,"seed":null,"sequence_length":103},{"canonical_id":"686","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":28,"rank_sha256":"47010c76b0b792ee5452a5eb55e8ba40d386f4a17d895b443eb1a66556c90596","ruler_category":null,"seed":null,"sequence_length":285},{"canonical_id":"860","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":29,"rank_sha256":"471b89737df3331c676e99243030a77140d460cde893f819d2c7916e0a16f453","ruler_category":null,"seed":null,"sequence_length":141},{"canonical_id":"698","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":30,"rank_sha256":"4a9f3f47f58cfaabd9afda89457088ca35397d7acc80568d8d218a723451116e","ruler_category":null,"seed":null,"sequence_length":359},{"canonical_id":"827","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":31,"rank_sha256":"4b073dd1bc9f2a79d60985c8d50aae572f2030e62a55477c64702d735d849023","ruler_category":null,"seed":null,"sequence_length":174},{"canonical_id":"651","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":32,"rank_sha256":"4bf2b2787c95b7b8a2f8ff089c36ea2662b4987dbf230b02b81c424382853b7b","ruler_category":null,"seed":null,"sequence_length":155},{"canonical_id":"930","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":33,"rank_sha256":"4ffef951fb9708b54468eaa75c7b42b9f58c198c366cfacd6b4f665c9d77984f","ruler_category":null,"seed":null,"sequence_length":138},{"canonical_id":"962","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":34,"rank_sha256":"51a6c027909f096835086346110e63a131c1462156020bb129058b320f27516a","ruler_category":null,"seed":null,"sequence_length":149},{"canonical_id":"853","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":35,"rank_sha256":"525322d1199e29de4998af2c95211cd981e642e9d602fe16554a8a63fc02d729","ruler_category":null,"seed":null,"sequence_length":219},{"canonical_id":"722","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":36,"rank_sha256":"52acb8744119319aaa1a68b3e192415c937a6822ceb2963069d4ede3e9b75cc4","ruler_category":null,"seed":null,"sequence_length":388},{"canonical_id":"705","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":37,"rank_sha256":"547e7ebc86a8992205db2eda5a7117796f15c3042e90200d556582e58f881c17","ruler_category":null,"seed":null,"sequence_length":258},{"canonical_id":"944","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":38,"rank_sha256":"577f992e266f3ffe7684e9483310b96df9b338bd836e00af79e0116821fddaa5","ruler_category":null,"seed":null,"sequence_length":117},{"canonical_id":"849","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":39,"rank_sha256":"59660871dc9200a74f124b924d6e85f743e541a39ae93af501968a7ed5d0d751","ruler_category":null,"seed":null,"sequence_length":161},{"canonical_id":"919","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":40,"rank_sha256":"5a21a04a8534d8dff43c2d02da1415ffaabed4a09662059b05e916108da7665e","ruler_category":null,"seed":null,"sequence_length":113},{"canonical_id":"629","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":41,"rank_sha256":"5aeb994e6b892f1741b894b66750ea3690b71585304601a66a17d599e3e96bc5","ruler_category":null,"seed":null,"sequence_length":150},{"canonical_id":"820","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":42,"rank_sha256":"6063af147dd771453be5c51054092d7aa85f751eab9d697cc1b77be9259ed7ff","ruler_category":null,"seed":null,"sequence_length":105},{"canonical_id":"627","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":43,"rank_sha256":"608304ac3176b904f89e46ddf06a578afe6f80161ceb1adb2b5049c5b31a8c5a","ruler_category":null,"seed":null,"sequence_length":198},{"canonical_id":"755","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":44,"rank_sha256":"60b435df2cb81d92c2de2d1edc7b06a50a523b8d40047dd521cf241475d6b370","ruler_category":null,"seed":null,"sequence_length":209},{"canonical_id":"911","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":45,"rank_sha256":"614cef397364dc767d15426e06989a10147811417996eb4fa91721038ecac80c","ruler_category":null,"seed":null,"sequence_length":236},{"canonical_id":"874","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":46,"rank_sha256":"62fc38937c372269a01f02df68b088b1fdc004f7ae4590a50cb3673f385737d0","ruler_category":null,"seed":null,"sequence_length":153},{"canonical_id":"918","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":47,"rank_sha256":"6326299b9eaf024dfc7efa6b26bb898a5b869898127d546067800b05ba8c5932","ruler_category":null,"seed":null,"sequence_length":229},{"canonical_id":"803","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":48,"rank_sha256":"65df1000f86d2eacc5ecb7d9b91504d3c6074a9e9c5e6cc8abebfc45c0f60974","ruler_category":null,"seed":null,"sequence_length":142},{"canonical_id":"868","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":49,"rank_sha256":"69fda125d9dfbbac0a44eaf6db26953f3f644a70389add149dd1c47e3ca90bf4","ruler_category":null,"seed":null,"sequence_length":137},{"canonical_id":"666","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":50,"rank_sha256":"6c8674e41d1ec38f03b2d176e08aef538902ff0c35234be50164674f4e52a341","ruler_category":null,"seed":null,"sequence_length":108},{"canonical_id":"768","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":51,"rank_sha256":"71cb4040b63cf195e0e702fd08f6b6e9eeef03a3de4a3c29ca0a715a42e2fbf6","ruler_category":null,"seed":null,"sequence_length":144},{"canonical_id":"636","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":52,"rank_sha256":"7357df8332514f1e676d0104bd545961a71f432580a835738fb62ad7dbc89065","ruler_category":null,"seed":null,"sequence_length":127},{"canonical_id":"671","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":53,"rank_sha256":"7a7a9eda4414dd86e6dbd5ebf5e1ac84df7df6e12e3242cb2574817453c497b8","ruler_category":null,"seed":null,"sequence_length":193},{"canonical_id":"783","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":54,"rank_sha256":"7aabfc7187ec019772ef6f0f0ae5203e0193909146a86ecdc6a1f789f86ff6c3","ruler_category":null,"seed":null,"sequence_length":428},{"canonical_id":"695","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":55,"rank_sha256":"7ce9ea381ac2a57a8373c1253b0dfbedb3b2e838c4dfd92bb0730f83c972434c","ruler_category":null,"seed":null,"sequence_length":168},{"canonical_id":"851","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":56,"rank_sha256":"7df3d4757b59a5be15e6e4b9208bfbdbf189f4590d7b2e93ca0f84047f2a97b4","ruler_category":null,"seed":null,"sequence_length":131},{"canonical_id":"945","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":57,"rank_sha256":"7fdc834d39787ed9f97e73645f7df6692b711d910c52482f0bbab1d41512f720","ruler_category":null,"seed":null,"sequence_length":132},{"canonical_id":"687","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":58,"rank_sha256":"80a7b335ef9b11db152007e896afdd070710cbee60d4766ef829826eacbbf56b","ruler_category":null,"seed":null,"sequence_length":153},{"canonical_id":"908","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":59,"rank_sha256":"8267d3b36347390e9dbe928d4f2af728975e565b8c89232c8fe011b762fb9e75","ruler_category":null,"seed":null,"sequence_length":175},{"canonical_id":"973","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":60,"rank_sha256":"85f2e19defa2f2194e4bd627c43470d94dfce10775f216d0eee0f5b7c89a8113","ruler_category":null,"seed":null,"sequence_length":107},{"canonical_id":"795","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":61,"rank_sha256":"874c549c5d365887eba08a0eac638427d2825d7bedc3da4b4aa6c213de86162a","ruler_category":null,"seed":null,"sequence_length":325},{"canonical_id":"822","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":62,"rank_sha256":"87ee9e94899a058de1bc21331e5ffecb56f11ce924750fc73a6a928a2501d281","ruler_category":null,"seed":null,"sequence_length":203},{"canonical_id":"728","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":63,"rank_sha256":"884e088f56e16b57342c6b5d6b1021b0f7e756e9d8d1720ce65c243eb1c95acf","ruler_category":null,"seed":null,"sequence_length":165},{"canonical_id":"895","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":64,"rank_sha256":"888165087ffe22467a3cefc2c892a649a93905d8f22cd2389a8c7ba0be70f967","ruler_category":null,"seed":null,"sequence_length":293},{"canonical_id":"764","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":65,"rank_sha256":"8981728482ffcdb96388f812ddfe70ae76010f928900d65b3f4a8c8490ed266b","ruler_category":null,"seed":null,"sequence_length":125},{"canonical_id":"804","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":66,"rank_sha256":"8996abd03167db364eb3cf521940939970ca3b49310060c371c0e447dca9f4d4","ruler_category":null,"seed":null,"sequence_length":134},{"canonical_id":"732","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":67,"rank_sha256":"89d9cbbf3458bcc72f432dd40166313978822085bc657d2cc2ff5fe6919c27a8","ruler_category":null,"seed":null,"sequence_length":134},{"canonical_id":"839","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":68,"rank_sha256":"8c48b54163b5a2b5d91d303abe4d2f5990fd2296dd47e1960208c1d0286d9450","ruler_category":null,"seed":null,"sequence_length":365},{"canonical_id":"823","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":69,"rank_sha256":"8df1ac0f53587cc228a0d68d4959d98a79d5c3046a25094f36d1698d013a1a7a","ruler_category":null,"seed":null,"sequence_length":205},{"canonical_id":"713","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":70,"rank_sha256":"8eb162011c59bc92c4728ac7fe2fd64ac0f7f78f7aef1ba048088ed9743e16d2","ruler_category":null,"seed":null,"sequence_length":121},{"canonical_id":"772","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":71,"rank_sha256":"8f08bc003b61873c2e9f27eceb15bf7a3a451a8f4877c759cc73b5f437a170af","ruler_category":null,"seed":null,"sequence_length":166},{"canonical_id":"727","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":72,"rank_sha256":"925293e3739b4a2498fb5d6c9cbf9fbd57520f22f99a7b3468fad6290361cf36","ruler_category":null,"seed":null,"sequence_length":161},{"canonical_id":"858","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":73,"rank_sha256":"935132e4b17a56013a0312175973c9a6b2b092902d5bf1d474b03a34da4953f6","ruler_category":null,"seed":null,"sequence_length":177},{"canonical_id":"800","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":74,"rank_sha256":"9400093971582810aed9b43834286c1bbcdbe15e12fd1f749b9e3a7abbefa8f4","ruler_category":null,"seed":null,"sequence_length":104},{"canonical_id":"751","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":75,"rank_sha256":"96679ba0da36bb95058654f8fea49ff5305709d80cd95a1017d570f0d7997ddf","ruler_category":null,"seed":null,"sequence_length":243},{"canonical_id":"674","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":76,"rank_sha256":"96c806e69e6b65095699a2192e225cc22ad2b21211dbb8eae36893932ef358e1","ruler_category":null,"seed":null,"sequence_length":128},{"canonical_id":"960","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":77,"rank_sha256":"97692bbfd642b241043c3eb82fd110341e33df034bcc65a60b79b26933a15b3e","ruler_category":null,"seed":null,"sequence_length":124},{"canonical_id":"867","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":78,"rank_sha256":"979109f8d629072c0302399d65fa7263972fa9ed22cefff82f4c11028507238f","ruler_category":null,"seed":null,"sequence_length":170},{"canonical_id":"669","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":79,"rank_sha256":"98ddb8c36da9cd7adc3852209dcda61e9da6d3acd0d1fcf1211f25ae04c07cc5","ruler_category":null,"seed":null,"sequence_length":296},{"canonical_id":"656","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":80,"rank_sha256":"9979d6795117639a4d1f8afae4ddfa11dfeeefbf2ca8687a7e67b84643a1744f","ruler_category":null,"seed":null,"sequence_length":169},{"canonical_id":"862","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":81,"rank_sha256":"99e8fb364eec07a688b1d0dbc74ba9372c47719c0193b304c88bc2659ecd75c4","ruler_category":null,"seed":null,"sequence_length":177},{"canonical_id":"635","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":82,"rank_sha256":"9cf09b9e19663d0a700e97d8d627776519ef82d0d8608b62a5645b11309ec58f","ruler_category":null,"seed":null,"sequence_length":266},{"canonical_id":"756","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":83,"rank_sha256":"9d6ace56a87e526ec18af2d10e07c6dd1a3287a2d8271415ac4a0c3a3f642035","ruler_category":null,"seed":null,"sequence_length":139},{"canonical_id":"869","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":84,"rank_sha256":"9d975c8aee95cb7e1e54deeee4073d57a63712f67247e5fb61dcf791dbb16564","ruler_category":null,"seed":null,"sequence_length":352},{"canonical_id":"857","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":85,"rank_sha256":"9e00a569dd1098d4912c32f57e2351709bf3ffce9a75edf00179d5f023c41d41","ruler_category":null,"seed":null,"sequence_length":293},{"canonical_id":"607","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":86,"rank_sha256":"a18910fa12a223a0ecb7d6707500711e3b376af22c72b99625792f523b548d89","ruler_category":null,"seed":null,"sequence_length":207},{"canonical_id":"929","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":87,"rank_sha256":"a39d16127e92e6eb192ad63a45fa7db3125509f3a86fcc647d0a72ffdfc25483","ruler_category":null,"seed":null,"sequence_length":168},{"canonical_id":"606","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":88,"rank_sha256":"a4d325b0ce1088bd97fbaac9299e2d0dbb32b002026f2528825225d44f1e1e60","ruler_category":null,"seed":null,"sequence_length":144},{"canonical_id":"662","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":89,"rank_sha256":"a572f1763ed2dacfc5b6f8bafb04903bbc369519be95c8ecfd7d26a2792dc619","ruler_category":null,"seed":null,"sequence_length":350},{"canonical_id":"877","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":90,"rank_sha256":"a7576eaa33a3fb7087c29ae14949e5bfc2e1ef4449b997384e88a2d5dd631591","ruler_category":null,"seed":null,"sequence_length":93},{"canonical_id":"946","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":91,"rank_sha256":"a884dbdb72cb7d11613027848d08702a496c419bc5c987e9e963e2184c774bd8","ruler_category":null,"seed":null,"sequence_length":209},{"canonical_id":"785","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":92,"rank_sha256":"ab824ea47e78d00ae4aec6a45480e122e3703f1da3037053af0aa7a891b637ee","ruler_category":null,"seed":null,"sequence_length":152},{"canonical_id":"920","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":93,"rank_sha256":"ad357f573b8f19c5b89f84aa9097d23ace8d1488e9cb2c1322577137d0d53c2a","ruler_category":null,"seed":null,"sequence_length":251},{"canonical_id":"616","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":94,"rank_sha256":"b09c89c6a2366a5a51caa203e1f363d01e8340abcd1ca162839843b86eef1161","ruler_category":null,"seed":null,"sequence_length":221},{"canonical_id":"840","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":95,"rank_sha256":"b2be594c39afb592547a351b8235b8061e103c0c75d777ad1549d2e015a87989","ruler_category":null,"seed":null,"sequence_length":131},{"canonical_id":"644","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":96,"rank_sha256":"b6557529f8ec5764b47cd34b0b579eb20371c1c581831a55709bb7d1b27348af","ruler_category":null,"seed":null,"sequence_length":186},{"canonical_id":"704","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":97,"rank_sha256":"ba157c706c3d65835a4022b531b73e3f1b65c1660d0699fd7293821d2ebfe350","ruler_category":null,"seed":null,"sequence_length":153},{"canonical_id":"712","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":98,"rank_sha256":"bb6cd04968145b1904af1e96e70b74699cb7138a72c4c62e0b82ee87a3375fc0","ruler_category":null,"seed":null,"sequence_length":219},{"canonical_id":"882","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":99,"rank_sha256":"bf3e15adb4c326aab0f479085d1e9971db834476e9ecaf1a9c3d7764fec4378b","ruler_category":null,"seed":null,"sequence_length":112},{"canonical_id":"710","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":100,"rank_sha256":"bf5d27c709dca8c30c14453c4e20b7c75f1e93bd5952da6bbc741e43a68f0575","ruler_category":null,"seed":null,"sequence_length":163},{"canonical_id":"821","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":101,"rank_sha256":"c0ab0a4bb636c134ba2233ecc3d1925c81df2a89060c09a2908c9dddbc14b591","ruler_category":null,"seed":null,"sequence_length":282},{"canonical_id":"707","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":102,"rank_sha256":"c43bbdda480f2ebabe97769f2958f17a19febca5a53140948914ad1bdd0a6e3e","ruler_category":null,"seed":null,"sequence_length":200},{"canonical_id":"614","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":103,"rank_sha256":"c5450b1ef19a8eaf7e4710ad9115e76118a4634fb769d3924067bdec342e2e70","ruler_category":null,"seed":null,"sequence_length":165},{"canonical_id":"741","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":104,"rank_sha256":"c7c6fbada9b7a2d3d38750bc16cb99ea97efb71c0413563d3f399cc0722c6bcd","ruler_category":null,"seed":null,"sequence_length":124},{"canonical_id":"702","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":105,"rank_sha256":"c8a56538eebae04a29a21ac3c638e71a9ea53a1fdc12603034b61d9ef4359cc9","ruler_category":null,"seed":null,"sequence_length":332},{"canonical_id":"824","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":106,"rank_sha256":"c9106dec19a3fa31e33d340927464d15b6150cbc251a6aff64336706d0400b98","ruler_category":null,"seed":null,"sequence_length":129},{"canonical_id":"809","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":107,"rank_sha256":"ca59973992bc52f67689974ae41022e87d598c8c31de31955f30bfaf21889c3f","ruler_category":null,"seed":null,"sequence_length":173},{"canonical_id":"886","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":108,"rank_sha256":"cc1cdebfc08022bfe9fae59cf187db791a655ca5b48463b8c2725e74349fa708","ruler_category":null,"seed":null,"sequence_length":152},{"canonical_id":"789","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":109,"rank_sha256":"d71db894fab22f0f681f7e02a1ec80d922272c111bb9462b9d066866ab601c18","ruler_category":null,"seed":null,"sequence_length":98},{"canonical_id":"870","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":110,"rank_sha256":"dacc01a92b59eb20074bd1a8255fdbbc3298f37030e34e3d4e074c0bb66134ba","ruler_category":null,"seed":null,"sequence_length":208},{"canonical_id":"854","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":111,"rank_sha256":"dd5f39a3a28901c30d626c95276606c2a6ad00b0f64384c782406f9a13ef8d1e","ruler_category":null,"seed":null,"sequence_length":235},{"canonical_id":"646","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":112,"rank_sha256":"deb0492a1dc5a716095faeaf6d418cf2bf74040a83940a2690a6ea83442404de","ruler_category":null,"seed":null,"sequence_length":129},{"canonical_id":"605","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":113,"rank_sha256":"e161ff3b2e28c4f02462643895bc6e95b48655efc7ac0ace9eec5c4226576cb0","ruler_category":null,"seed":null,"sequence_length":125},{"canonical_id":"902","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":114,"rank_sha256":"e1ebf3f6386478a8625ad8c53858193b08d3e9a3ec267789cd98e85c340f314a","ruler_category":null,"seed":null,"sequence_length":310},{"canonical_id":"848","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":115,"rank_sha256":"e2bc9c9a9b23e33f1abf22dce38357e1e5c10acff55d2a208423270bc02a7289","ruler_category":null,"seed":null,"sequence_length":129},{"canonical_id":"847","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":116,"rank_sha256":"e351b9058c9e5c2e9e58444e1e4f2346c4886c6705652807fc2baab4f1806bd0","ruler_category":null,"seed":null,"sequence_length":143},{"canonical_id":"630","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":117,"rank_sha256":"e553afed5e6fb0d6843dfe2f22bdb6aa65d8f483b406e1b83ee5a2bf2e87fba2","ruler_category":null,"seed":null,"sequence_length":327},{"canonical_id":"793","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":118,"rank_sha256":"e76c7363df6aa1bca2a659469266d4228a66b4a2e10bf91c06346d0eaf0da89e","ruler_category":null,"seed":null,"sequence_length":209},{"canonical_id":"725","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":119,"rank_sha256":"ebb93bbe303910a142ceeda734226a9701884eaee00b68852b557cfc23fc231b","ruler_category":null,"seed":null,"sequence_length":164},{"canonical_id":"903","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":120,"rank_sha256":"ed33b084d9852e0df6989759582108080423613a5014d3f1150c3cbbc0d60573","ruler_category":null,"seed":null,"sequence_length":161},{"canonical_id":"797","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":121,"rank_sha256":"edb3a9f5d11a94af9db154ef9fee83d36402a77de27eb0ba830a3902dca5536c","ruler_category":null,"seed":null,"sequence_length":147},{"canonical_id":"792","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":122,"rank_sha256":"ee24c9054290bd8ccfeccee4993ac8dc6e0e20c72cdf4fbe5d28593c947a2246","ruler_category":null,"seed":null,"sequence_length":130},{"canonical_id":"615","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":123,"rank_sha256":"eed8e28b1d920cec5525bff2f0f861ca6e43429ed75ae657e5754ce34b81d71a","ruler_category":null,"seed":null,"sequence_length":338},{"canonical_id":"640","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":124,"rank_sha256":"f08156d84d873d2ae9184c8db61828a6c9a15f94b5a43b4ef8fe5c1867c2f16d","ruler_category":null,"seed":null,"sequence_length":118},{"canonical_id":"709","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":125,"rank_sha256":"f94bcd46daf0c1bd69bc60844bac9acd167ddc7981f10db96385b6d755574ec2","ruler_category":null,"seed":null,"sequence_length":390},{"canonical_id":"846","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":126,"rank_sha256":"fb81297eb57060d653be5ab5aa61ab169ac4bb236168fbda838bc333f61d4bc7","ruler_category":null,"seed":null,"sequence_length":305},{"canonical_id":"936","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":127,"rank_sha256":"fefd4316fc5fd98157422b169451622b064b3709b67d87d78e00278992813ea2","ruler_category":null,"seed":null,"sequence_length":297},{"canonical_id":"http://www.gutenberg.org/ebooks/1219","config":"default","configured_length":null,"group":"pg19","half":"a","rank":0,"rank_sha256":"061889c1d506cd1d5fb9d49ee4d63712d7e12e6056ffcc25a15ed400766bb5b8","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/38160","config":"default","configured_length":null,"group":"pg19","half":"b","rank":1,"rank_sha256":"10e82e9edc65238326568f332ff5556646b86707e652e24b051769eb8ad7fb02","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/39447","config":"default","configured_length":null,"group":"pg19","half":"a","rank":2,"rank_sha256":"14037bdad952830661d418776aaf4616905c877bfdb291977c357381de5c229c","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/29413","config":"default","configured_length":null,"group":"pg19","half":"b","rank":3,"rank_sha256":"217a13c587e9ae2e779aae14a77fa4a49dbbcbea34bfd717de53c2ac37ea9675","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/1253","config":"default","configured_length":null,"group":"pg19","half":"a","rank":4,"rank_sha256":"2a24d338d8456351047da3a9c12b7feb6c9e39c040ea62275c9284d4bc3b02f6","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/35786","config":"default","configured_length":null,"group":"pg19","half":"b","rank":5,"rank_sha256":"5b4d0ebce948cb01a07e9166f2db8c4d3163c3ce8780be96639120dc13fb26e2","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/17774","config":"default","configured_length":null,"group":"pg19","half":"a","rank":6,"rank_sha256":"5cfbeaf96d6b8bf5cbe8754a74cd22b63225e5c1775594c88f7ff24fc3be1d90","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/10609","config":"default","configured_length":null,"group":"pg19","half":"b","rank":7,"rank_sha256":"73a499a3165431d2fcf4fe5184ef91447b8184d5bf8f0683adb49a4bf058e0fb","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/20279","config":"default","configured_length":null,"group":"pg19","half":"a","rank":8,"rank_sha256":"7da541ed6b56ea9ddd7749aef82c4f66b36beb7ca9ebd5b3ab2623c60996cac9","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/28065","config":"default","configured_length":null,"group":"pg19","half":"b","rank":9,"rank_sha256":"beee6f7ab98cda2f3595fe22f5221f16fd4ce8717a7fe77344ea7e851c85287f","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/24903","config":"default","configured_length":null,"group":"pg19","half":"a","rank":10,"rank_sha256":"c4b007d21ff91cbeb1959618d1f954e864dc7b874ae3986598c860a30b425cc0","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/33622","config":"default","configured_length":null,"group":"pg19","half":"b","rank":11,"rank_sha256":"dcca53b6159ae6063a9e62f780ecd4130f39bf20f2342fe194d39248032cb43d","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/26008","config":"default","configured_length":null,"group":"pg19","half":"a","rank":12,"rank_sha256":"e5734f2587660eccf1f1cae256502a6d7f22fa8d8898f4dee591a3e3a931de27","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/36808","config":"default","configured_length":null,"group":"pg19","half":"b","rank":13,"rank_sha256":"e9713427a3a331bb20abf5840a4a40e37e72569131f89b7ab0691653bc679901","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/2773","config":"default","configured_length":null,"group":"pg19","half":"a","rank":14,"rank_sha256":"f0e4622ba87c3913d530c0f8f7f20c419b2375ff1ca073e6062914dbabe1a4f5","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/32024","config":"default","configured_length":null,"group":"pg19","half":"b","rank":15,"rank_sha256":"f8ecebbe43befef18b00a5780b7383b2a5e3971aca73ed9caf82dd87b7be9b58","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:fwe:length=2048:seed=12339:sample=0","config":"fwe","configured_length":2048,"group":"ruler:aggregation","half":"a","rank":0,"rank_sha256":"407456144cdf7e7f879eeb8c590fc8039a5de9c5d2a99e094f58c354dc7a2e14","ruler_category":"aggregation","seed":12339,"sequence_length":1951},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:cwe:length=4096:seed=12340:sample=0","config":"cwe","configured_length":4096,"group":"ruler:aggregation","half":"b","rank":1,"rank_sha256":"6eed2f87558224dfd68001e835c96568c05e5f827c533e157752f375f4500e19","ruler_category":"aggregation","seed":12340,"sequence_length":3576},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:fwe:length=4096:seed=12339:sample=0","config":"fwe","configured_length":4096,"group":"ruler:aggregation","half":"a","rank":2,"rank_sha256":"9ba9392d1dc6140c8a464c6ab01eae04fd883d8520b997db8d8e7a81ec69412a","ruler_category":"aggregation","seed":12339,"sequence_length":3987},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:cwe:length=2048:seed=12340:sample=0","config":"cwe","configured_length":2048,"group":"ruler:aggregation","half":"b","rank":3,"rank_sha256":"acfea4ad2d1ba514b26bd0835657efe402f76ae3d519c669004fe8f3398c41ba","ruler_category":"aggregation","seed":12340,"sequence_length":1862},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=2048:seed=12339:sample=0","config":"vt","configured_length":2048,"group":"ruler:multi_hop_tracing","half":"a","rank":0,"rank_sha256":"089158a73f7f141394a585c20b3642a57e30b83534a971bffe8d8cc58d1d88a0","ruler_category":"multi_hop_tracing","seed":12339,"sequence_length":2011},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=4096:seed=12339:sample=0","config":"vt","configured_length":4096,"group":"ruler:multi_hop_tracing","half":"b","rank":1,"rank_sha256":"0c5867a34d4c0109a91512496cb821eeda3eacfff858dc77cce30e9fdd0dec20","ruler_category":"multi_hop_tracing","seed":12339,"sequence_length":4056},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=2048:seed=12340:sample=0","config":"vt","configured_length":2048,"group":"ruler:multi_hop_tracing","half":"a","rank":2,"rank_sha256":"47daa45868dba636bc69249fc76653ed16cc4674127f6229ffb7b5c6e0dc816b","ruler_category":"multi_hop_tracing","seed":12340,"sequence_length":2009},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=4096:seed=12340:sample=0","config":"vt","configured_length":4096,"group":"ruler:multi_hop_tracing","half":"b","rank":3,"rank_sha256":"cea1514b80ab90b346af0afd72d4e596f64be107a08c70781a292c2824193d97","ruler_category":"multi_hop_tracing","seed":12340,"sequence_length":4057},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_2:length=2048:seed=12340:sample=0","config":"qa_2","configured_length":2048,"group":"ruler:question_answering","half":"a","rank":0,"rank_sha256":"362ee5e975e3602b3afddead8d433435ec23dbad4defa23337e192b538091cbc","ruler_category":"question_answering","seed":12340,"sequence_length":1893},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_2:length=4096:seed=12340:sample=0","config":"qa_2","configured_length":4096,"group":"ruler:question_answering","half":"b","rank":1,"rank_sha256":"4e8e5919e5dec4b8c1e7aed66e68abf8414441353d6669352f7bb5eb6f3560ba","ruler_category":"question_answering","seed":12340,"sequence_length":2281},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_1:length=2048:seed=12339:sample=0","config":"qa_1","configured_length":2048,"group":"ruler:question_answering","half":"a","rank":2,"rank_sha256":"743c29a00e4cbe2d6bb62a01300b447fc3a78ded5878e9c03e7d80686d62e6c3","ruler_category":"question_answering","seed":12339,"sequence_length":1368},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_1:length=4096:seed=12339:sample=0","config":"qa_1","configured_length":4096,"group":"ruler:question_answering","half":"b","rank":3,"rank_sha256":"ec18351546c72bc1cad5f950ced4793e70ee2fa618a341924278809a62774186","ruler_category":"question_answering","seed":12339,"sequence_length":4040},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_single_1:length=4096:seed=12339:sample=0","config":"niah_single_1","configured_length":4096,"group":"ruler:retrieval","half":"a","rank":0,"rank_sha256":"4a756a6ea055639254af12c8568b7afc07bb626029548efa51a12d72a0109efb","ruler_category":"retrieval","seed":12339,"sequence_length":3955},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multiquery:length=2048:seed=12339:sample=0","config":"niah_multiquery","configured_length":2048,"group":"ruler:retrieval","half":"b","rank":1,"rank_sha256":"5ac13727e634d2432f98b2670f23761bdef39b61ebfadcecff6aa6e1dc1966a9","ruler_category":"retrieval","seed":12339,"sequence_length":1289},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multikey_2:length=2048:seed=12340:sample=0","config":"niah_multikey_2","configured_length":2048,"group":"ruler:retrieval","half":"a","rank":2,"rank_sha256":"8deff28bb1380b3aa1be3b678ea1793381004620e9b4a67fea888e2a859cb53c","ruler_category":"retrieval","seed":12340,"sequence_length":1828},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multivalue:length=4096:seed=12340:sample=0","config":"niah_multivalue","configured_length":4096,"group":"ruler:retrieval","half":"b","rank":3,"rank_sha256":"b13ed20ccd6d9f0502b4d16468a25828441ad0e8b8e8063b1dd3f5d674a1815f","ruler_category":"retrieval","seed":12340,"sequence_length":3964}],"assignment_sha256":"a42cf4b332cc8cf58b27709d7d261fc03a356b27ec1c9ccd56914d99e60c1797","namespace":"recurquant.experiment013.calibration-split.v1\u0000"},"claim_boundary":"This artifact freezes Experiment 013 data, tokenizer, source, runtime, and metadata-only model-file identity. It is not quality, latency, novelty, state-of-the-art, deployment, or breakthrough evidence.","content_manifest_sha256":"ee72483a8f8b4370c9e667e4287747e5bc358aeb0265a58167140f4e780a7b29","datasets":[{"canonical_id_field":"task_id","canonical_id_manifest_sha256":"f46a3c772f1416c75bdd8668c69f4886a32a243c1737b13bbddfdc23b9605c70","config":"full","dataset_id":"google-research-datasets/mbpp","formatter_id":"recurquant.mbpp-prompt-code.v1","formatter_sha256":"882e20ec9f5cbcb7e6f1310cbf46d19153721beac41ecc7ee308c39be17532ff","key":"mbpp","revision":"4bb6404fdc6cacfda99d4ac4205087b89d32030c","split":"train"},{"canonical_id_field":"url","canonical_id_manifest_sha256":"8d7ed934210f5e78f13559cdf48518b922d3c203df9258d0de3c4d62a18bdaa7","config":"default","dataset_id":"emozilla/pg19","formatter_id":"recurquant.pg19-token-slice.v1","formatter_sha256":"faea2480bf85adcd34339cae88a0e9b631b705eb547020572db74402bf525730","key":"pg19","revision":"c021754c8e01c5b1cc83a1f549c1f97fbbb756b8","split":"train"},{"canonical_id_field":"configuration_id","canonical_id_manifest_sha256":"d7fec5f2a57409fb842dcb967de5ed1efce00253723f2d891b28681b8d57952d","config":"official-generator","dataset_id":"NVIDIA/RULER","formatter_id":"recurquant.ruler-official-generated-record.v1","formatter_sha256":"4f797cbca4dfbe802c47dc5b238111b629f1497e46c84ea116d06319012016b6","key":"ruler","revision":"c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a","split":"generated"},{"canonical_id_field":"task_id","canonical_id_manifest_sha256":"c53bc1db6c7965cb13d4238321ef09c0c7b3dbfeb4f4736bfd6e46e2914449da","config":"default","dataset_id":"evalplus/humanevalplus","formatter_id":"recurquant.humaneval-plus-prompt-solution.v1","formatter_sha256":"12204389715ddc210e5a8b1b291f4fbcaf2b64fde94c22699873de80d682204c","key":"humaneval_plus","revision":"d32357cf319e50e9c8d8dab5ea876c72b0fd321b","split":"test"}],"execution_bindings":{"calibration_runtime_manifest_file_sha256":"80ca233a29af4facbb334fd4fb51a4f6e9a3d6815465cb79b9f3db63ef668d6a","model_file_manifest_file_sha256":"586d9c7e520f3bbd99ecef30663bf07d283eb14622475c58891becd8e033b05c","parquet_materialization_manifest_file_sha256":"ee5628e50e5d3516fd79077542d355fd915455ac0e53128d372f4177ad63d39c","repository_source_manifest_file_sha256":"024111c86c2086620d1a214b5dbce719c580fe1571714691391f55d41590934c"},"identity_only":true,"identity_schema":"recurquant.experiment013.identity-frozen.v5","model_contracts":{"conditional_scale_check":{"cold_start_peak_hbm_limit_bytes":8053063680,"id":"Qwen/Qwen3.5-2B-Base","revision":"b1485b2fa6dfa1287294f269f5fb618e03d52d7c"},"primary":{"id":"Qwen/Qwen3.5-0.8B-Base","revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68"},"weights_loaded":false},"phase":"calibration","promotion":{"candidate_canonical_evidence_sha256":"54cf522ede2f6c958bb4199e83453c97a2bd7bc74b8e0d2440f3939c77e66223","candidate_file_sha256":"ec676d00fc80981c88b53cfb1ca045e55289169ff0ae98e7f77883e210e8de4b","explicit":true},"promotion_required":false,"protected_identity":{"ordinary_tests_may_read_protected_content":false,"stage_b_read":false,"stage_c_read":false},"record_count":160,"records":[{"anchor_manifest_sha256":"9727dad06649abe3aad4a15772fd49b9a0ae5c0ef55e61bba6ccfd0445ab8f73","anchor_positions":[7,15,23,32,40,48,56,65,73,81,89,98,106,114,122,131],"anchor_positions_sha256":"24769466c0c30738992bfaea31b322a83686427b8c8c2f42d5977ce98da1e86f","canonical_id":"945","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,31,39,47,55,64,72,80,88,96,104,112,120,129],"fisher_boundary_sha256":"98d4c6b32acad28c83342cab67b79f665cc5144d363b8e6653d190aedaddc45f","horizon":1,"input_positions":[8,16,24,32,40,48,56,65,73,81,89,97,105,113,121,130],"input_token_ids_sha256":"5d42f41e2436cb702ed1f8eb971b20537d123d7e1f41409244fd8dd8129656fc","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,33,41,49,57,66,74,82,90,98,106,114,122,131],"target_token_ids_sha256":"9a4cf735c9ab0560b97058fccd7a2d08e1a7cc3d126701d11007e641f736719b"},"formatted_content_sha256":"aa6feb35c931bc49027c89402eb6fe154ef150b92030a5f3e730756026205a4c","generator_receipt_sha256":null,"identity_record_sha256":"3c49ec4e6136f525892d68d4cf7267b90d1836a9011dc55e73ece42bc882546e","prompt_token_ids_sha256":"5766712b5c83365901b89092414948a16c2a894641c0580978a6720d74c7bb4b","ruler_category":null,"seed":null,"selection_rank":0,"selection_sha256":"006481602de32e6484f069e30640315b4902f198bc2b977318b0a90084e2503d","sequence_length":132,"sequence_token_ids_sha256":"91c8172e893e28a3e7c965916084be5fd76619bd9dc7881f155bfceacf710738","source_content_sha256":"dbd241865f0bf98d18ba09369665294382ef86e1be0440c9b18c083d5cb8529d","target_token_ids_sha256":"16acd83d1b28de14a0fcfcbad7f9a2538e0df11843ba645c1a55bcfeb285cecb","token_span":{"cache_exposed_start":132,"cache_exposed_stop":132,"prefill_start":0,"prefill_stop":112,"scored_start":112,"scored_stop":132},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2efa60ce513767dd377aebb493099ff3ebdc8a0362f30593887e54997aaa3ffc","anchor_positions":[8,18,27,37,47,56,66,76,85,95,104,114,124,133,143,153],"anchor_positions_sha256":"e227da918c51f69b24ec0ca4e8d85817a5330fb6e593d88068dce4e00d5d6d1c","canonical_id":"794","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,18,27,37,46,56,65,75,84,94,103,113,122,132,141,151],"fisher_boundary_sha256":"f690f6f75e34bb40e62a6b7fc6b3bb68573fc9ae30bacc4e60a641b17ec90391","horizon":1,"input_positions":[9,19,28,38,47,57,66,76,85,95,104,114,123,133,142,152],"input_token_ids_sha256":"bf4865a5ba31ed871b7bd8e3db64daa50d06ca08cbecab0fe5005d6b1a8347ad","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,20,29,39,48,58,67,77,86,96,105,115,124,134,143,153],"target_token_ids_sha256":"68567590e4faf2baa82a41a27758544c2001b1f8b881cbca709b0b85b1bb0ad4"},"formatted_content_sha256":"d8535b9c9deb650f87372133fd68f93792a2190a2ef9fe56ca55a611be6956e0","generator_receipt_sha256":null,"identity_record_sha256":"19a48bcd8ab87c826d9cf5e9160f127b2f1f1be8e45a068c8e44863d1648ee1e","prompt_token_ids_sha256":"b21de4f99224333b5f102dbb403f23e8367304e199e27380c91e192867b7cc1b","ruler_category":null,"seed":null,"selection_rank":1,"selection_sha256":"0172341ebe7bf74bb15297781b17837005ed9a9efbcb89d7bac34d2b25000aba","sequence_length":154,"sequence_token_ids_sha256":"b4da2809a0dead622eddd5200a89a8d4ffcc43b2c108c805e70f4449f813d222","source_content_sha256":"27e38eda1acb40e360eceaac200a88d7b2850555fa05a1efc123ec8c4a915f21","target_token_ids_sha256":"348ffb18662c2da5694709279e9caf22da5687654401cf9aa365e6382bba0ba8","token_span":{"cache_exposed_start":154,"cache_exposed_stop":154,"prefill_start":0,"prefill_stop":101,"scored_start":101,"scored_stop":154},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e0ef0af8afe4b6c4a81f3cabbd81a8ee7f213579fe41ab99f6103d4b0219b7e6","anchor_positions":[9,20,31,41,52,63,73,84,95,105,116,127,137,148,159,170],"anchor_positions_sha256":"fa5efe05ea2fd4bb11defcbff4d3edc28a1c8dc11911a2450816825e972052d2","canonical_id":"657","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,30,41,51,62,72,83,94,104,115,125,136,146,157,168],"fisher_boundary_sha256":"68596f206280b567de8e4d47c206f7bf68a9bb74b4435cf78f400f9f1ab7d29d","horizon":1,"input_positions":[10,21,31,42,52,63,73,84,95,105,116,126,137,147,158,169],"input_token_ids_sha256":"e3067d1464515f4ac7220f562d34c1c90289997d9fb5fefc653bd53f5f81f18e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,32,43,53,64,74,85,96,106,117,127,138,148,159,170],"target_token_ids_sha256":"a85ac1c61455daf925b47063d1ff1f66b29773ceec176e8263f5e6f25a1b3ea1"},"formatted_content_sha256":"058b7f8c5600efa023e5b2daf39dfaf3c3ed7a333e538e1af40e7b0b66c74221","generator_receipt_sha256":null,"identity_record_sha256":"53d315ac0c1c29a162fde0f0fcec7febb3700ec10fef28e6ed4749c118c383e8","prompt_token_ids_sha256":"d229c54edd42db7f254ae246f41c7e3e4207ee238c734a8f628e2b15ef6d2ea8","ruler_category":null,"seed":null,"selection_rank":2,"selection_sha256":"01c4ac54c87a838a046990fc691e469914fa8fffc3bd5230e6a5954f963ce09b","sequence_length":171,"sequence_token_ids_sha256":"2cc570399bb13417211fedf4c35131117f7d22b47aa4bd94e603f6303bbecf3d","source_content_sha256":"87238f15d4968d72a1caf68b3a7d6d09014ca1cd79cd8a42053ba1d0df51fe15","target_token_ids_sha256":"470c291e9349adfcd9620fddf232ec6756cdb6300d975612d2a94ad69e7deca4","token_span":{"cache_exposed_start":171,"cache_exposed_stop":171,"prefill_start":0,"prefill_stop":76,"scored_start":76,"scored_stop":171},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1a950a1c62dc0a9cadc655d3670aa54a1e6cb647d9a1892093d06959656b3aa9","anchor_positions":[19,40,61,82,102,123,144,165,185,206,227,248,268,289,310,331],"anchor_positions_sha256":"42314f586870267235ff8a21de13271c7bc12dbde0efe06732f5f12b3dd32240","canonical_id":"702","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[19,40,60,81,102,122,143,164,184,205,225,246,267,287,308,329],"fisher_boundary_sha256":"1c176e9d1164044fc7633185803af174aed6b6dde3205b6a93344da5dde70a67","horizon":1,"input_positions":[20,41,61,82,103,123,144,165,185,206,226,247,268,288,309,330],"input_token_ids_sha256":"6439978ab1e3336355d0e247c88f80b5fe217a10e6f08dbf7d49232dc4526902","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[21,42,62,83,104,124,145,166,186,207,227,248,269,289,310,331],"target_token_ids_sha256":"1e85d0fddb9002f2699f987057a43eb9c49e9fa5ef1f342f1e03060dd826b7df"},"formatted_content_sha256":"154dfb8b73bb69a8b67592d334c059c4ce3fa2a8dfe9716538e93896567d0906","generator_receipt_sha256":null,"identity_record_sha256":"08e19720136cb0b9e54d61f609f904ecacefee69429d33d7a9794d8b46085bf5","prompt_token_ids_sha256":"bddadf4e931724e7fd559f3f6b3d7128e8ca27b120efc2124cf05a63e7b2f6c4","ruler_category":null,"seed":null,"selection_rank":3,"selection_sha256":"01d2ae83b6bab88a4805a43141ed140fa1eb40a0d24f5579edffc9db44efe0e9","sequence_length":332,"sequence_token_ids_sha256":"91c517b2851f293a1c925d44556ecdd49cf6fc1874d095521f4dba829c5d5237","source_content_sha256":"97404d0bd18d3e30402ad3b46bee420986959afd67ef272b725a56c88634eec4","target_token_ids_sha256":"c94cc439fcffac9c155039ed067b4cf6d4dccca08e579f728aaf004b72be2d8e","token_span":{"cache_exposed_start":332,"cache_exposed_stop":332,"prefill_start":0,"prefill_stop":151,"scored_start":151,"scored_stop":332},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3ed130f8f695b8d6eb298a7742bd1582483350b3fd739da368175309b21ef0b1","anchor_positions":[8,18,28,37,47,57,66,76,86,95,105,115,124,134,144,154],"anchor_positions_sha256":"dda22fb55d9a1168e0db21fb1e43033ab66ab6d1c06af8fc27c118e4671afeeb","canonical_id":"651","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,18,27,37,46,56,65,75,85,94,104,113,123,132,142,152],"fisher_boundary_sha256":"9efbc09f1ef4234ec391fd7fee3a2a47a03dcf28666c0aa550bfc04ea6bcde0c","horizon":1,"input_positions":[9,19,28,38,47,57,66,76,86,95,105,114,124,133,143,153],"input_token_ids_sha256":"b5b770374e871c6af58aaed89d7fdf049c3423bc742e960e5f9a6f4e991faf81","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,20,29,39,48,58,67,77,87,96,106,115,125,134,144,154],"target_token_ids_sha256":"8271372b97b8f4205c2218769701468573205610c8d62b3474ad0d357b160840"},"formatted_content_sha256":"1ccb4c308ca379db3003d12f2a08486a59d19d9a104a1208f6f91af5104409dd","generator_receipt_sha256":null,"identity_record_sha256":"aa519005f371474709ebcac572071735625e58adcc2ea5484d987a610bd03f36","prompt_token_ids_sha256":"d3dc05d28fe582fdfab921a7ad3bac6d80d1d9368804e5955561df1f18f29e1d","ruler_category":null,"seed":null,"selection_rank":4,"selection_sha256":"030870e9654f434f272f49b61535aad38a9c0eacc093ccedb4b1328a163d00f7","sequence_length":155,"sequence_token_ids_sha256":"b4fc5ea0199d736cffaba0bfb4e5db058abd8417d844063350138206ca085d62","source_content_sha256":"c093103b7e2b0fc0cff523616e4dd14264349d2add37f8cc2e248dbd630be4f3","target_token_ids_sha256":"39e1f51bdd6ad42a474c0dde935c2ac6a6696b58b1ccc288e77f6bb849d587ca","token_span":{"cache_exposed_start":155,"cache_exposed_stop":155,"prefill_start":0,"prefill_stop":117,"scored_start":117,"scored_stop":155},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"30f57c28b1d21a916283d0e281b04bbb0087858a98279771835d257c5187478a","anchor_positions":[17,35,53,72,90,108,126,145,163,181,199,218,236,254,272,291],"anchor_positions_sha256":"8d9aa0aedd74a7302174328c9e98110efb9351fb25a7e9f001ca338898298fc3","canonical_id":"720","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,35,53,71,89,107,125,144,162,180,198,216,234,252,270,289],"fisher_boundary_sha256":"c47ca1eef3f1024a8ccaaa6a3156efd81373c0981d1b7794d0e74a6a4596fa0e","horizon":1,"input_positions":[18,36,54,72,90,108,126,145,163,181,199,217,235,253,271,290],"input_token_ids_sha256":"699513d81a38caf1a3166032551a6daa9d12868c8369d97465ec8d642a321141","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,37,55,73,91,109,127,146,164,182,200,218,236,254,272,291],"target_token_ids_sha256":"12845035b9a29a7b08b584abc1f28701bd58dc87b1360013b830d1863deed34d"},"formatted_content_sha256":"08569657d1fd1af97815b4671f162ad3cb646b9db6f4ade1b53b237267f3360c","generator_receipt_sha256":null,"identity_record_sha256":"4c68c0c7fd3b4f5ef68188ca20cfd016414d07212968996c98d64d097d36f013","prompt_token_ids_sha256":"41dcd0cbf8f176b6eabaa2189a2cc9dad19e1d0e1b7466a57fcee4fcd8695223","ruler_category":null,"seed":null,"selection_rank":5,"selection_sha256":"0331ac5ad147dee4ef983f58ff124f388c082a848eb8f7a1e2df5c58e3096b83","sequence_length":292,"sequence_token_ids_sha256":"f0361fd70f4b1f9d2a2cbb95c296333e1e9f146ad837ece7cb669c2993a27bab","source_content_sha256":"e916cd640a7665ee439bd983ca0769129de74af589e7e5eba3b559f7ea146c60","target_token_ids_sha256":"bf0037e9004b38848b545334c18c95d9234edfda314a7fc61ac5ba9fd36b25af","token_span":{"cache_exposed_start":292,"cache_exposed_stop":292,"prefill_start":0,"prefill_stop":240,"scored_start":240,"scored_stop":292},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"076f7a7d27bd0edfad63c949645fb3eea8f9d8d32c238b4af19053c5407a8b2b","anchor_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,160],"anchor_positions_sha256":"cea919870992245499fb8d8a858d848e1e76f02a4e2230ada935442533c03198","canonical_id":"903","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,18,28,38,48,58,68,78,88,98,108,118,128,138,148,158],"fisher_boundary_sha256":"dfb2af41b7aacbbc1cb03c5a460d70edf42a188290ce3b01ee741cbb663a7aef","horizon":1,"input_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,159],"input_token_ids_sha256":"8e0cb14d6334e3493c75e96c63848e991fa05b6bc027f04b99b09dd2cbddaf7b","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,160],"target_token_ids_sha256":"6800fb22eb04b0dcd2b08692779155cb065925a066e2f97f52600c51c38ed71b"},"formatted_content_sha256":"f296f132da28a9fa5eeea0a099d9ad97955044e13b9fe30f6d9300ff9e041c78","generator_receipt_sha256":null,"identity_record_sha256":"a64089c9832bde6f4f730dea47c3c93a3ac71dd538c45b2169607180571708d9","prompt_token_ids_sha256":"a454237da9066ead36df97766389f9b28ac1b5df54bd0d9073baa9459aae696d","ruler_category":null,"seed":null,"selection_rank":6,"selection_sha256":"03a537026beb3ae778851b82f172083dc21eb07f0ce7d1683756019ad86e6d98","sequence_length":161,"sequence_token_ids_sha256":"86cdcf0d01cc99b913a8495e4074583c23687d2a77e68c6b0cda5700932a8937","source_content_sha256":"edfb4574ba6b7cb41c0f0d1dc6452e8ac7fea922b120c3ad057a2e77b15fb36e","target_token_ids_sha256":"5fb56e9b4dfe8c41e4ff0043461c0b44248cf165cac01b9f42beb4f0e756f516","token_span":{"cache_exposed_start":161,"cache_exposed_stop":161,"prefill_start":0,"prefill_stop":83,"scored_start":83,"scored_stop":161},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"6ed1f32d878fa277d86681ed3b0a9e7b9b172c3cf90d4213020217eb3296cf68","anchor_positions":[13,27,41,56,70,84,99,113,127,142,156,170,185,199,213,228],"anchor_positions_sha256":"e3fc0847eae8451ba50c7fae586b6479de5eec6dd1bcdf44504db22f0d700d62","canonical_id":"918","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[13,27,41,55,69,84,98,112,126,140,155,169,183,197,211,226],"fisher_boundary_sha256":"53c76b32efa019cb22ef480e27bc5b9dc7c12c0683b0d1b8ad250ac6571380a9","horizon":1,"input_positions":[14,28,42,56,70,85,99,113,127,141,156,170,184,198,212,227],"input_token_ids_sha256":"14e214e41d049733a9ea7e8c02fce42e7aba76edb16178e6d3405bad4c23055c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[15,29,43,57,71,86,100,114,128,142,157,171,185,199,213,228],"target_token_ids_sha256":"24028e01155b1e01eca8ccfc7c2080a6a56b4a3e7c15839043e13b8a00a8ee8b"},"formatted_content_sha256":"f4ed4188dc61ab3e90526a061246e459b1d5fe2fc68d01226a124501eec0be2b","generator_receipt_sha256":null,"identity_record_sha256":"4f00e2535ec2aa52070ed18120472b9d45845790809ac1fb5a7a26a8f6b51d20","prompt_token_ids_sha256":"d8f70964c5c840cc4c74b02a5c432adb08de3a38f706cca7ff9f50f513f616e7","ruler_category":null,"seed":null,"selection_rank":7,"selection_sha256":"03f2b70cb41aa2df2ca06599a3740907c1bd89976be4f7d79613cd3428eb75d7","sequence_length":229,"sequence_token_ids_sha256":"b42ae62c8481ac868fb8f833b03b35ab7f84490d87329022e4548df9b8a49299","source_content_sha256":"1c74c6e9767910ee576993d40b0be7e4f8a3653223ce943b2b48da881f043551","target_token_ids_sha256":"c0130989066437b78efb8eb68c747a3645ecb631cb508775b1f7b001deb4e1e0","token_span":{"cache_exposed_start":229,"cache_exposed_stop":229,"prefill_start":0,"prefill_stop":96,"scored_start":96,"scored_stop":229},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"32d8375f975ffd5157bb123bebf58d926b2fb6e715d79b1c29711866e8ae1d7a","anchor_positions":[11,24,37,50,63,75,88,101,114,127,139,152,165,178,191,204],"anchor_positions_sha256":"48082390de232d2e911509462169a4b3ebe8a03329ea579a66fe38623c52b3bd","canonical_id":"823","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,49,62,75,87,100,113,125,138,151,163,176,189,202],"fisher_boundary_sha256":"ca339ac21962b12f69ea2e43cd5efc5fc9f045e43e311f236cff0f3fc7a9532c","horizon":1,"input_positions":[12,25,38,50,63,76,88,101,114,126,139,152,164,177,190,203],"input_token_ids_sha256":"9060d0dc9e82be8da8cc47571caa8e03245ea38a15689377bf15f3403304608c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,51,64,77,89,102,115,127,140,153,165,178,191,204],"target_token_ids_sha256":"9c64639807a2aba57e44b252f7c9204c7994ff505668345c293c352b795673ab"},"formatted_content_sha256":"72e3934a85cf2d4cb4b035ef066f9a69edbe304668167f633c705ac5a7c3ee74","generator_receipt_sha256":null,"identity_record_sha256":"6775751ca115709c8dd0187f2d2309b90237ea599613ee56420cc077c5a86d67","prompt_token_ids_sha256":"a7322a6a594060b8983da2adc969063889ad65c9524cfc9d9ace5e4856362612","ruler_category":null,"seed":null,"selection_rank":8,"selection_sha256":"04318f2e0a32de485412bc0aa95414984aabdb5973974b77667909ca919923cf","sequence_length":205,"sequence_token_ids_sha256":"c39424dfee203bd58c46a3bbaafc51b57eb8c76c9f8ad175fd4355d227510f19","source_content_sha256":"7be1174fc06994281445e57b76099586c2214066f9358928e22281d793715d3f","target_token_ids_sha256":"c0ca3637b8b1da8af549e87df7bf8e67da2eb444227889016f2678ec5d8b420b","token_span":{"cache_exposed_start":205,"cache_exposed_stop":205,"prefill_start":0,"prefill_stop":120,"scored_start":120,"scored_stop":205},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"bb690c1557693c9a1151a55a056b413abe207a0ede6130a1c4eee76cd7832910","anchor_positions":[8,17,26,36,45,54,64,73,82,92,101,110,120,129,138,148],"anchor_positions_sha256":"34299c685b72af2a076ee41a9cc49b613c0e3c8039202f3412e50762385cd13f","canonical_id":"962","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,26,35,44,54,63,72,81,90,100,109,118,127,136,146],"fisher_boundary_sha256":"f483b9a7b54d8e98c4fb81ac8e8c508a85638477f08243915cc53d3ee584a92e","horizon":1,"input_positions":[9,18,27,36,45,55,64,73,82,91,101,110,119,128,137,147],"input_token_ids_sha256":"562a3beb47bbf20a72e84b88e739d67bf0a6fd83ea3ec4a06ce7d8527bc32c9d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,28,37,46,56,65,74,83,92,102,111,120,129,138,148],"target_token_ids_sha256":"8d91bcd0f3417c6e6ff0e30b8ef2ad4a5200ab497081407b0de689fac826fe32"},"formatted_content_sha256":"35f9c6bf72383681061847ec15dc2a4d69a179546f872d72042b1dbeb7eb3c94","generator_receipt_sha256":null,"identity_record_sha256":"7f528a9f6b6f25735216e2ddd0cf3bc970efee42f4ddc18607d7615c585b5da5","prompt_token_ids_sha256":"ad2690a82eb727187859d0918f78622936f9220218fa708cc305b79f9e3129b2","ruler_category":null,"seed":null,"selection_rank":9,"selection_sha256":"053a8089119cb6c0a1699960c729b5d0bbf904347c2611f2bdbc20d68ac98896","sequence_length":149,"sequence_token_ids_sha256":"39192da46d67ba8a6a1aecc11d2517cb72d4adbd21944f31d854f4aebde4f950","source_content_sha256":"6d510f0a092393c876a9d3827db9cccd4b422742e08dfee9a9094f3786b67fbf","target_token_ids_sha256":"24d4ddf4dc36fb254790754f2826967503025402abfe4dc1c81a57b608e358c1","token_span":{"cache_exposed_start":149,"cache_exposed_stop":149,"prefill_start":0,"prefill_stop":87,"scored_start":87,"scored_stop":149},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"510bbf0acc641074d0ed15a7484a726b97a35b663fe88ff076ca0808ea574c8f","anchor_positions":[5,12,18,25,31,38,44,51,57,64,70,77,83,90,96,103],"anchor_positions_sha256":"307c6a1944f2d94c59465c6af96e251aaf0ea665f8f1a3e0e55e3eb0568d8551","canonical_id":"641","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,11,18,24,30,37,43,50,56,62,69,75,81,88,94,101],"fisher_boundary_sha256":"74d1a793166b47319560030b1ea4243c84cca2ef770c4bd539a7069d57a6516c","horizon":1,"input_positions":[6,12,19,25,31,38,44,51,57,63,70,76,82,89,95,102],"input_token_ids_sha256":"693ad260a9464069c62f03bdc9f97044efaadaf73fba426e3993f726d373516e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,13,20,26,32,39,45,52,58,64,71,77,83,90,96,103],"target_token_ids_sha256":"24fc6cf6c761d9d2d39f6a0e2d1988fd8c0568a28c602b83a298a2efe14f5b75"},"formatted_content_sha256":"0d5d8389772326b9a23b37c5d64023b87ff76167c738f0f04a73f350d75146a1","generator_receipt_sha256":null,"identity_record_sha256":"d257e6e664de7884c47c6c8efd30d9049f908be0c1f0f15604c5b1e680278db7","prompt_token_ids_sha256":"8b73365de43af24e2b250e80fbb6c87c821ce915961cb60bb5f92dd492aa4e25","ruler_category":null,"seed":null,"selection_rank":10,"selection_sha256":"0561b3053aedb5bf75595d41c4c1d56dffc9d7d22d4f594c77acd661612fbbda","sequence_length":104,"sequence_token_ids_sha256":"edaafb16efe7861b6641b4284c1b03cabf4553b33c20400911590227e2717873","source_content_sha256":"b4af7687b1fdeeba11f23fbb43c11d07ffdb4918405f0574dba17f6932087d51","target_token_ids_sha256":"10c4c8e1ab1664e434fe26b8093fe1929571437707a4a4c5082c71bb6879f108","token_span":{"cache_exposed_start":104,"cache_exposed_stop":104,"prefill_start":0,"prefill_stop":80,"scored_start":80,"scored_stop":104},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"77e62d656a470f85ca90c98dfc91b52de62fec0d714ab54d9d78b6111588449a","anchor_positions":[16,34,51,69,87,104,122,140,157,175,192,210,228,245,263,281],"anchor_positions_sha256":"d3c1ef41c3fc59649a4da8be56fc51bf8a16621374c8218a7236e1c502a9ed29","canonical_id":"821","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[16,34,51,69,86,104,121,139,156,174,191,209,226,244,261,279],"fisher_boundary_sha256":"6ce814085aa258c14254daa52b6c87e5c35bb95bb9ce43819236089ed670a938","horizon":1,"input_positions":[17,35,52,70,87,105,122,140,157,175,192,210,227,245,262,280],"input_token_ids_sha256":"52e2a2814003d5a0c9558d2807c349140c1daee27300a41a386f98243a887244","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[18,36,53,71,88,106,123,141,158,176,193,211,228,246,263,281],"target_token_ids_sha256":"a841d33aff758b3642820a6e5c0b714add3840501814ab31d90f6dc5b41336dd"},"formatted_content_sha256":"9b4c39a7f47e7c5485594bfc1f0bc255be70f74e8e73be2bfab32e77bd11cb37","generator_receipt_sha256":null,"identity_record_sha256":"bbe20a7f23dfdd53afdd97893a491105ab3d0d770f28da4cafd4d0f760a10e07","prompt_token_ids_sha256":"8438715e3ee27d1c3e5e158afeb22e85f9543635b4c24c7151828b62262082b1","ruler_category":null,"seed":null,"selection_rank":11,"selection_sha256":"05ca9ce71ec3ccbff2dc9051e6519ff621aeaf051f13006ef2a903db53cfa322","sequence_length":282,"sequence_token_ids_sha256":"d4a708e94f01dc74ebdbaa4a0850da22f19f8c7616df125de0fa93a03cd941d1","source_content_sha256":"6cdab710688743670db0377d5ff8806298470ee86aa94a2f4fd430f2542e26b4","target_token_ids_sha256":"52fee9ad2633a9725f62006b3a2a598f494e32fd6e12dcf43893a911e7032a91","token_span":{"cache_exposed_start":282,"cache_exposed_stop":282,"prefill_start":0,"prefill_stop":246,"scored_start":246,"scored_stop":282},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"88e7f4c26333024cf786b4c1d640f43c25aadabb8055c4006984b0b773283a63","anchor_positions":[6,13,20,27,34,41,48,55,62,69,76,83,90,97,104,111],"anchor_positions_sha256":"70a9c6d084ac7d3e411adc69adf277b706895b48989c28a96164561595a44b13","canonical_id":"882","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,19,26,33,40,47,54,60,67,74,81,88,95,102,109],"fisher_boundary_sha256":"af33587dbbed535e25e0f9ff381dc426652e98f76482f907c3a1b91a43afe65b","horizon":1,"input_positions":[6,13,20,27,34,41,48,55,61,68,75,82,89,96,103,110],"input_token_ids_sha256":"2e82b13f62352679ed5dae9b03694e1c2eeb56e00f762a0373672ed48440f408","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,21,28,35,42,49,56,62,69,76,83,90,97,104,111],"target_token_ids_sha256":"d9c7534eb6dbff16502f2748b6ab6576ab5d76645faba1e8bc35a67323be8c39"},"formatted_content_sha256":"8164f8fb3728db418776886f3233778e01a1c5b7e54b4f350ff9c98d6f4c3018","generator_receipt_sha256":null,"identity_record_sha256":"79d822ab45c87b51b8aa2c0ecb29bfbbd85453f82f6a3e764bec8805d2f9708b","prompt_token_ids_sha256":"4c61f175c320b5f738697b92438e1bb158f00bd6a061e49a529ad3158d4f02d5","ruler_category":null,"seed":null,"selection_rank":12,"selection_sha256":"06182bf7b5db58b26f203c4f99a7fb97d3252f5d92dd5f4333d47e49043b0f9d","sequence_length":112,"sequence_token_ids_sha256":"a69b8ee713bb9e4fdf0cdec61d49a10f148243a2cf8fe92a4420823b82bb3839","source_content_sha256":"f9edf8d912223ba1c68389438b5a930e3ff8116a79ef7e9e4ab5b7f48cad2e1e","target_token_ids_sha256":"b9330c3608f52265a4d54352f0d4d3821513a3adbea5b89bf9f5146143a012fd","token_span":{"cache_exposed_start":112,"cache_exposed_stop":112,"prefill_start":0,"prefill_stop":90,"scored_start":90,"scored_stop":112},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"026a11f123c6c02b349d154973681d1484316586c77571635cb75396928e8801","anchor_positions":[8,18,27,37,46,56,65,75,85,94,104,113,123,132,142,152],"anchor_positions_sha256":"75c1fd0cbacbbfa75bfedc1647ca1a15436ad446c2021b146a8d7af4567321f4","canonical_id":"874","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,27,36,46,55,65,74,83,93,102,112,121,131,140,150],"fisher_boundary_sha256":"69de8bf124376d38cfd76f8801bc74e0e352d8cd33f8dc6b34cb10e4c6ff73d3","horizon":1,"input_positions":[9,18,28,37,47,56,66,75,84,94,103,113,122,132,141,151],"input_token_ids_sha256":"75967a14501363b155570c3d4d66959cde5cce50c53c8fa1fadf71f39e4218de","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,29,38,48,57,67,76,85,95,104,114,123,133,142,152],"target_token_ids_sha256":"38400510781c2bec0042957302068be57d0438f8b57f3a4a775204a7ae26b03b"},"formatted_content_sha256":"8ee29791043cd7726bdd815083a307f0d693443f77d0fc9af6c9249422f82a60","generator_receipt_sha256":null,"identity_record_sha256":"5f505f1a07e986bb3a69f547553e68edda687e89817230981e3d65a20f3b5d61","prompt_token_ids_sha256":"af780c9a24c687de5d7bb86c8de2e3a05d07e1e769e2bed9aef9bc82e0c2b3e0","ruler_category":null,"seed":null,"selection_rank":13,"selection_sha256":"06e6c099b79a31351c8aa115e8c35681452320d265ba3e7c9deb647189b5aedf","sequence_length":153,"sequence_token_ids_sha256":"059517b1e6703c7276b532d66d5dc53b137178142a0fbcebbf7a85ed806bb2df","source_content_sha256":"7be81d34f54ea71fa53a94438dc656b1e14cf83c1f0cb8b85a03aeec675a95cb","target_token_ids_sha256":"a00664d6a993b427df9b77218f9f6ab241e98e255c2754127a6aeb30818bf82b","token_span":{"cache_exposed_start":153,"cache_exposed_stop":153,"prefill_start":0,"prefill_stop":82,"scored_start":82,"scored_stop":153},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7e8e992a57a2c58864e789d43316ec4ca5557685022402a123eb7bec86e0a31d","anchor_positions":[8,17,26,35,44,53,62,71,80,89,98,107,116,125,134,143],"anchor_positions_sha256":"486e7f47d72e2b3d4a862156a2b41e9dbb05a70d78349fdcac0b01e09667750a","canonical_id":"768","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,25,34,43,52,61,70,78,87,96,105,114,123,132,141],"fisher_boundary_sha256":"2f57f4c3f66cd0d31d7220f66c2563c2f42ed9d860ccd032cab629a4a62b2d7d","horizon":1,"input_positions":[8,17,26,35,44,53,62,71,79,88,97,106,115,124,133,142],"input_token_ids_sha256":"951070a8ec9f2842c988081eae148ff3e650f6e041bad423eb8cf80577aa09ca","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,27,36,45,54,63,72,80,89,98,107,116,125,134,143],"target_token_ids_sha256":"be7a411acd1f8c53e98dbbe661a8f0e08925a5287a3a082d567b4348f5daea4a"},"formatted_content_sha256":"19fb2a15b1650f22847079c098721ed0b4ae36db1faae49cb10e7f8b7f8a951c","generator_receipt_sha256":null,"identity_record_sha256":"4f37dd32d3554ba3f0e3842355c0812d1532bf5eeb4d09edf28e778ec744907b","prompt_token_ids_sha256":"5abd62e58720f9b71da73a2f319d99d1dd3f0fe89c148c1e335674d33e0db6bc","ruler_category":null,"seed":null,"selection_rank":14,"selection_sha256":"08d405d25f1c1656f2532ca7d15875b63c46516c4e46a564326b3696f5943347","sequence_length":144,"sequence_token_ids_sha256":"d924aa1344bedd7c5de7beb28914b5f02d01c9242e93916c3ebf5e13a1b777e2","source_content_sha256":"96a4ff323c23dc1eba72d4b800a65cd77f46bad4c7b017e7716f69b066cf251e","target_token_ids_sha256":"2bb90e5895fb397d1b3239e2f50aab932ff08813656890b44cefc5327ce9f0fc","token_span":{"cache_exposed_start":144,"cache_exposed_stop":144,"prefill_start":0,"prefill_stop":79,"scored_start":79,"scored_stop":144},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2a32d45497a46d8765c2bef47acddb45c96e41bdb29282abb99a946dd86db58b","anchor_positions":[15,32,48,65,82,98,115,132,148,165,181,198,215,231,248,265],"anchor_positions_sha256":"0348f2e9e936ebff365da2919e86c0d7b23f030deacd2557e2975ff2c1bb6238","canonical_id":"635","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[15,32,48,65,81,98,114,131,147,164,180,197,213,230,246,263],"fisher_boundary_sha256":"bbb2193a791665a372d6f0871c8b972270ed33e5afe4d627b3f13c69caa5172e","horizon":1,"input_positions":[16,33,49,66,82,99,115,132,148,165,181,198,214,231,247,264],"input_token_ids_sha256":"54ae9eda3529af89c8491dfa14c002e2bbbddacc5f7fc9bd8f98dfd841106b17","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[17,34,50,67,83,100,116,133,149,166,182,199,215,232,248,265],"target_token_ids_sha256":"8e9dff8e5415e27d61633f2fbae7b6f5a404922b80ae0dc2d994004e0c00ff77"},"formatted_content_sha256":"252812434def51f2af7fe0f324a5922cddb2523bd81d2dfc65720c67ba9d9871","generator_receipt_sha256":null,"identity_record_sha256":"e2908b95f86b0b763a5d77e597f5d8cc0e9ff3b96e4129645e8ee178160dc9e8","prompt_token_ids_sha256":"99adc1c15eb492246a3ad45eb9cd6dc0a5f949c95b99aa332c2c8fe742dac882","ruler_category":null,"seed":null,"selection_rank":15,"selection_sha256":"09bdeb42fd2df571b9c13e3aecc461eec43355320ee267a53066bcbb23769161","sequence_length":266,"sequence_token_ids_sha256":"2ca3fac260d1b58347c86dbb21f794efef0df7c42a0de4d942d549481c583b2d","source_content_sha256":"b3bbd3ae7ff06e057b52c17af8b75ac83d20e8b436ca1c82f6f7ccf94d0a9214","target_token_ids_sha256":"b44b18d625486bf3a519bc0a87d4dbfbfb4851798e7e50bacb07a66cab69939c","token_span":{"cache_exposed_start":266,"cache_exposed_stop":266,"prefill_start":0,"prefill_stop":214,"scored_start":214,"scored_stop":266},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e6416dbb61ce7ce16cf3fff7ce25a1bacfecbb3dbf2fde6f06d8fc6699998cd6","anchor_positions":[5,12,19,26,32,39,46,53,59,66,73,80,86,93,100,107],"anchor_positions_sha256":"b9647c4baef8fee686d8a882e4d530418b6f7cc8044a0a83370fa394c3bc5ef2","canonical_id":"666","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,18,25,32,38,45,52,58,65,71,78,85,91,98,105],"fisher_boundary_sha256":"1a644132ebeecd04edf2c98217977f7fcbdedbec84e64b6b50a88a6151232772","horizon":1,"input_positions":[6,13,19,26,33,39,46,53,59,66,72,79,86,92,99,106],"input_token_ids_sha256":"32f5ac39d606f1ddf6a4e4346735931cd6faad7feec35e950f17b8edbcb045a2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,20,27,34,40,47,54,60,67,73,80,87,93,100,107],"target_token_ids_sha256":"eed792902f0c9e240b65aaacd0467e2ff2f4a7eba83a333214fd67d942feac55"},"formatted_content_sha256":"e3a3c1e1ed1b292d4877ff78da3ca92b8112f6b28b41a2eb63f18c8d72057088","generator_receipt_sha256":null,"identity_record_sha256":"ec5fb87c296ab7d9fc8671afd53e5bc11fd05c7ee0bf20f88d758453cb95da2d","prompt_token_ids_sha256":"92348f208743eebc810c3eeffd27e797bfbebe693b75b29192878c2766c17cb5","ruler_category":null,"seed":null,"selection_rank":16,"selection_sha256":"0ad2bb04e64cf7bafe43963d9c975c7597c00f97a065f172225e36b8de1be401","sequence_length":108,"sequence_token_ids_sha256":"05aa7077f2114bfc39c337134a54171570e1bf7f5e40e875dfd4ccc76cd3c5bd","source_content_sha256":"c5657a0fc079e76b566db9dad6a041aa51a885e91a3ce60f4b56e268d4f2283e","target_token_ids_sha256":"34eea4ea001c4d78cc33cc3368d1d71873c37b7e910507eceb9597f35255e3ef","token_span":{"cache_exposed_start":108,"cache_exposed_stop":108,"prefill_start":0,"prefill_stop":69,"scored_start":69,"scored_stop":108},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b641a12066174c147da77baa5576307cddc4326d490da447977a9880c63d2be5","anchor_positions":[19,39,59,80,100,120,141,161,181,202,222,242,263,283,303,324],"anchor_positions_sha256":"229481c1cae947d75547887104b565fe100ef01406aaa1ae33b6680fa4942d03","canonical_id":"795","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[19,39,59,79,99,120,140,160,180,200,221,241,261,281,301,322],"fisher_boundary_sha256":"075b4606ad628cb7bea3693f59ccdb0efa6c6599cce58a6552534c47baf39ab1","horizon":1,"input_positions":[20,40,60,80,100,121,141,161,181,201,222,242,262,282,302,323],"input_token_ids_sha256":"23b5ebefaddbeeafb86642c66656119a5122ac4590adfc291caa30b36d9aa0f0","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[21,41,61,81,101,122,142,162,182,202,223,243,263,283,303,324],"target_token_ids_sha256":"67fe7ce1a7b687bd4b008457e44ac3e68602c818d9241e2c029c21d035fb039c"},"formatted_content_sha256":"ba441afe1323472e606fbc265ed2df392c0910aa7b69c72034eb237f6f1eac12","generator_receipt_sha256":null,"identity_record_sha256":"462faa7c85aef3e98a366b53fff20c26ce10eaccece6d0cf5e990a957d47bab0","prompt_token_ids_sha256":"276d4bcbbf513e902b3e32cea17826db3f7c89340cf3d4a5944f77e4fe398378","ruler_category":null,"seed":null,"selection_rank":17,"selection_sha256":"0b64b061a95be11408788bd7d48178a22fe67b04451aacb2824273de604613c6","sequence_length":325,"sequence_token_ids_sha256":"3bed81ccd0aff452ebe2792b7538c8dab5d74a0ddf5fc9de5e67201d559dd9f4","source_content_sha256":"0c82e760835310eb8ee865c643c6bbd2fe9a92e58f43e818f9fe857fa8bb2b34","target_token_ids_sha256":"07898ad18dc9917a62d1d7d2c673264e506ffa3cd0296f61f5355d73ff74e2a1","token_span":{"cache_exposed_start":325,"cache_exposed_stop":325,"prefill_start":0,"prefill_stop":290,"scored_start":290,"scored_stop":325},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"73863421843464e12bf37b9bdf19118b2a0edae3ea6b28b7bb83ae34025fdb99","anchor_positions":[6,13,20,28,35,42,50,57,64,72,79,86,94,101,108,116],"anchor_positions_sha256":"65c4371e0db2357f72608fb79361ad62ba950063eae1f1df95567c3178cfceb4","canonical_id":"944","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,13,20,27,34,42,49,56,63,70,78,85,92,99,106,114],"fisher_boundary_sha256":"8f181fb4592d87100a01b9b5022fe28dec60a6c1a630f0253a95ef609f751477","horizon":1,"input_positions":[7,14,21,28,35,43,50,57,64,71,79,86,93,100,107,115],"input_token_ids_sha256":"bb992854b746a2d0322b0c73747a5b6e481b4ac55013af85dda8fc8c0bc4e324","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,15,22,29,36,44,51,58,65,72,80,87,94,101,108,116],"target_token_ids_sha256":"efc4327e8bc745b94eb74df47cb9d34c605ce69f88ab01258f192e2678c5c5c8"},"formatted_content_sha256":"588cb6ded28fd05719867c9a3c2ef078440822880f6fbe41587f8ab604fc41cf","generator_receipt_sha256":null,"identity_record_sha256":"a019ad696f9c4fe577d6efe7e08f14e983f53e6a023062d550dc4ffd16fef99d","prompt_token_ids_sha256":"6430dca518b59b80172b9552c64bfa32b6116fc5855d53c075e496d174d33f72","ruler_category":null,"seed":null,"selection_rank":18,"selection_sha256":"0b9a44c3022b9f3f3616c768c1535452329656aeea799dcb45a719751b3126e4","sequence_length":117,"sequence_token_ids_sha256":"a3b8d96a49a5559eb5401f9e9f9b40c0785ee84f73cf189a884618ba0cc77865","source_content_sha256":"2099f30aac30b18f4a8fa27a7204c64734c587cf50057d835eae933211f12c7c","target_token_ids_sha256":"73123ac6d971eeb25450ba23964d4d95870b3421781c2b721b8e3f5217fe1ffd","token_span":{"cache_exposed_start":117,"cache_exposed_stop":117,"prefill_start":0,"prefill_stop":91,"scored_start":91,"scored_stop":117},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"a3fb6da730653e3034c80d7d7da56cd1a27244994198f932d4b653175e07528d","anchor_positions":[16,34,52,70,88,106,124,142,160,178,196,214,232,250,268,286],"anchor_positions_sha256":"88ca5e0c73dcbbad356df2a5c320d1e65852f5459c97030e41b86ba39ef13c38","canonical_id":"653","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[16,34,52,70,88,105,123,141,159,177,194,212,230,248,266,284],"fisher_boundary_sha256":"f3b0c45ebf98940a7a54cb33ffec71487e06a0482e824289ffd032de00327ea1","horizon":1,"input_positions":[17,35,53,71,89,106,124,142,160,178,195,213,231,249,267,285],"input_token_ids_sha256":"d7dccbe6c436987756ad6166a8d563fa686494edff0e090f19a94996077cdfc1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[18,36,54,72,90,107,125,143,161,179,196,214,232,250,268,286],"target_token_ids_sha256":"8493cc8e2ef026b2ca43108aa6e7db63ec761ce211452609a1bb2760a0f66f2b"},"formatted_content_sha256":"157b49e7f32965e28ef9bb626baf1d0356b6d5f38d74f4c673a15e0198ce6751","generator_receipt_sha256":null,"identity_record_sha256":"31193a44e28b9b97aab66d0eff836463790dd908e352239fd1a3dee6e1ad0a3a","prompt_token_ids_sha256":"a3e2cc82d1e964fda655ec40dc4f4058a3f6f4adaa8b6def330531e94950c8e6","ruler_category":null,"seed":null,"selection_rank":19,"selection_sha256":"0c32b016c1fcca024f19127827b8ece87776b1df963f55614f9a15b354218018","sequence_length":287,"sequence_token_ids_sha256":"1d7f3ec55af99be19b85291e5b80c9d9555e9360cd4af99ded037154beaaad61","source_content_sha256":"e8146f7d1066326e2120f2f49e89fe13318dde45bc338d1b6a350b475840face","target_token_ids_sha256":"f4ea61620cf4a9d1cd26e8136680d5f09ad7d79a197bd08433ab22b67e683560","token_span":{"cache_exposed_start":287,"cache_exposed_stop":287,"prefill_start":0,"prefill_stop":249,"scored_start":249,"scored_stop":287},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"adeda0b376d342263631c70ee440fab7010e921ef7e9bc2997f75798d31a644d","anchor_positions":[17,35,53,72,90,108,127,145,163,182,200,218,237,255,273,292],"anchor_positions_sha256":"c72256a78f816fa7bf8e540cc3aaf3fc1795369a3859efe48a9c4a8cce0d2489","canonical_id":"857","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,35,53,71,89,108,126,144,162,180,199,217,235,253,271,290],"fisher_boundary_sha256":"f5b74e46bd51ccc196ee08a6967a1597fb09f1c1b2ba66309ed0bcb627f70190","horizon":1,"input_positions":[18,36,54,72,90,109,127,145,163,181,200,218,236,254,272,291],"input_token_ids_sha256":"555cbfa7f8498b7200297ecda4d0d95381d1e0e366a25c75a729a308b819e751","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,37,55,73,91,110,128,146,164,182,201,219,237,255,273,292],"target_token_ids_sha256":"ded8ec9b50bd3a7f2b508c6b34745e277cf32245748a23e733afeb28ab5be811"},"formatted_content_sha256":"4156119aeb73684201921a09e3d2fef7e8426ebd10babb52788c0f7e8e9813be","generator_receipt_sha256":null,"identity_record_sha256":"670b8008445ce0955e14e6aa12268d4f91c343a57d01f2618609843306761743","prompt_token_ids_sha256":"5e5ccf81697344dfc507d09f593651afc2e0d360fb85f31936aa419b069701de","ruler_category":null,"seed":null,"selection_rank":20,"selection_sha256":"0f3520a5dd05d707070cfc5723f57c992db4df4d8d98965890fa13a516a29c32","sequence_length":293,"sequence_token_ids_sha256":"ddd8c7e579f53e5c52e946378d772b93a5ed813a063da195e831d38abb0cc958","source_content_sha256":"22096f4ec39e2e444074f7f2cc3873ad56b3e27bb407587437fb05693324c674","target_token_ids_sha256":"66d83ee4ec91a16b9bd4d5c77a1b45e1850fd3594f689ead1e55c781506cfb6f","token_span":{"cache_exposed_start":293,"cache_exposed_stop":293,"prefill_start":0,"prefill_stop":271,"scored_start":271,"scored_stop":293},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"fa5b57e6457aed64b16e39b74ac6aa26eccd3fc3019570266249d9c77b045ed2","anchor_positions":[10,21,32,44,55,66,77,89,100,111,122,134,145,156,167,179],"anchor_positions_sha256":"fe99fb9f0105e4d1413d20b0bbf51aa70554cf263494d9944696f0b9ca84c89e","canonical_id":"884","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[10,21,32,43,54,65,76,88,99,110,121,132,143,154,165,177],"fisher_boundary_sha256":"7a8012c30b1d696314102eab8bd4402dcf733de71d67674552a2c672d81c2cc8","horizon":1,"input_positions":[11,22,33,44,55,66,77,89,100,111,122,133,144,155,166,178],"input_token_ids_sha256":"391312b446f027fbfa057f16da7c25100505a8356e160bc745be4b022c43ebba","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[12,23,34,45,56,67,78,90,101,112,123,134,145,156,167,179],"target_token_ids_sha256":"4f09e35ae3b70a36f49caa3032145ff047fcc34b2e0c9615bdfa4ec5effcacf4"},"formatted_content_sha256":"e2917eaf992f7bb16e2280d5b40f59553c293a9c2ff814a21cdbf4193246972d","generator_receipt_sha256":null,"identity_record_sha256":"170955cbe700df9f79cb32d7fd38d8902e1a304a3a7a3844b606bfcee8b8366b","prompt_token_ids_sha256":"676841a61aa3135e7a3ec64d03888de021cc34698ee1b25aebfff640fc502cf1","ruler_category":null,"seed":null,"selection_rank":21,"selection_sha256":"0f7355686d92107703dcf05c4240667b8c62bf7fc1f0c45fe1704ffc1686d685","sequence_length":180,"sequence_token_ids_sha256":"fc621b262cdee5eeb2686915a8d202aba1e489894daef49f381679515598da1f","source_content_sha256":"9cea206d7dc3308fcc17678f64598526990d06b799c3a6060837e47d5e0ac2ea","target_token_ids_sha256":"18f70329692e7510f6bc5c35dd7288681cb2d7fc3a45398cb46fe8c4064fc5b4","token_span":{"cache_exposed_start":180,"cache_exposed_stop":180,"prefill_start":0,"prefill_stop":112,"scored_start":112,"scored_stop":180},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"c56fd94d27ac497a874a2872a9378539bce23e6f40c0c88da8548530343fcee1","anchor_positions":[9,20,30,41,51,62,72,83,93,104,114,125,135,146,156,167],"anchor_positions_sha256":"eaf3a0e814f88177633695c30e9a05b6b492aa64c90bf98da98cb9be3917c812","canonical_id":"878","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,30,40,50,61,71,82,92,102,113,123,133,144,154,165],"fisher_boundary_sha256":"45cdd8b5f6368af3ea5312cd27a1db7a47955c6f64da0b88e27faf49b3c816e1","horizon":1,"input_positions":[10,20,31,41,51,62,72,83,93,103,114,124,134,145,155,166],"input_token_ids_sha256":"800e65a128ae4172d15f3bf9d35062d514957f43f113ef8390515f7bad4eb648","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,32,42,52,63,73,84,94,104,115,125,135,146,156,167],"target_token_ids_sha256":"cf7c2293e05be3baaf208adda2a202c2dac0599b886a00e133b951f71671b17f"},"formatted_content_sha256":"9fa1e6f58327152b9fd6caf18d856e0920f7b962af65f17c0b516a39a15e889c","generator_receipt_sha256":null,"identity_record_sha256":"9c45e96b568ec62ab8d97f8c4b13adc325eb92bf34bc8cb045cbf49deb8d226a","prompt_token_ids_sha256":"280373fa3a2da39e89a082829b753432d7f22d6ef6366e83598df36823d26b55","ruler_category":null,"seed":null,"selection_rank":22,"selection_sha256":"10b149086df38d9067b558f5886d640b68f4204287a8886f65495246268313d4","sequence_length":168,"sequence_token_ids_sha256":"0bd5f59a58f0f806de221fe3c637774e5c9e47f72c11f0b19b9f2cb06d977dfe","source_content_sha256":"44ecb08d42ef8de88bcb0e909c2e9befe4a195daaebf6c8c10e1baaf0438944c","target_token_ids_sha256":"5b3eced18f97c0b17b291bb1e4d2e732f1ef7b1b304ac11beb67409c8f2d3bc6","token_span":{"cache_exposed_start":168,"cache_exposed_stop":168,"prefill_start":0,"prefill_stop":139,"scored_start":139,"scored_stop":168},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2d6e6a2df8f80c87c48f19e05e8d274114304b958532988537c2cfd6c7ba447a","anchor_positions":[11,24,37,49,62,75,87,100,113,125,138,151,163,176,189,202],"anchor_positions_sha256":"0e1a219b40ec9118f3f3cab251874074a9782d1da2d399b19337c5a52ff6788a","canonical_id":"822","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,36,49,61,74,86,99,112,124,137,149,162,174,187,200],"fisher_boundary_sha256":"6667326d144985716907297f9fcad0d67d899e5951cfd594d7c70e6f0b3e7ec4","horizon":1,"input_positions":[12,25,37,50,62,75,87,100,113,125,138,150,163,175,188,201],"input_token_ids_sha256":"70f7035d5ef6a263a7316e7d5fe9a1d64361df51aca8f8b97408bc66942765a2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,38,51,63,76,88,101,114,126,139,151,164,176,189,202],"target_token_ids_sha256":"b72dca6074d7b5b12b92fb70cfe49900c4d0b2869f86c323e978902c8635fa80"},"formatted_content_sha256":"e207600ca200fa35202626097b7ce35aabbc9a909f74a7f4a9634823421b3f85","generator_receipt_sha256":null,"identity_record_sha256":"6f1cb4fcebbe64bdece660cf8e72d4b0d933e1d5e76363e0fc65bc84df091ccf","prompt_token_ids_sha256":"a45819695adc36f87315b6277308921c06ea8a973dd4acd584c641b42a489685","ruler_category":null,"seed":null,"selection_rank":23,"selection_sha256":"10e329cfe9c44c390c0748888b664bc73f711559a095943ffb716a5c49a43bfe","sequence_length":203,"sequence_token_ids_sha256":"7ca67bbfaf0cf4d58cd3fbf7999f0bdea010a29908aa1277c433c3640ccc93fb","source_content_sha256":"c34011770653c990605bfd0878f3caa642227458bbd94408174d49073634373c","target_token_ids_sha256":"d355095fa08c0462073247a017dce85a1e1739895f047802a3f3fba9380d16ac","token_span":{"cache_exposed_start":203,"cache_exposed_stop":203,"prefill_start":0,"prefill_stop":71,"scored_start":71,"scored_stop":203},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"89cd198269ee29a7bf3a758e494f86cdd7b4c8eb634e87418ec24fa6977ffb90","anchor_positions":[8,18,27,37,46,56,65,75,85,94,104,113,123,132,142,152],"anchor_positions_sha256":"75c1fd0cbacbbfa75bfedc1647ca1a15436ad446c2021b146a8d7af4567321f4","canonical_id":"687","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,27,36,46,55,65,74,83,93,102,112,121,131,140,150],"fisher_boundary_sha256":"ce22804019d01ac4a4b1ecc051fc015d0fc2edfdc083fe9b46238115a22d2681","horizon":1,"input_positions":[9,18,28,37,47,56,66,75,84,94,103,113,122,132,141,151],"input_token_ids_sha256":"fc68589c14fc3c7f27c207419457fdea2a3ca0ea5556ab837c7e77422872e50d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,29,38,48,57,67,76,85,95,104,114,123,133,142,152],"target_token_ids_sha256":"2aafef74e6bdf1473faf98f2e2f51258d9876ed6aa14fd3c8d901e57a6a67670"},"formatted_content_sha256":"34caea8f7845c77c8c7c5cb04e37b6b1594df9805f377e4ad243d7b6bd787df4","generator_receipt_sha256":null,"identity_record_sha256":"5ead1d0afb62e76c6d447e766cb0e770b7b3122bc85e14984b945f91559084eb","prompt_token_ids_sha256":"7231469d89e6746cffb3134204ab6225bd06db47e1a21a1b29978270a92818f3","ruler_category":null,"seed":null,"selection_rank":24,"selection_sha256":"1241cc732d851bc4d6e31425c413658747a68ac5bf1df371701b6bac3339de38","sequence_length":153,"sequence_token_ids_sha256":"0ff8fac034540d5fae83061b024846460ae750e94227132be74a1f8b13d0c5f7","source_content_sha256":"570e823f5d18678380cee81c7f8171a7dccbbe1956f576e5bef831ef0417fbfc","target_token_ids_sha256":"9dd750e2b89888cecb138a1c442f682c640b1ce4caf2a92c920fc630aff9b969","token_span":{"cache_exposed_start":153,"cache_exposed_stop":153,"prefill_start":0,"prefill_stop":89,"scored_start":89,"scored_stop":153},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4c2c007c3a069505a8984ea48151216aceab5ff41919fef5caa62a9c95724294","anchor_positions":[5,12,18,25,31,38,44,51,58,64,71,77,84,90,97,104],"anchor_positions_sha256":"3d14c7e60b70361fe3f48971fff7c78a0e3a89500325f79e8590e96f876c6d84","canonical_id":"820","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,11,18,24,31,37,44,50,56,63,69,76,82,89,95,102],"fisher_boundary_sha256":"a5d52b2d880ccd9e8c57da79f29d3e8871447c9521ac6f1afce731259780a7b4","horizon":1,"input_positions":[6,12,19,25,32,38,45,51,57,64,70,77,83,90,96,103],"input_token_ids_sha256":"f6d753d6a779f0b43d8eb24d7213ecc4ab0ee16118bcbc17c3cb053d5730670b","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,13,20,26,33,39,46,52,58,65,71,78,84,91,97,104],"target_token_ids_sha256":"75597ca42c1c682f0e8f425a875b8483f8c45067ec1d096e0fd49b964402322c"},"formatted_content_sha256":"c5473e872de3d1d10c5f923d4246d5e6e41987ce6e60afff50ad39c665b75d2b","generator_receipt_sha256":null,"identity_record_sha256":"bfa5a8c5d9a1e111c3b629cdc0f09fc8a0dada73721cc93beb5ce531328d1f35","prompt_token_ids_sha256":"9edc0e91d3b033f1f21804597df8021ebb7cede1211331d0898ea29ce2062107","ruler_category":null,"seed":null,"selection_rank":25,"selection_sha256":"1269c5ac64a35cc30e3aab0138aa3e2e5cf40fa1a583bd687f3718d0b78632b5","sequence_length":105,"sequence_token_ids_sha256":"d9f20cb4b5b15d6acba099a4e472ad67b130db359488d35bb838bd17655f28b3","source_content_sha256":"1cda14e8c3edcbccfd8ccbee369285b27f5c9203af7f1d0171339a6fc431ddbd","target_token_ids_sha256":"845201cf3c1564028c1f00ea5c73aa8c447fdf960863b76d2f859ed548bcd8a1","token_span":{"cache_exposed_start":105,"cache_exposed_stop":105,"prefill_start":0,"prefill_stop":74,"scored_start":74,"scored_stop":105},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"f689b6853781e9f787183224c1fd3cee86b9e7cc33bd38b263db4768f634cf37","anchor_positions":[14,30,46,61,77,93,108,124,140,155,171,187,202,218,234,250],"anchor_positions_sha256":"ee1858c2e927a82713536b65cb17e66e4c3e2dba47ef6753229f8cbe7def8d65","canonical_id":"920","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[14,30,45,61,76,92,107,123,139,154,170,185,201,216,232,248],"fisher_boundary_sha256":"0d18b72689a6cc158337087ae0bf96353aba3ba691e2c3ad3383f821d707d7a0","horizon":1,"input_positions":[15,31,46,62,77,93,108,124,140,155,171,186,202,217,233,249],"input_token_ids_sha256":"af45fdc8e583f1a48a51520447e6ae51e48ff97bf7106924892afcd5b445f380","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[16,32,47,63,78,94,109,125,141,156,172,187,203,218,234,250],"target_token_ids_sha256":"9fb5b7afdc77178b1ad18dc9c00bf26c57c1c7f7283fef7c2915df9d37953a04"},"formatted_content_sha256":"ca28cff2764b8d05657413d50f09faa64113177e180abfde5843c1dd2e08ff63","generator_receipt_sha256":null,"identity_record_sha256":"af618c65e0f65410c7799e66f057af21d4ef73c2941c226d581f4f95c21ec761","prompt_token_ids_sha256":"5b15d5261297dc4ac22a17ffede7ef05db15e3f11f0c33e02fb3a3b0e0716a02","ruler_category":null,"seed":null,"selection_rank":26,"selection_sha256":"131778dc6e39fd27c980789805b135c6f9d71c73da87e2cf5d8f360898a2dfd2","sequence_length":251,"sequence_token_ids_sha256":"c20ed19761b4f9b066a1329b36e80f89c62a064c75eaf0dc86e28e961252022c","source_content_sha256":"a36a2af1cadc864f5b43b1a1232f93e477ab83df4d2faa13fb766e215ba061ce","target_token_ids_sha256":"3f83098fcd8caa7e8285f24d6317b53da289468363aa33dd7c3ad415fa2f3ee7","token_span":{"cache_exposed_start":251,"cache_exposed_stop":251,"prefill_start":0,"prefill_stop":215,"scored_start":215,"scored_stop":251},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1df227dd0db9ee9876c5933450c5e9d7dad0c0b569bfe3c3c9999cb603466a66","anchor_positions":[12,25,38,51,64,77,90,104,117,130,143,156,169,182,195,209],"anchor_positions_sha256":"33a74989d1780ad7a110110bf9c662c1607a2b0974227d3d4808405cc10473b3","canonical_id":"771","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,207],"fisher_boundary_sha256":"ca40d8ca4200a96f42fdd96325a52572dfd4c03985d05bc5122da59ebb5411f5","horizon":1,"input_positions":[13,26,39,52,65,78,91,104,117,130,143,156,169,182,195,208],"input_token_ids_sha256":"4a16859fc5dbcd936377e8ff10cf7ee24881b00e341b233f5d9eb6bcea0473b2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[14,27,40,53,66,79,92,105,118,131,144,157,170,183,196,209],"target_token_ids_sha256":"5ca4ff0a723a1b4d28096b092591f8877fbcca9586d49c5ca5cec329031ee210"},"formatted_content_sha256":"12a4cc60de825c6e0b4030d980e6efa58ef1883c0dc4e19c9247b74a5cf96589","generator_receipt_sha256":null,"identity_record_sha256":"b0f0620e81f02da6395b8984ee9a7ea9030dc636bfe539b7bdc6cada58f330c8","prompt_token_ids_sha256":"f7dad551ce5d1c884e8df0a56f12ba6a9086a14bf88ab24c0585392f67253369","ruler_category":null,"seed":null,"selection_rank":27,"selection_sha256":"13342bc1fc998ac6effb2746ad6dd189bd6e08647608c26dfca8ba6e4aa9bc7d","sequence_length":210,"sequence_token_ids_sha256":"880adca8dd7b7e2a5ad8761db452ab4f71a3a2ddb89a2fa3d3f5a58203aefb0b","source_content_sha256":"f9bc388a30026b9c50d24c57c12c983ad57c610e3b1805c0f31da6113e7f0655","target_token_ids_sha256":"946d1f65de1ab807e8d9cdfe1ab226186db882b34df45f0fb02c5dc1c917ec65","token_span":{"cache_exposed_start":210,"cache_exposed_stop":210,"prefill_start":0,"prefill_stop":75,"scored_start":75,"scored_stop":210},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"ba5d2d0908e27069abed969317f8767e700f0653a11ffa619097bf013257832e","anchor_positions":[21,43,65,87,109,131,153,175,197,219,241,263,285,307,329,351],"anchor_positions_sha256":"cac63358bd3a7b31df1079c37c0d5ebcf2a9d198f8fbdbfabcbd20682cead52a","canonical_id":"869","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[20,42,64,86,108,130,152,174,195,217,239,261,283,305,327,349],"fisher_boundary_sha256":"2f61fd72d21168bcf628f22bfa5c23cc9ae7adad9c324ddc9436931a5fc1078a","horizon":1,"input_positions":[21,43,65,87,109,131,153,175,196,218,240,262,284,306,328,350],"input_token_ids_sha256":"4681d4fb7732b42586111bbc62eecd3d429bea617e31837702eba8e398a2cdec","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[22,44,66,88,110,132,154,176,197,219,241,263,285,307,329,351],"target_token_ids_sha256":"4703a2598c12a6054f3c45ce108e141b4a06b5c95d21417d8999f4cbef5fb5b9"},"formatted_content_sha256":"a0eb7c677531c1708849fe663f5664ca25a2bb7202ed10517fbd94dbde91daa8","generator_receipt_sha256":null,"identity_record_sha256":"f65800d83e860fc9520ba9f8b4f750dbd13615d735ec53de0c7592e72d90220a","prompt_token_ids_sha256":"b39968cf98321965c22a9c256c9561e9cc6e27d9a79af006cc89227ec7120efd","ruler_category":null,"seed":null,"selection_rank":28,"selection_sha256":"146f84886fd8ea499b01ff6f18e9470af408d245c5e61abbaa90c362cb26c801","sequence_length":352,"sequence_token_ids_sha256":"1b54a34c4d7ffc13d96aefb4e29514f90f431dde94601897a2dbb235edc52720","source_content_sha256":"aaffc31c45370a56d2068d7903cea3700521215fda95a64f3df629df4b8eeb38","target_token_ids_sha256":"604ca3456eb59e105b48ea010ab6e7f9f2dcfb395b4632901098473fb57640a5","token_span":{"cache_exposed_start":352,"cache_exposed_stop":352,"prefill_start":0,"prefill_stop":305,"scored_start":305,"scored_stop":352},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2bb8e9815739f4c21c430d24915060fe6bd9dba2fc037408c960a316957378f8","anchor_positions":[7,15,23,31,39,48,56,64,72,80,89,97,105,113,121,130],"anchor_positions_sha256":"4b59ffbfdc6171d05db7e4ce13fd81c13fb1c2704ef54cbfaede26eafe2b1382","canonical_id":"851","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"fisher_boundary_sha256":"0b7b6d1d65846d9e17cf0ca2a441e8df727713097fccdd92a3cd8bdb20be9263","horizon":1,"input_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,129],"input_token_ids_sha256":"08877abaf9a7a17c7f7f67ebc2737c37d6453391c6d7b8ec1fbcd14c32c4e1fa","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,33,41,49,57,65,73,81,89,97,105,113,121,130],"target_token_ids_sha256":"f8b2c9f022f43056408bb17bc5c2c95096dde7d6dfeeade0d28c5bf51c33888e"},"formatted_content_sha256":"571149c507816476043bbda2c1fdd724f3ecb9950bd16943ff2ad7eb4ea3f436","generator_receipt_sha256":null,"identity_record_sha256":"7f27dea0fefa17b65ffacb46cfdb9b6d59002d7ca0b9361a0895371d7110d026","prompt_token_ids_sha256":"286d768c4aa541e076ba087112772071c0ed8c5cf796ee399892f394bcbf99e9","ruler_category":null,"seed":null,"selection_rank":29,"selection_sha256":"148757e7e40834cbdd797fc8415b8ff9af1ad4582e23345b11d6f4df20524e54","sequence_length":131,"sequence_token_ids_sha256":"8d6714078aaa6f0fbd4d5dffc75dbd163c2b9867713c0dffd9d5fd8f7017a68b","source_content_sha256":"a2daebc00878723580b55d11ecc7c733a3b30dc6ac7d2162e0a5ddb14516f888","target_token_ids_sha256":"2007499fc1bb16a5e15e13b3ebe6e9c56b039a7ed23a50d2c653934a2788085c","token_span":{"cache_exposed_start":131,"cache_exposed_stop":131,"prefill_start":0,"prefill_stop":95,"scored_start":95,"scored_stop":131},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"348a971192939b69d748687327be2a9dfe6f51af36995ecaa6f6c112cad31b71","anchor_positions":[9,19,29,40,50,60,71,81,91,102,112,122,133,143,153,164],"anchor_positions_sha256":"e70afdac88bd21ff5ffd057462f0868a1aaa408a52296b58ecd4900f7f13759c","canonical_id":"728","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,39,49,60,70,80,90,100,111,121,131,141,151,162],"fisher_boundary_sha256":"de5b5407dcf2dbe46dcd273dbcb9112fd1e97d1b3786ce35fa8124c57a372da9","horizon":1,"input_positions":[10,20,30,40,50,61,71,81,91,101,112,122,132,142,152,163],"input_token_ids_sha256":"164cace43db8303a6bad35155786734ad14df0e29d82329399b4eb77c2ab2a51","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,41,51,62,72,82,92,102,113,123,133,143,153,164],"target_token_ids_sha256":"cf736f31488418c09e969cb79604c6e24ee27824d7b99f2cf3615788cd99df11"},"formatted_content_sha256":"876263584044b675e092dfa08f39f07f3de5267b9c2cc8ebc0f0dcd3f7e69bed","generator_receipt_sha256":null,"identity_record_sha256":"07b155ad0595cc23e7b6f3ba638e79b47f8790d682a99a574040b9b316eb1770","prompt_token_ids_sha256":"7673bf381655d73528d4ffb834dd0461fec79f874ac5706819d772a4fd92f274","ruler_category":null,"seed":null,"selection_rank":30,"selection_sha256":"1488999c83a1784fe27b3d4c152a98b782b1ce9c84a2375a4dba8bc8a3845802","sequence_length":165,"sequence_token_ids_sha256":"cbf68229cee850133e8ccd5fc251f1c822ecfb6aa1c9466194360609b2644aa8","source_content_sha256":"1e63d65d4f23efcd1ebde304ca66d531e9fdfa44f73cf04c72637826af3e7c94","target_token_ids_sha256":"519538cb0a590f8242c78a833368585a5650cdbbaffd4159acb21311b25be8bb","token_span":{"cache_exposed_start":165,"cache_exposed_stop":165,"prefill_start":0,"prefill_stop":128,"scored_start":128,"scored_stop":165},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"fd0a190e30ef1d0dac28f65cd40bb5a6bcf3b1813b383cf8c3595fc399c3cf68","anchor_positions":[8,18,27,37,46,56,65,75,85,94,104,113,123,132,142,152],"anchor_positions_sha256":"75c1fd0cbacbbfa75bfedc1647ca1a15436ad446c2021b146a8d7af4567321f4","canonical_id":"704","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,27,36,46,55,65,74,83,93,102,112,121,131,140,150],"fisher_boundary_sha256":"3f642dc0ec35708247c2582e0525688edfba25c8e20f052b50fddeb5a552f23a","horizon":1,"input_positions":[9,18,28,37,47,56,66,75,84,94,103,113,122,132,141,151],"input_token_ids_sha256":"c7f609f9659eee383626ea3792b82efa4d88eed93f90d975eb30c098b4b461a1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,29,38,48,57,67,76,85,95,104,114,123,133,142,152],"target_token_ids_sha256":"04c201a63e6b0f0929e297c5714da1234f09af3ca5cb45bc1b660e160389b369"},"formatted_content_sha256":"cae02a47f70b35282c8bb71d3b57044d50c57479d6d860ab99959bd7d6a92fc2","generator_receipt_sha256":null,"identity_record_sha256":"5ee66dd333ec6248d85a5f68ed91d44953b5193938257c58e4a4d12487288565","prompt_token_ids_sha256":"049fdd2c0e15e77a6301829c690e8a3c665ce6861ab91933d27aa964bb51de01","ruler_category":null,"seed":null,"selection_rank":31,"selection_sha256":"149396b0bf9de8c384e257ecabf9d186fe87150d089c3ff0d04e2adce108c249","sequence_length":153,"sequence_token_ids_sha256":"d06f1e44e281b3cfff0ab509061ec63103257ce65b9382636808622420a3847b","source_content_sha256":"9acfae6a9b6bf5138640d082e7cd35c1728c6aef929c1163bdc349fd4539d647","target_token_ids_sha256":"413994788ec19a2c46be1007e6767b52c27d228534ace24859c56b31248a539d","token_span":{"cache_exposed_start":153,"cache_exposed_stop":153,"prefill_start":0,"prefill_stop":114,"scored_start":114,"scored_stop":153},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"30b1518b021112b74a55061d787eb229a1e0d9413d6dacc4728a4dab232a67b7","anchor_positions":[9,19,29,40,50,60,70,81,91,101,111,122,132,142,152,163],"anchor_positions_sha256":"868d4ebf721b69ebc32ed80fb23287a93509d3cda6e8df7dfacdd2b56b0be622","canonical_id":"725","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,39,49,59,69,80,90,100,110,120,130,140,150,161],"fisher_boundary_sha256":"9143ea3e85262814307335f186e336430d15da7ab0b7b3d21349bf2995db95b8","horizon":1,"input_positions":[10,20,30,40,50,60,70,81,91,101,111,121,131,141,151,162],"input_token_ids_sha256":"596886915afefd5186de7498848bbc83ca47716f8e4d7009a44f03810ca08ca5","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,41,51,61,71,82,92,102,112,122,132,142,152,163],"target_token_ids_sha256":"3039436dc9e159ba1ad775ff006ed4075d724a7118b474bb9fb8a1f2e8adf4fa"},"formatted_content_sha256":"ba9dc9a284681815182d925a5298e2f7da2b4d824b73ef346adbda8f64016575","generator_receipt_sha256":null,"identity_record_sha256":"e596da18d1c37026ef17f3e29e6e8ff1d0a69c227e1cbbad9c536f121a0a3fa1","prompt_token_ids_sha256":"9d320289cf51463daa5b252b98ca89089d036d231b6c861dc2405bc54707a135","ruler_category":null,"seed":null,"selection_rank":32,"selection_sha256":"14ed1df91ab46cf4aa753625f81c4925778f6febc4e611062de8ad6126487c18","sequence_length":164,"sequence_token_ids_sha256":"b6e36e161450f26908c1d62afda5bc88c353a091bbe87f9de4dd698d1df5afe4","source_content_sha256":"d222c883a2e980af7460497236d6865d722ce738a2d6303dcb39cefe0ea11658","target_token_ids_sha256":"da7b35aeba5cd5e4fc9de97bd08c7c8e930056923fea9d072dfd25b57ef54703","token_span":{"cache_exposed_start":164,"cache_exposed_stop":164,"prefill_start":0,"prefill_stop":138,"scored_start":138,"scored_stop":164},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"33a12c7dff5a4e3f2b0e78cb96a2e15ffca6d15f728512f48ad3d2a521547859","anchor_positions":[12,26,40,54,68,81,95,109,123,137,150,164,178,192,206,220],"anchor_positions_sha256":"9cd7cad8fc6a3e6b59742658c86bba1f77992157cd1a23eb295add08927945d5","canonical_id":"616","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[12,26,40,53,67,81,94,108,122,135,149,163,176,190,204,218],"fisher_boundary_sha256":"7fb746d0e7bad7bf9d3e8daeb30cb4fb5e6d8058e6e2f60f208e45d9d3074ef8","horizon":1,"input_positions":[13,27,41,54,68,82,95,109,123,136,150,164,177,191,205,219],"input_token_ids_sha256":"2984e5c39ef7c7a9017c275ad13366ddae1e4a944113a0d7634a8743fe7d838e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[14,28,42,55,69,83,96,110,124,137,151,165,178,192,206,220],"target_token_ids_sha256":"f8ac64591035ce55623fdb95289f20ce8ce2512016905cd8c55cdbd1eaf0892e"},"formatted_content_sha256":"4c31755567c8f3a5c363166b6b45afe0e9756596cab355453c6d1bede973f1cb","generator_receipt_sha256":null,"identity_record_sha256":"2d8025f16c114b643173084fd6a55004c4cc8d89c19ba719eaf77c614f3de5bb","prompt_token_ids_sha256":"e3e1a17d69f2fe4102ab51a9bb887707430d5bd517f752ff79cb5ba3bc19372e","ruler_category":null,"seed":null,"selection_rank":33,"selection_sha256":"15619d70a9d1e84b42b7fe6f67fb25251ed4e033a22a1f6b44afdb5190e0d9e4","sequence_length":221,"sequence_token_ids_sha256":"c14c8b8afb98ee4e0db1a7d15b4a5ad8f4ceed0533d70f5c7f3e3069aa019030","source_content_sha256":"1754fb38b7c558a69da8e9273fbe27d53c7428292d713cdcc926533238ea0dd2","target_token_ids_sha256":"67f9de9099caf2a8542295769dccf90b9868362889179ae8905fbdfffbfa259d","token_span":{"cache_exposed_start":221,"cache_exposed_stop":221,"prefill_start":0,"prefill_stop":172,"scored_start":172,"scored_stop":221},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"df20597eac46fe01ffe3b097acab98b2815becb962637bf63848ef96325ccb51","anchor_positions":[16,34,52,70,88,105,123,141,159,177,194,212,230,248,266,284],"anchor_positions_sha256":"875344e57ef3cc2aeb5a93339807a07f348a76ff8e531ac8b017af57bdfad6d8","canonical_id":"686","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[16,34,52,69,87,105,122,140,158,175,193,211,228,246,264,282],"fisher_boundary_sha256":"a3d4c1cea37694ff9e48138187338acaf151bbb01faf6beacef99dbb88b7a3f4","horizon":1,"input_positions":[17,35,53,70,88,106,123,141,159,176,194,212,229,247,265,283],"input_token_ids_sha256":"e11b1d7a44256148ceeca485fc09b3a0bbcb82bb1fc8f781be0549ae79fe1f2a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[18,36,54,71,89,107,124,142,160,177,195,213,230,248,266,284],"target_token_ids_sha256":"89eebf9531df2b055714e83202f980d1b550ad965a5daf4c7bdb06671f9842cf"},"formatted_content_sha256":"7351330811ac6c42a440a1ec8bc6d19bdf453494f7fd6204e144c59693722149","generator_receipt_sha256":null,"identity_record_sha256":"66b94f7added65074e9aa08de692376e4e61c77477886d2f1f02bd7a5f752106","prompt_token_ids_sha256":"c9ee635d4f345c508119836f02286e8a883aaafc0a5504f5149f7ce65b3e8a58","ruler_category":null,"seed":null,"selection_rank":34,"selection_sha256":"15a952922cfb59e1c69a11112c4ca386841fc6ef24bc7d4b5506686777d97a8b","sequence_length":285,"sequence_token_ids_sha256":"fcc6b59259d07edcb03ac21e620699ab241c7ce75c923d03ee63d3146e5b591b","source_content_sha256":"84f9793e00ac40962b5d2488ed7ea603278d041e9ce4d1fdb4c80a3cb038287b","target_token_ids_sha256":"f0132a38a41b59f822f698b45717035a5147aeda26ea2ca039b6d45f96067ef7","token_span":{"cache_exposed_start":285,"cache_exposed_stop":285,"prefill_start":0,"prefill_stop":239,"scored_start":239,"scored_stop":285},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"969aa08fc8cf0fcef08b212a868ffd0ef6cba7360b8cdfd4c7e057bdeb42b613","anchor_positions":[23,47,71,96,120,144,168,193,217,241,265,290,314,338,362,387],"anchor_positions_sha256":"5de510290d52f91a9e5aaaf002df55793d821f7c4325536087b3fe975a00c5db","canonical_id":"722","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[23,47,71,95,119,143,167,192,216,240,264,288,312,336,360,385],"fisher_boundary_sha256":"5bcdece273089c73eeb060133d26006686f4ba55e2f679475c817201f0f06663","horizon":1,"input_positions":[24,48,72,96,120,144,168,193,217,241,265,289,313,337,361,386],"input_token_ids_sha256":"387e8436656479c468480e0d7ab6fb8a04966a25430d2cf7ba6b946e933a5988","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[25,49,73,97,121,145,169,194,218,242,266,290,314,338,362,387],"target_token_ids_sha256":"d4c1910bee8959bc2af95badfdaa811dfbfff9ce3708801ed5e9602140453d3f"},"formatted_content_sha256":"66886a05110a3b2882fef0d7f3d69fe9e4813e5a230a25f32ee61406ab9d5da2","generator_receipt_sha256":null,"identity_record_sha256":"dae3872dc358589c8a222ed2b8f364150c472c88b23243e89cbb751bec3bf394","prompt_token_ids_sha256":"e0dd20f7abb2fd8622ac76878d7d8dba366a9267d17eb23cde37996d51815bad","ruler_category":null,"seed":null,"selection_rank":35,"selection_sha256":"165983c8c5992c88338b5a83c3e955a3b379e2938838aa804f752b6af24364d5","sequence_length":388,"sequence_token_ids_sha256":"be18de19017b46d5b7be59183a8ecda4c60a214ae50f068dd8ce1b4ea6848323","source_content_sha256":"20253504a8b3a9c5fa26e7d29c98f783efb48d1960c35ff104acfdc24f41bd57","target_token_ids_sha256":"5e021e0c05f5c5bc55c907080f16d71f9b17ea980d1c4a1c35db2a4193541b91","token_span":{"cache_exposed_start":388,"cache_exposed_stop":388,"prefill_start":0,"prefill_stop":344,"scored_start":344,"scored_stop":388},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"da5073651bb69cf285a41628d42d9f0b3d2b37c1bf90aa7aab1b73c4d9096d21","anchor_positions":[6,13,20,27,34,41,48,55,62,69,76,83,90,97,104,112],"anchor_positions_sha256":"311e7702507c904e70997e0b842193b8c519ddd45b6861c3e52fdd12fcbe662f","canonical_id":"919","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,19,26,33,40,47,54,61,68,75,82,89,96,103,110],"fisher_boundary_sha256":"46adb9e95c41a48fc3a2299f313d710d273451aec915fcc44050066d7e62fbe7","horizon":1,"input_positions":[6,13,20,27,34,41,48,55,62,69,76,83,90,97,104,111],"input_token_ids_sha256":"27444c193cd674bda4df496bd61a9e9daaf7ab3670462bf76e183b0cb9f16771","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112],"target_token_ids_sha256":"0c19883626f27df1a14cd3ef14afef3bc348c0cd1bda790d1e08185bca73a595"},"formatted_content_sha256":"032c4c4ef8d61477f89f627c7da791c2a394cc3d48b63ae5e73ee2fc12217e59","generator_receipt_sha256":null,"identity_record_sha256":"3ff17e45b728b8142b6678e3a33eac324c6262961cd2c9f7c19d1310c440aeb7","prompt_token_ids_sha256":"23b1b41e6716b12d13650e7aa105d3a76602e537cb02e4accc5f592ffd9c138a","ruler_category":null,"seed":null,"selection_rank":36,"selection_sha256":"165a7fd7caa5808dac419a6b8728f561f43cf78710437e2638e5c6a82bb7617e","sequence_length":113,"sequence_token_ids_sha256":"4582364354dd4ea26e506fc7c1845a9a6a878d03218ac93a0725170aef66c313","source_content_sha256":"30238651c7d4eb4dd16bbcf4c54ef3c91884f1d06eeb02ff38b97cb5e03a6225","target_token_ids_sha256":"9f6a8727f6485bb7bce051a35976a9ccfdc83c3af9309cb39f3a93789a51f928","token_span":{"cache_exposed_start":113,"cache_exposed_stop":113,"prefill_start":0,"prefill_stop":86,"scored_start":86,"scored_stop":113},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0a798f70d770e513f2e911c70a33835f666289b6e23ff31551a773ef75b38aec","anchor_positions":[6,14,22,30,38,46,54,62,70,78,86,94,102,110,118,126],"anchor_positions_sha256":"5dd8de14849d09f9037c328d85f9f1208f24e656ecfb3354483ee5aa62c57bab","canonical_id":"636","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,30,38,45,53,61,69,77,84,92,100,108,116,124],"fisher_boundary_sha256":"5f2e24eb47dfbebce18f2c2f64fd45917172c9eab2bd9cec54bb4fd111eb2142","horizon":1,"input_positions":[7,15,23,31,39,46,54,62,70,78,85,93,101,109,117,125],"input_token_ids_sha256":"4dc0a187eaaa7554b02100bc49019061f57a4f625ac418d8d6b7708a4c85f659","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,32,40,47,55,63,71,79,86,94,102,110,118,126],"target_token_ids_sha256":"3904fbb873aaac279d16ea071f1eecd284ddfaf982e8b4b118e527a7c5cd1663"},"formatted_content_sha256":"38120716e8579745533e38a1b2e5c3ab60964ce9826f4692be5d38a23625c7d5","generator_receipt_sha256":null,"identity_record_sha256":"6b00dd2ed569a9e5852db2dae0aa92821b9dbc00778205f14f3b67a886091087","prompt_token_ids_sha256":"c53edd29ef5c93c57c169235bf2b32cb118e20df561bc94d9b39faebc2a587c0","ruler_category":null,"seed":null,"selection_rank":37,"selection_sha256":"198ca758f04e266276eba3d3b469b52d783bc89308214f93d72af2c6d4e11dc0","sequence_length":127,"sequence_token_ids_sha256":"60e0f64c1dcb89512f1401f29f0d940ea152343ebc4764e52809ea4c5588f847","source_content_sha256":"054f097d046f21661577c0ed913168b649add5d35032de9efef4938f8a1a24ee","target_token_ids_sha256":"d1bf3a7c3a1a467f3f8e1eac6c4887bdfbdc387af6b7c29fcb19d45df9277f3b","token_span":{"cache_exposed_start":127,"cache_exposed_stop":127,"prefill_start":0,"prefill_stop":94,"scored_start":94,"scored_stop":127},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"d11ddad979bbc50d6db14cf77f968c46cb786a0ce2dc77ba5f5b28d500fc3596","anchor_positions":[11,24,37,50,63,76,89,102,115,128,141,154,167,180,193,206],"anchor_positions_sha256":"c25b8f380f538bd903f3e5b66a99e3d50700aeecbba5fa891f2350659ebff705","canonical_id":"607","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,50,63,75,88,101,114,127,139,152,165,178,191,204],"fisher_boundary_sha256":"a15bb1e5b2d2a3301ccdb2e0cb8998f04674305617ecc9881b7d20c208dd21c1","horizon":1,"input_positions":[12,25,38,51,64,76,89,102,115,128,140,153,166,179,192,205],"input_token_ids_sha256":"7ffaff1442181b24e662e144cb14110cb590e37477796426bec7dca4fc4d39ae","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,52,65,77,90,103,116,129,141,154,167,180,193,206],"target_token_ids_sha256":"05b028901595cb91b630adba80dd4555b614c059c122db592c5aa980143feafb"},"formatted_content_sha256":"ef35e3704df515714d76b547483c483743548e7cc069f31e95b66dec72a510e5","generator_receipt_sha256":null,"identity_record_sha256":"80bb8f3d83b1925af377e081ac063f63989368883ef4e04dc49b8fd8f00c500b","prompt_token_ids_sha256":"5bd1a90f46714aa81aed8226efa90f47442e2892377804149115218b9fd3b0c7","ruler_category":null,"seed":null,"selection_rank":38,"selection_sha256":"19faecf7ffafd480a6fa766dc937ba0709e42a5d5263c93130eb1037763f8871","sequence_length":207,"sequence_token_ids_sha256":"020df0554b8bcb9a4b7f491a4513578dc9b60d47ffc0d439886b3997c36eb986","source_content_sha256":"c541b88b979a7ec60640a49c7a9db7db502b97f5fed4d1970da91470204697c0","target_token_ids_sha256":"28dd01dbcdd5f0d19a9bef9f9fbc6491786783fdb97ad1fb07f945a93ffc7cc9","token_span":{"cache_exposed_start":207,"cache_exposed_stop":207,"prefill_start":0,"prefill_stop":141,"scored_start":141,"scored_stop":207},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"46371c5d2e1a407eb39f03172e7b9a21c7b932f5d655758a3c0ce64ba8c669a2","anchor_positions":[23,48,73,98,123,148,173,198,223,248,273,298,323,348,373,398],"anchor_positions_sha256":"8d9eba8a281ab33b27dd14ed9d345bfd773fffcabf08f24d986bb5ecd2f5015a","canonical_id":"950","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[23,48,73,98,123,147,172,197,222,247,271,296,321,346,371,396],"fisher_boundary_sha256":"843c418f1af24b45838fcae84492105e4320ec3725733998cacab7b6853efde8","horizon":1,"input_positions":[24,49,74,99,124,148,173,198,223,248,272,297,322,347,372,397],"input_token_ids_sha256":"d9a45e7a4afdd75464b73c4c6185b6888c636708adbfe2ece93d9586f2e48950","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[25,50,75,100,125,149,174,199,224,249,273,298,323,348,373,398],"target_token_ids_sha256":"b25ab7ecce47ea372a3f59f87969ac12e02c7358ce537fa47921278d5bbace65"},"formatted_content_sha256":"99720600b7a4e17df2bcc2762c83cdf70cc53626e5ec631c9bb32d7398d93e45","generator_receipt_sha256":null,"identity_record_sha256":"6d22eef549de3a81fc0a232affc18a615e9f0fce2de79d390b05e11ebdbf763d","prompt_token_ids_sha256":"8801aebf9d0be65bf1d4cac5f7ba35ceab845d7b0e20d5f6bfe717cf70fcac04","ruler_category":null,"seed":null,"selection_rank":39,"selection_sha256":"1b30f64a04ff98347d4a23bd49f8964fc6d3976e5d73b89babb36d891b544775","sequence_length":399,"sequence_token_ids_sha256":"0c20245c48d7a840676ca88457d31004701b407e4992f56ed7e4aca800bd3cee","source_content_sha256":"67c27b9d1db42865751e90ac9eb8dce913fa954d522bc50f1a124ed869dcc3b6","target_token_ids_sha256":"80ad07987d1074046d11f85abd02309f81753af95f7b2510f15b49ce074774dc","token_span":{"cache_exposed_start":399,"cache_exposed_stop":399,"prefill_start":0,"prefill_stop":85,"scored_start":85,"scored_stop":399},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"88045b346b8da1ff4c8b2efa7461ce5138643eb8b76bb78bed0cb8b0de1efbf4","anchor_positions":[11,23,35,47,59,71,83,95,107,119,131,143,155,167,179,192],"anchor_positions_sha256":"97c036645b9b6a78e53e848879a7e3ee08a70ff938ed6e98aa7098b21ab4319c","canonical_id":"885","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[10,22,34,46,58,70,82,94,106,118,130,142,154,166,178,190],"fisher_boundary_sha256":"810182ed20f2eb26e385debb7eceaceabe95cdd144936b72eb2f3945a5610256","horizon":1,"input_positions":[11,23,35,47,59,71,83,95,107,119,131,143,155,167,179,191],"input_token_ids_sha256":"76887b11d917341d902fd087a797d1b8148960fe5a5c3bd03223991175a7d00d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[12,24,36,48,60,72,84,96,108,120,132,144,156,168,180,192],"target_token_ids_sha256":"6d849b678d0b30547bb1a54e0ea0ac4ce468ce61603c7c10ffdab4c4e48459ab"},"formatted_content_sha256":"a16af1d88bf778a540f042c4c4f117a8a55e110b844fac976861bf833b58928c","generator_receipt_sha256":null,"identity_record_sha256":"210e74e96d878fcdcc906fc9925e4f204d9363fe486a768730fb41f707b3047a","prompt_token_ids_sha256":"731bddd1572f8ac6e10c56476d8d41ce0f5b35ad3da79217ad289e53697fc68f","ruler_category":null,"seed":null,"selection_rank":40,"selection_sha256":"1bf4005ab722efdcf542cb1b134158d2a2ec80c4bb99a978bf37320204ea651d","sequence_length":193,"sequence_token_ids_sha256":"221d7396fd6ba0730b59b2d163b37447727eeada7ea48a5bc944748d4c03df26","source_content_sha256":"c09a41add9e81f949a02fef6dde8a12287741692b045d26b3459d9cbf51a329b","target_token_ids_sha256":"a28fddf10e4bd937c83e1787639bbae521e60b6a3d5d421086e7e8d04ced9afc","token_span":{"cache_exposed_start":193,"cache_exposed_stop":193,"prefill_start":0,"prefill_stop":82,"scored_start":82,"scored_stop":193},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"5631703787387bea51be9c3d8508c71583e012079113a9f10bbec4245f396cf1","anchor_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"anchor_positions_sha256":"debc2f860d51bf30c9ccfe7850d3428800e80a005f976ab2943a8642c33fb808","canonical_id":"646","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,30,38,46,54,62,70,78,86,94,102,110,118,126],"fisher_boundary_sha256":"ac073c216400b3bc77300dbc3f3bd0735b111cf717a637bf602d1117e5d70dd5","horizon":1,"input_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,127],"input_token_ids_sha256":"06dec0ee7d5561ebf4a6924837605f275d606eb9ead89de4bbe6b7ec86a0c69d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128],"target_token_ids_sha256":"819be8fbf775e438d651a64a9a89d31a86d639d85622f2e6a979e72bf5469879"},"formatted_content_sha256":"42d2696d713ac8654462eed80e51a0f4a279ae3bd0b2fac3a803e706e31df249","generator_receipt_sha256":null,"identity_record_sha256":"a62e891eaa4a9d3d4ea47d185ffa6c03a9b1d28dcc7dec3429aece7413ef0936","prompt_token_ids_sha256":"e5220dd3a6c5ee6f26bffcedb6d05bd93a890b9b7d74832a9669975115e37150","ruler_category":null,"seed":null,"selection_rank":41,"selection_sha256":"1de0cdbbd1b50e7684dc3deead2b93036bacf525241fea00a2c68aeb5d1e9cf0","sequence_length":129,"sequence_token_ids_sha256":"eba6b177f82a24193e70a824138922ac8eee52933ebb80cb81a777d6e0f85943","source_content_sha256":"14c86e5b0221ac25a5d693c8996c478c0b60503bd396981a5ba2629d0c6b6f99","target_token_ids_sha256":"6c75f6536a4d7c5a2942fd540f9268e2e1961c4c3dd4523408da058253f543b3","token_span":{"cache_exposed_start":129,"cache_exposed_stop":129,"prefill_start":0,"prefill_stop":88,"scored_start":88,"scored_stop":129},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7105aa2d8387cb83f89b844f1f0a3176a54c1be40233dec8811a37cb7bee09d4","anchor_positions":[5,11,17,23,29,35,41,47,53,59,65,71,77,83,89,96],"anchor_positions_sha256":"bbeb8538883fad44ee21dd0032311ae447587513a5cfd0b0e896721a67ec397d","canonical_id":"708","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[4,10,16,22,28,34,40,46,52,58,64,70,76,82,88,94],"fisher_boundary_sha256":"977350ea6b433e2f68043b28b121e4816ce8fb2f7358ecf47291b22c9adc1129","horizon":1,"input_positions":[5,11,17,23,29,35,41,47,53,59,65,71,77,83,89,95],"input_token_ids_sha256":"a2ea5635b427055cb1e2e99736b9dd1cada7580fac25e57e539b72697c2a8ea9","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96],"target_token_ids_sha256":"81ee8d37b3f38a8c027e847f395edfdc8428e67a0cfd5bb3a5d12bfdac304ab3"},"formatted_content_sha256":"c810e44140744e432b3b9ff12b48167b78e7a00f4f078b3d9b5a3f539a0438dd","generator_receipt_sha256":null,"identity_record_sha256":"ab8e48a41997cface254c352c24a699bef6c43ae1858c93baddd4ba95bbef8e1","prompt_token_ids_sha256":"5a9bc94048f86f543dd1e7c3e8040a400a75b1f99b6ef637101d4590969f22bd","ruler_category":null,"seed":null,"selection_rank":42,"selection_sha256":"1e4c5d9ae6b7070238249263248d1c8593c90a70aeeb70af89325a5b3be72629","sequence_length":97,"sequence_token_ids_sha256":"23e232e07e81e613d2b01d8540dfd4e20039938471bf3dbebd3ba8c57449e8b0","source_content_sha256":"a670341f8d1f903c87ab584beb9dd6355bf23c707ac087606d85d5e1094169ba","target_token_ids_sha256":"bc70ef552d90fd9d0e6a01a2dad09bad247647bd98b2af5d9afba83f63d97b8a","token_span":{"cache_exposed_start":97,"cache_exposed_stop":97,"prefill_start":0,"prefill_stop":79,"scored_start":79,"scored_stop":97},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"21fdfa6c62576c444b2ede715e8c0f394692bdae740be5657a09fb9ac5465380","anchor_positions":[7,15,23,31,39,48,56,64,72,80,89,97,105,113,121,130],"anchor_positions_sha256":"4b59ffbfdc6171d05db7e4ce13fd81c13fb1c2704ef54cbfaede26eafe2b1382","canonical_id":"840","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"fisher_boundary_sha256":"f1a95ea699018f29238a0e61c6067bc03c5622df1f9f38ee7310c244a4f3bc65","horizon":1,"input_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,129],"input_token_ids_sha256":"db9ca78375767d3dc1fc43cd6274a248a1200a4e579725f08ae34f2d1594030c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,33,41,49,57,65,73,81,89,97,105,113,121,130],"target_token_ids_sha256":"d4ee1e1651e25e76151be8dfefda3d59a9153f9a6418389ce485081fde471280"},"formatted_content_sha256":"9aced07cb336136d232b88c651523d949c3ed1b58608f3e91ab8eb1e0f92a7d3","generator_receipt_sha256":null,"identity_record_sha256":"baa3355a5d2917d2ff4b713073abc3c98d3168719b1d9a669dfc8d2745e62db8","prompt_token_ids_sha256":"9b97f8bc9144022005c6d732aa6daf7482cce1d0462c9f41e4e54f455d770a9b","ruler_category":null,"seed":null,"selection_rank":43,"selection_sha256":"1e5afc59d9964484ca6906bae912fa0a9c9871e246e9a5aa6f4e2d2cd74a61e4","sequence_length":131,"sequence_token_ids_sha256":"ac96beecd6d569ac72fd89a9d15d46adb2c7e2ff3ab53a38a47e2ebac4b5c19a","source_content_sha256":"144d4c36a2f15b6079c87286abc57c5c33d11947809eec3b55d2c56b0abdca35","target_token_ids_sha256":"60cc8ab098d798eaf3a7d1177d7c65092515a6756c2fb167019eb1588a7a5f57","token_span":{"cache_exposed_start":131,"cache_exposed_stop":131,"prefill_start":0,"prefill_stop":98,"scored_start":98,"scored_stop":131},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"92d974a85b650173fa2852040052d155e429f141353a391c32eb9c8b22bf9582","anchor_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,208],"anchor_positions_sha256":"e3c9aaa391385675ab3475417e405da92d531052d937e046ddc609185dc2c7d4","canonical_id":"793","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,50,63,76,89,102,115,128,141,154,167,180,193,206],"fisher_boundary_sha256":"bcee19793289e8c6540ea881a8f492bb16b99a21c7f08133aae2e4a62163e585","horizon":1,"input_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,207],"input_token_ids_sha256":"ea265790ccf2b7ad1bc5812d8c7ad1653088a1b938adf116bbe32e00c4b3161c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,52,65,78,91,104,117,130,143,156,169,182,195,208],"target_token_ids_sha256":"03166cef7fd0abd95fc728344274b68579c18cdff9751955cb55d37472750e8e"},"formatted_content_sha256":"5990b44a405865af219b0ebfe72ca5f4807c7fa9ea8d17a998d181a60e7e6e2a","generator_receipt_sha256":null,"identity_record_sha256":"7694eeb6f1cf845cfd47b3f8c037b4453af3e6f1af57cbc1b271567d78747feb","prompt_token_ids_sha256":"a215965eef2fc3b7d4240f33fb78e20e16353f44c552a86876020a731cfb23ca","ruler_category":null,"seed":null,"selection_rank":44,"selection_sha256":"1efdf31f8359ff37ddbd6972a5b1f70a89aad0090992f1439e95a68b17f5805e","sequence_length":209,"sequence_token_ids_sha256":"b0ada4ec9a33d0cbd5317f8965026c54cce944770d6db49dedf1045e3e120210","source_content_sha256":"f51880de3d4f3f8df181ab757d4737250300a2719eb925110ed1a7025baefea4","target_token_ids_sha256":"1bb894993d9298db11a3f618f8c67b31b484d0fa664e5d9a74950bae08de8292","token_span":{"cache_exposed_start":209,"cache_exposed_stop":209,"prefill_start":0,"prefill_stop":108,"scored_start":108,"scored_stop":209},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"467b274cf4fd1ebd3fc1f65e47b4ebcef7ec6daac053d9c806d526d02b1aa9d2","anchor_positions":[7,16,25,34,43,51,60,69,78,87,95,104,113,122,131,140],"anchor_positions_sha256":"adf29a4dd3126019eaa451ccdb6bbdf039b38d8dc8e8cbe224d799517db47f03","canonical_id":"860","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,25,33,42,51,59,68,77,85,94,103,111,120,129,138],"fisher_boundary_sha256":"4298d52c37b3378977ffe8fcdbbb8de8a2001e201f91fa007b5bc8caf344f153","horizon":1,"input_positions":[8,17,26,34,43,52,60,69,78,86,95,104,112,121,130,139],"input_token_ids_sha256":"ff88adf412ffcfaf2eb5e6577b0406f784344c338d07181aab928b4e45c4aedb","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,27,35,44,53,61,70,79,87,96,105,113,122,131,140],"target_token_ids_sha256":"cb42fbfadbf84d40f31a090f455333c32b8d82097fa30bbe81c411778c79d549"},"formatted_content_sha256":"9fa46af05168957f4bcef1fe5cc161d5fd1f8aec927a8f95b00c13ce334a2346","generator_receipt_sha256":null,"identity_record_sha256":"f03d6c0de51871dd6a4c550ca9bb84df67fdd8703da60597ef107f6b4b5998cc","prompt_token_ids_sha256":"a9df32740687672e9bcea5fc074876046b128fd5263c7a057d9f57e081343d25","ruler_category":null,"seed":null,"selection_rank":45,"selection_sha256":"1f2c607266f18f654c50ab67f529127bf81f03531b6d79c46ac97f5081f09a65","sequence_length":141,"sequence_token_ids_sha256":"13d911b52ef7a2c203854b8ab41f9083733e9bda1ca8ed67607907aacf4bb0f8","source_content_sha256":"6948e55f2c0c12a3d6c826e33977a3b358f503e8cfccd743ee937a4f53cbcd15","target_token_ids_sha256":"37c5ab0e675f04a104215395c9a2a8503e4871823234c8abbb133ac6babb0449","token_span":{"cache_exposed_start":141,"cache_exposed_stop":141,"prefill_start":0,"prefill_stop":95,"scored_start":95,"scored_stop":141},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"37d7574d8d2b723c82778a768a2a566b7f49e8ebb69db0fb5d1e6794a930a2a8","anchor_positions":[5,12,18,25,31,38,44,51,57,64,70,77,83,90,96,103],"anchor_positions_sha256":"307c6a1944f2d94c59465c6af96e251aaf0ea665f8f1a3e0e55e3eb0568d8551","canonical_id":"800","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,11,18,24,30,37,43,50,56,62,69,75,81,88,94,101],"fisher_boundary_sha256":"54b8850d26dd8a6a43bc711ad584ae0be7143cd933d1768dbf78ab9d2e4721bf","horizon":1,"input_positions":[6,12,19,25,31,38,44,51,57,63,70,76,82,89,95,102],"input_token_ids_sha256":"5182574bad58e65b680691603b61908c731b487efc2d522fac61ffb07e8601e2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,13,20,26,32,39,45,52,58,64,71,77,83,90,96,103],"target_token_ids_sha256":"8836df71d734324c70c31d4de1e0ff0cdde8edd0e57b420ca0ed65319cddd528"},"formatted_content_sha256":"b47d463d8300a4f8eb63bc1a7f94408c9b894a1853d6b4b63d05eed61f97e078","generator_receipt_sha256":null,"identity_record_sha256":"83d65e18df6abcd124b640d68f38f75296045dc30136a8095de98e9a56ff6059","prompt_token_ids_sha256":"f32d5f100044bbea939023be6610b696d665c2e49ac55c3b2478fd990e92c75d","ruler_category":null,"seed":null,"selection_rank":46,"selection_sha256":"1fda23ecb8320de65f93f8ffeca9bc66d1d6ba82fa80bd9e2d70cb080ea19be8","sequence_length":104,"sequence_token_ids_sha256":"2095ea60e58d7a748f80d99738c7213d620ebe1064bc23e828b8298509455223","source_content_sha256":"251053bc7b023ed45edfd2a618f866b0c06fd77cb43c5c6c4b6ad81c515873fc","target_token_ids_sha256":"8ddb65e178ceb2fa2d463b1981c1b2a0d308bd079411c8156ce1cba86f28173f","token_span":{"cache_exposed_start":104,"cache_exposed_stop":104,"prefill_start":0,"prefill_stop":83,"scored_start":83,"scored_stop":104},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e3228f24ef2326d02eac4d51adfc4dac5f98a046b3f1fa2c9d460bc590e7ad00","anchor_positions":[9,20,30,41,52,62,73,84,94,105,115,126,137,147,158,169],"anchor_positions_sha256":"1de8f9c2ad33c2ca63d619d4e737066294b86b501bd566dd6f84cd9fa731efd0","canonical_id":"867","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,30,41,51,62,72,83,93,104,114,125,135,146,156,167],"fisher_boundary_sha256":"6cdf085217e9a92a862d08a98b72af716319adcb02639b733b87d792f63b1261","horizon":1,"input_positions":[10,21,31,42,52,63,73,84,94,105,115,126,136,147,157,168],"input_token_ids_sha256":"b3f41b7d18076730b37f241371d12f38791b7734d302d899fcfb2ae98bfa7e20","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,32,43,53,64,74,85,95,106,116,127,137,148,158,169],"target_token_ids_sha256":"f68aa42c7401aa2aec59c231aba070689444ab604d38e31d8644873caf22a672"},"formatted_content_sha256":"c5d4a00fec65b64c0d75bc557b515c301fadade8597055f2ebb3d094051ad750","generator_receipt_sha256":null,"identity_record_sha256":"ac31836b26155996736a3cd0eebcf92e2d25df1fbba42f351c07926a34d557cc","prompt_token_ids_sha256":"3965eb8ca467ffc97d00622d372a5e88bf51d885d8f90daaad663ac6a3125362","ruler_category":null,"seed":null,"selection_rank":47,"selection_sha256":"207c6e9619ca7dfd647996b0ddc78892694e1df2e515a63deabfce4784b7300c","sequence_length":170,"sequence_token_ids_sha256":"a45517724ce7682620b190d1954ae73f1c66fd716f768ae8411b3ebd337b8523","source_content_sha256":"2da1fe1d0cc596300fd25d9a0f0e96da35c84b424c329057298f8e3540157f26","target_token_ids_sha256":"c7498f64ba251e766cd9dfe1464d55b2e2348ec90765cbe0ec26f908cbac2a9c","token_span":{"cache_exposed_start":170,"cache_exposed_stop":170,"prefill_start":0,"prefill_stop":114,"scored_start":114,"scored_stop":170},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2f3dfc43992ed341ed7978295644b024d0c191c9ec9c0ab7120ce608b8e69aa6","anchor_positions":[22,45,68,92,115,138,162,185,208,232,255,278,302,325,348,372],"anchor_positions_sha256":"3ce5659702a26cd1b59b1d16461526035d0580f1b9adb943021fa501179be80c","canonical_id":"622","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[22,45,68,91,114,138,161,184,207,230,254,277,300,323,346,370],"fisher_boundary_sha256":"3dd21721af5c559d0bf97f5a0d8335eac58669690242a9a65414cb07123ceab2","horizon":1,"input_positions":[23,46,69,92,115,139,162,185,208,231,255,278,301,324,347,371],"input_token_ids_sha256":"b7ce16f1b70f8a32ac421274ec67536496f84420741a5dccf20534b7d1c9f94d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[24,47,70,93,116,140,163,186,209,232,256,279,302,325,348,372],"target_token_ids_sha256":"d9f650368349431e7cbd102c1fa8783d9dc57c0d60cd0a5c4c784738af1f4c19"},"formatted_content_sha256":"216e5e12034d35e11932dcbdc1b82573653565c8944e50d962328af76cf74782","generator_receipt_sha256":null,"identity_record_sha256":"6ac868c67f1d02fb4ef0c5a4472eb0b28625461b9b81108f54d3aa3aed1b7d70","prompt_token_ids_sha256":"521456ee3e26ec55cb42c2589084173f76e5275561b51201d380d4eed502f265","ruler_category":null,"seed":null,"selection_rank":48,"selection_sha256":"210000c62df8127b95ae009f6518f2d6201b9506f6c8506835dca4eb8bbf293f","sequence_length":373,"sequence_token_ids_sha256":"1407f58699c66ad8f21f74fcf9606d047865d22dea2021a94e18700ba9dec3a8","source_content_sha256":"2000263b26b6240378d05d98725621f6c3e2bdd39265a3d356bf34ba4967d9ee","target_token_ids_sha256":"4676880dc99f1babf2689d76705c312671bc2ff81019432ec05b98c6156e2587","token_span":{"cache_exposed_start":373,"cache_exposed_stop":373,"prefill_start":0,"prefill_stop":189,"scored_start":189,"scored_stop":373},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"38589b6a6ebd3baba9bd23ed78bdceef6846e6529843ad4232525292bdec2cbe","anchor_positions":[6,14,21,29,36,44,51,59,66,74,81,89,96,104,111,119],"anchor_positions_sha256":"e83bd2e331c896fb6e9946121cbf13d77b4819161e7591880bc3055c1dbc069a","canonical_id":"677","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,13,21,28,35,43,50,58,65,72,80,87,94,102,109,117],"fisher_boundary_sha256":"683f76b65c4ebceff74d9ba1f3da44c4048119e0043947c4b3fe743eb29acf47","horizon":1,"input_positions":[7,14,22,29,36,44,51,59,66,73,81,88,95,103,110,118],"input_token_ids_sha256":"5605175529bcf8df342d6991176c6dcd594b6850a1f276f1e516f05daaf07e33","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,15,23,30,37,45,52,60,67,74,82,89,96,104,111,119],"target_token_ids_sha256":"25df50b9075a9d7a5a4d1e2bac00e89268616661a6d17f51d339ca2bb9a7ad99"},"formatted_content_sha256":"441853ff923cc424ab0aa7fec31c4aaa73bd11a24cff64d1daad0da0b63dda69","generator_receipt_sha256":null,"identity_record_sha256":"0d876619f10e2042398c541c7464cf9b347fb64e1ac8107841be3973e213cc36","prompt_token_ids_sha256":"0d2e23bc0f9b51a958475c21ec2c44735aa0f223221380fb0cbc53e877eea9e9","ruler_category":null,"seed":null,"selection_rank":49,"selection_sha256":"213beddb6f6d7179f5bc0d037d084c135da38be5e6cebc224e9ee64574990d1b","sequence_length":120,"sequence_token_ids_sha256":"ef85c6f458a4c28417a4ebf89c6986376b20129c419177fae9b89453d5ee6bad","source_content_sha256":"33110b4565d527f05bb143f7b956a5df85324d2da6b82595171ebb52b1c28695","target_token_ids_sha256":"cb45a27d57424658ef3029fa23e359436aec54c0636af1e913e0b7b0b1a63c83","token_span":{"cache_exposed_start":120,"cache_exposed_stop":120,"prefill_start":0,"prefill_stop":85,"scored_start":85,"scored_stop":120},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0af2d86cf1695b8e2f191abffee4365a9731009779912cf6c2993138f08a66fc","anchor_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,160],"anchor_positions_sha256":"cea919870992245499fb8d8a858d848e1e76f02a4e2230ada935442533c03198","canonical_id":"849","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,18,28,38,48,58,68,78,88,98,108,118,128,138,148,158],"fisher_boundary_sha256":"dacbce55717fb2ad9e2c1c565b4cdc586f652e71481489ba9f2fe0fbdb01d223","horizon":1,"input_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,159],"input_token_ids_sha256":"abbb2fdf32082b7edb9cffcb691959ba71fa426d6242d133388daf5b12207e54","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,160],"target_token_ids_sha256":"b6dd6b170c917647555dae8b2934c8757a2075a9fd62bdcc5c741d21a155b8c1"},"formatted_content_sha256":"b97e246c5fb6006e691f8dcc86a87679113f7a78d64aeac8c4bdcded6e3527a5","generator_receipt_sha256":null,"identity_record_sha256":"4e41b786817605fe72e6c9fdd1fee25d51c2700b517525bce5f2ed34d6095542","prompt_token_ids_sha256":"57f92b2bb27768c978d8e3c41a9efc9e0ebbc4d569c63d7d84dd70b17515d47c","ruler_category":null,"seed":null,"selection_rank":50,"selection_sha256":"21f0779cb4b05ae80d3df3e594cf70e52fde9b8adc7fd7457a46be4c7a8aaf45","sequence_length":161,"sequence_token_ids_sha256":"aec4c452b53bb10d8fe6bcd4d1f7770db728b6d1f1928e376b6757e469976806","source_content_sha256":"fedc2450a7af5f522a92d0e1052fa8033d9fb19a48f2d667a7f31093a26f405c","target_token_ids_sha256":"45ed26c261c710a2cc312afbf071387265818704baa2885344b040c5f1d5b70f","token_span":{"cache_exposed_start":161,"cache_exposed_stop":161,"prefill_start":0,"prefill_stop":75,"scored_start":75,"scored_stop":161},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"f8a20553c70e4c3efaf0458db6c7c1d06d0e2dfc9ab4f2783f010af84e82721d","anchor_positions":[7,16,25,33,42,51,59,68,77,85,94,103,111,120,129,138],"anchor_positions_sha256":"08b85b98703c7ea55928e16b78a87011e7254eb0d0516881c1cef449b5ad76ae","canonical_id":"756","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,24,33,41,50,58,67,76,84,93,101,110,118,127,136],"fisher_boundary_sha256":"859315451573e13275eaa90fef292f4abcb53e2d7979744ca1a559cd552531ab","horizon":1,"input_positions":[8,17,25,34,42,51,59,68,77,85,94,102,111,119,128,137],"input_token_ids_sha256":"c70dd0ee0947d887043e4ef090390d9b55dcb4940348ebb271a5299668f94c6a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,26,35,43,52,60,69,78,86,95,103,112,120,129,138],"target_token_ids_sha256":"844ca5f05e19572138e424833fe71cb51a25987fd79b470130b42fa975c1a39b"},"formatted_content_sha256":"79bb829fa41f1b56a259ec1d6f01ea3454814c377128ca4380d61741eac44e55","generator_receipt_sha256":null,"identity_record_sha256":"1eb2616b62b6f8e94a42917d3cba2e04e70983835585b47745bb90e8e2aba08c","prompt_token_ids_sha256":"f39e99e8861e0fba4e540dc710f8fcf78133eb7be50e166ae431c650dfe3c4c9","ruler_category":null,"seed":null,"selection_rank":51,"selection_sha256":"21f1cc6a98b0063838189643fcd01e4b5a84fbc7b4cc8effe18cc4d33580ac9c","sequence_length":139,"sequence_token_ids_sha256":"c8504b3fc7419f64b9e0f081c9f129180471f53a90800918c11449b486ba7987","source_content_sha256":"cb27d1dd0a93a0f5b25d8f2486a5017c8d78e16aaed3c07e71a04cc440d1aa5a","target_token_ids_sha256":"66b6c358b326ed60b26f60ece3aa3a72bae96813ca059d87e44cc7dfcdb72c3b","token_span":{"cache_exposed_start":139,"cache_exposed_stop":139,"prefill_start":0,"prefill_stop":91,"scored_start":91,"scored_stop":139},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e32814316554b0f79cc5669981964cddf032347b4da8365b25ddadb67a9a91c9","anchor_positions":[6,14,22,30,37,45,53,61,68,76,84,92,99,107,115,123],"anchor_positions_sha256":"4161f0e0da9e01e9fca23a34e70979683578e6b7ae825c0791088249cd49ed8e","canonical_id":"960","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,21,29,37,44,52,60,67,75,82,90,98,105,113,121],"fisher_boundary_sha256":"1e87180a8c7bf1f65c301b64b88ff3c34dd59beba4dc53810695b834c0c5e9de","horizon":1,"input_positions":[7,15,22,30,38,45,53,61,68,76,83,91,99,106,114,122],"input_token_ids_sha256":"3f2fcef30c7d427d14cbcac1db88f5e74dec170dbc068333bce564584133cb71","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,23,31,39,46,54,62,69,77,84,92,100,107,115,123],"target_token_ids_sha256":"cbc1bf9d8cb5df620060a192384df8638cb094155f8978e9bcebfa5f6b4e3276"},"formatted_content_sha256":"3e4da97d0bddb12409d97ddcd69beb695eb66abac525bc59990220b07aa13b42","generator_receipt_sha256":null,"identity_record_sha256":"f15b0404946c938be5d272c212d67ae55ec18c9f4abddd2bf888443f303e871c","prompt_token_ids_sha256":"d4bcbfe95a1ebf023872fe53289683ee41d253e100dab52d598c34e9239b284c","ruler_category":null,"seed":null,"selection_rank":52,"selection_sha256":"2296dea570e1139150ccd317f7f4064da5f91c662c46dc92bacd2cafbd8693bc","sequence_length":124,"sequence_token_ids_sha256":"9c259585007e9ef7b2dde2fe8a3ecce5850274464b4c662d95e215932a3c753e","source_content_sha256":"b9eab9a91cd92a96f9a16006fb04105309ec0c2a0fe6dcbb8b8d6fd5c5ad7e81","target_token_ids_sha256":"ff3e6cdf6ef65136f4d5fc6ee8370dd4b879f47033e6e5ea58283168cdb8eae9","token_span":{"cache_exposed_start":124,"cache_exposed_stop":124,"prefill_start":0,"prefill_stop":65,"scored_start":65,"scored_stop":124},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0dad65eaaf28f71b8151a7dd5bfd62b3f371591729ce9a325cd6fdb1ee5d99d2","anchor_positions":[6,14,22,30,37,45,53,61,68,76,84,92,99,107,115,123],"anchor_positions_sha256":"4161f0e0da9e01e9fca23a34e70979683578e6b7ae825c0791088249cd49ed8e","canonical_id":"741","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,21,29,37,44,52,60,67,75,82,90,98,105,113,121],"fisher_boundary_sha256":"b7544cc2c06d73eb766987d95229a15c4109c2c9f31ee7a8109e1411b1f01f8f","horizon":1,"input_positions":[7,15,22,30,38,45,53,61,68,76,83,91,99,106,114,122],"input_token_ids_sha256":"523dd1079553b7be63d71e5a3bbe2ebd41dd50b1105590b40352ba6d4ac94529","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,23,31,39,46,54,62,69,77,84,92,100,107,115,123],"target_token_ids_sha256":"2ee4f36ffbf325d3c0f22cbdeb5f755c29cd1766c4b85bc98367f2e4e3a615fd"},"formatted_content_sha256":"d9e493674ac0bc6520c04b25bf6f2a33ab4010024516b62c9111a1e85fd4b7ed","generator_receipt_sha256":null,"identity_record_sha256":"09fd441b743d4db344555f2991adfeebeff8cbe76ceaaaa25dd4bfc5c8a9c2cf","prompt_token_ids_sha256":"a05bc2e81b72ead55f4ab23081aa3740bd5e94b12c6910ff66b4d5cd8eadc104","ruler_category":null,"seed":null,"selection_rank":53,"selection_sha256":"23920e4407f636372457bfab6052016160b52662fc2489114a897651a77f02a1","sequence_length":124,"sequence_token_ids_sha256":"a1e0d450709d8ec3da71149a480fb07c80668b98425c657471e339ba4b919bc2","source_content_sha256":"1b782afb3ea77d326167c0cdc75cf3bafd8c54f38e6656373399c6745f9a6cda","target_token_ids_sha256":"97d18e68c69d67cc25a2770126b11ec9a4009b8103ec5fbc12cf9840d69f3f7f","token_span":{"cache_exposed_start":124,"cache_exposed_stop":124,"prefill_start":0,"prefill_stop":77,"scored_start":77,"scored_stop":124},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"5220a6e57a98f18ce0fc3020e6cb3e579946006f270473a46fff4ee28d307f6e","anchor_positions":[10,21,32,44,55,66,78,89,100,112,123,134,146,157,168,180],"anchor_positions_sha256":"ecfeff7f0aa8026e3a3e67c7fccf117f33e2cac59220b1af92163b27f13f2667","canonical_id":"763","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[10,21,32,43,54,66,77,88,99,110,122,133,144,155,166,178],"fisher_boundary_sha256":"10832530119cc9dc7e8f231ec358a5a567e0064379fc6c4d752e727059021017","horizon":1,"input_positions":[11,22,33,44,55,67,78,89,100,111,123,134,145,156,167,179],"input_token_ids_sha256":"d8e6511cf865ccef1847855dd93e81b5f9a377da5712f1e2b5e6b7a1d615ed7b","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[12,23,34,45,56,68,79,90,101,112,124,135,146,157,168,180],"target_token_ids_sha256":"98ea0caf56819312967e777bc6cb39044d229924079654c58f5313c625d125a9"},"formatted_content_sha256":"1b9193d68d5fbee2ffb750c200030e38b128c512a5dd53901cd0fbc6e9110bf6","generator_receipt_sha256":null,"identity_record_sha256":"a6f87c07fe76e3512de8f93c16a19c9ddf32b2cb8f73284e08fa27a983b4e54a","prompt_token_ids_sha256":"e163147f03192bd26bc2e593039155967245812898896a5b3ccb488a19a3c2d5","ruler_category":null,"seed":null,"selection_rank":54,"selection_sha256":"253e3c374826f96651ce10f0c8bf234e0019e15677351f0a5fa4ac234821073d","sequence_length":181,"sequence_token_ids_sha256":"b058735b952391450e5ec12b9e1a916c6996ae263f8a33dfff1ecab2519a6f62","source_content_sha256":"c7874b2b046cc08bdc1d3485818b5e8aeab82684fbbfe9b46528e3f93a8c34c0","target_token_ids_sha256":"6c2dfbee49b175c1cadd33dbec419143d5d228a7875f0d9be04cef929c3fa52f","token_span":{"cache_exposed_start":181,"cache_exposed_stop":181,"prefill_start":0,"prefill_stop":113,"scored_start":113,"scored_stop":181},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1f12a81f4bcfa4113d746de8d83b09b12d5bf22d5365a3e007cbd5533bc387bf","anchor_positions":[5,11,18,24,31,37,44,50,56,63,69,76,82,89,95,102],"anchor_positions_sha256":"09c822b94eb395a26f0742b1f48efa1c07ea626dba571c077685b9f6cf7f997a","canonical_id":"924","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,11,17,24,30,36,43,49,55,62,68,74,81,87,93,100],"fisher_boundary_sha256":"9076cfcc1469ae3fc7502f41192086d69ec6e14f6981282dccd0613cf5f2b721","horizon":1,"input_positions":[6,12,18,25,31,37,44,50,56,63,69,75,82,88,94,101],"input_token_ids_sha256":"9c80dd3459e67298ce46335c51e9430667ca9c812c445dd5bda165182570e1d0","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,13,19,26,32,38,45,51,57,64,70,76,83,89,95,102],"target_token_ids_sha256":"5e9a4a1aad513778895ed6ffc2980b76512073ab29d4f8af14ae4e26e16fedfe"},"formatted_content_sha256":"e2c6271b02f2f53ca02cc8d25e64ff5633054d7c8d080fc11d7ea4f2182c8230","generator_receipt_sha256":null,"identity_record_sha256":"3c4356ae7b50150676b6594053c05745a4c6c37d4741a1a66db1c3426dfd2121","prompt_token_ids_sha256":"ec37b31ed3931f32732446c5e1b53853e57d83b71f6a60ee0973fc8544644cd2","ruler_category":null,"seed":null,"selection_rank":55,"selection_sha256":"257fcddb413e4d8b091f6e8d2f933e3a3702db72c89eba0d10b57f6f9f5e5204","sequence_length":103,"sequence_token_ids_sha256":"cfe2397b7ef9e35d26711486c836646e4598be218d361850d477f09eb5e24830","source_content_sha256":"fd2c6ff7f8bea809d74c434b1e914cc7cad37c2c16e71760033867b551f8b17a","target_token_ids_sha256":"e984af6102dc89335e836cc40d95bac564e888d124d337cc3b4d2293c69a593c","token_span":{"cache_exposed_start":103,"cache_exposed_stop":103,"prefill_start":0,"prefill_stop":79,"scored_start":79,"scored_stop":103},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b4b039a0bf0a677663a27b0e6225ea5659bf81bf4816ccd4aa2be56fbfe16c56","anchor_positions":[21,44,67,90,113,135,158,181,204,227,249,272,295,318,341,364],"anchor_positions_sha256":"2f350e6327acbc5ad68acdc17f3114f18631a5418462d1efa07225d17dee32b6","canonical_id":"839","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[21,44,67,89,112,135,157,180,203,225,248,271,293,316,339,362],"fisher_boundary_sha256":"b3aaea1f73f541b61f04c11357ca1b117be9c6ed377f11f60a993f9550d29493","horizon":1,"input_positions":[22,45,68,90,113,136,158,181,204,226,249,272,294,317,340,363],"input_token_ids_sha256":"14db27ee204411b76eac1110cc2eae63a9f7b2b51b8dbe5b39e057e911aad683","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[23,46,69,91,114,137,159,182,205,227,250,273,295,318,341,364],"target_token_ids_sha256":"db3248528512ace9a5926f28510801745ad1c3f595f00a85a70220785378a26f"},"formatted_content_sha256":"1fa99f6cf27a6488f0fc65f628a1282d325113dc82a07170cace8f0ceb591a14","generator_receipt_sha256":null,"identity_record_sha256":"6b6998a44c9be26bdd92e8fd5c97777ab53fadd0893edc4f08c5b3238dd07613","prompt_token_ids_sha256":"4be0192a68dd883cc6c905e04481fe02c58edb320e51cffa3148acdb235c27c9","ruler_category":null,"seed":null,"selection_rank":56,"selection_sha256":"25acfdc7bd97d8b8918d952ccd13767a2fe3d9db392ff96de0914fe990c96767","sequence_length":365,"sequence_token_ids_sha256":"64f01ee3dd9e450ba6406f15a3eb8206442ffeb1f8bad35135cdc10036187849","source_content_sha256":"e16fff97f3fb1d8593c7f03f52f248825604aeb9e11bb511b613867c12fe0d69","target_token_ids_sha256":"beb8edb12ebba49a31efbb316488aaea2754c69c9a528a3fc8ce2e25cfd56f2b","token_span":{"cache_exposed_start":365,"cache_exposed_stop":365,"prefill_start":0,"prefill_stop":292,"scored_start":292,"scored_stop":365},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4e7d65195091102780a3b899d1d9c169f8999fd52b4fce6cd4807da888ce72a6","anchor_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,207],"anchor_positions_sha256":"db093e77a831efd90fa9c853cbfd07cb4a1ce073a3d26ebd584589b17a67dcac","canonical_id":"870","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,50,63,76,89,102,114,127,140,153,166,179,192,205],"fisher_boundary_sha256":"e55af1a7964dd59bb74a21c91ffa723d2be196dc03399d61f54ed5e27acc11f1","horizon":1,"input_positions":[12,25,38,51,64,77,90,103,115,128,141,154,167,180,193,206],"input_token_ids_sha256":"7e51406e9532ccd83f5838a79143de8b54bddcb3cd4af48ff57ce8fd2068004b","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,52,65,78,91,104,116,129,142,155,168,181,194,207],"target_token_ids_sha256":"8dfb99d49da149bda798f4f1bea7693afcc685391f66174bd7dbd7f9f212d23a"},"formatted_content_sha256":"0671c01c177d3d976dc26f74b1532f125da8dbbcba201bd969a8df89be404212","generator_receipt_sha256":null,"identity_record_sha256":"80065f067cbcd3b6b24a83d3b0920140345cdb44209861d9942cfc40e6285afd","prompt_token_ids_sha256":"11ac1cd8c2c5a0bfd9a43b00c6dbafc129c255c7fcab76b0fccec71b0a1ca47a","ruler_category":null,"seed":null,"selection_rank":57,"selection_sha256":"25d5e31e60deea1f490886549d37023191fddfed4a7488c2ecb258dc7b690a34","sequence_length":208,"sequence_token_ids_sha256":"b95e7638baa81487e766778b92d51418007619c24c88596cb9d255d17707ad82","source_content_sha256":"25b998a219b3e624a18b48a424e9201d591646f6ccffd2c34ccd994d27c33467","target_token_ids_sha256":"2c9edc0f2e71ceb46be5cf74a768dd85d0f0568d35da9346930b4437902992c2","token_span":{"cache_exposed_start":208,"cache_exposed_stop":208,"prefill_start":0,"prefill_stop":171,"scored_start":171,"scored_stop":208},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7179a32a80e84307ba53490fd3d26ee4976520fb95463fd4d99c5eeea5f95e3f","anchor_positions":[10,21,32,43,54,65,76,87,98,109,120,131,142,153,164,176],"anchor_positions_sha256":"1a7be41f5dd5dc6797d88e73a587747cdd29dd38e9373f3f584d8051e8e93686","canonical_id":"862","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,31,42,53,64,75,86,97,108,119,130,141,152,163,174],"fisher_boundary_sha256":"1eb3dcd5d0f40487914047d0829182878435e58383687684c8e2311b203aae37","horizon":1,"input_positions":[10,21,32,43,54,65,76,87,98,109,120,131,142,153,164,175],"input_token_ids_sha256":"ab632089e4e539c8bd8472cc5c658a6c7457d8f007cff89f0e00204509423230","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,33,44,55,66,77,88,99,110,121,132,143,154,165,176],"target_token_ids_sha256":"a1faf4cf5f2ef1412bad68d667096b1c2dd483ffe1a6e641ce38ec7132214cfd"},"formatted_content_sha256":"16b2614d3f18e7e4f72755cb6542e4c740bbd15964d402f5b8deeaaf2091d4c5","generator_receipt_sha256":null,"identity_record_sha256":"40dd54e4e34693f385579ecbdbbfe9d627292003fa2bb0e3fa8bb8b2106964dc","prompt_token_ids_sha256":"a20a042120ba3b2dda09c804a843b20532bf4809c88d3fbe22620ef7ac56bdcf","ruler_category":null,"seed":null,"selection_rank":58,"selection_sha256":"26d4c81b5d924024df658aeaf2f322dcb51e74e59299d6c5bbd15f755f66be5e","sequence_length":177,"sequence_token_ids_sha256":"f51964e8ff8c85d8f090c0e12f2c0d8d6234b49c9b6256141d05ab65cbec2f56","source_content_sha256":"4ca1375a378716ffef02c6f4407d8935736ee95bd694754667893b4d008fe701","target_token_ids_sha256":"643aa7765da4c82ea6a56654b78d8a8b5489c8fe86d2c44a8b4ac1960295ddbf","token_span":{"cache_exposed_start":177,"cache_exposed_stop":177,"prefill_start":0,"prefill_stop":130,"scored_start":130,"scored_stop":177},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"c008fb00c4b75ed5a7d4956b63ea532e7def48f60c4527826df4af3da44bf419","anchor_positions":[4,10,16,22,28,33,39,45,51,57,62,68,74,80,86,92],"anchor_positions_sha256":"adf3767c4c3aef64d448368c378ec2e8623df9971d13f03dfbf0a245e0d32c61","canonical_id":"877","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[4,10,16,21,27,33,38,44,50,55,61,67,72,78,84,90],"fisher_boundary_sha256":"2b4cbf4ae8461adf7ece8f2d75ed8ba8fa4760a94e8ccbc37beabe48a2e53822","horizon":1,"input_positions":[5,11,17,22,28,34,39,45,51,56,62,68,73,79,85,91],"input_token_ids_sha256":"11d9eb276cb6e38836f4280a877421b19798f0a03c8c6a955cf798cffbc5e066","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[6,12,18,23,29,35,40,46,52,57,63,69,74,80,86,92],"target_token_ids_sha256":"9a807734bc0b75cf522407e78a2432d110e1c4f600034c299f2c1f1025bf651b"},"formatted_content_sha256":"f0d53963f87750fbc186ec354a9b0e5f12a9937a4a1e0aa5b14a393ae9209437","generator_receipt_sha256":null,"identity_record_sha256":"b69efa65a89b1b762f27440c2fa6d016a863c179526e602947c476c8cce0c500","prompt_token_ids_sha256":"decd083790a01b0d5ff5f3cedced5d9fc60db4ac239215f87d18bd13829340ee","ruler_category":null,"seed":null,"selection_rank":59,"selection_sha256":"27e1898502ba84f86677c9a8c281f209e5e5ca3ef3fa0ac74ccb2e5a0f96925e","sequence_length":93,"sequence_token_ids_sha256":"0c186015d98c4b3b835615571d680037c4e9ff03bc0bf360d7edcc64ea7e02ce","source_content_sha256":"a4d868515f48168874e3345764d92c8f9bed1be2211fa137d653955ce91e9231","target_token_ids_sha256":"05325f40cfa6f59d7dae81f8cf700730aebfff74fb1d0bfbf6df3b22888a7ec9","token_span":{"cache_exposed_start":93,"cache_exposed_stop":93,"prefill_start":0,"prefill_stop":71,"scored_start":71,"scored_stop":93},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"69bc3d226f2cb8a6dd05d7b71e6b82bd59279baf47672ff9264b7f90657b2ed6","anchor_positions":[9,19,29,40,50,60,70,81,91,101,111,122,132,142,152,163],"anchor_positions_sha256":"868d4ebf721b69ebc32ed80fb23287a93509d3cda6e8df7dfacdd2b56b0be622","canonical_id":"672","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,39,49,59,69,80,90,100,110,120,130,140,150,161],"fisher_boundary_sha256":"824632fca0be6e4fc45bd8a157d6c3d7331848be3ae62303610c9ced5cdc4fe5","horizon":1,"input_positions":[10,20,30,40,50,60,70,81,91,101,111,121,131,141,151,162],"input_token_ids_sha256":"4a43f0478e855c786a9ba41f412abef099a0aa6d336add8b63253a02536aff3a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,41,51,61,71,82,92,102,112,122,132,142,152,163],"target_token_ids_sha256":"5a469a6d9ca0e01f116bfc1713d0adff5f5c3384fa311978e20183e1b4e4cf16"},"formatted_content_sha256":"bd36f5a6ad55b181f08602b3596e02127fc18650dff36d4e78601272f208d8d5","generator_receipt_sha256":null,"identity_record_sha256":"b5b377fb132254b8031c1002255dcb18bec16d00c60ca2ad998a8649aee007f6","prompt_token_ids_sha256":"6b00d6059701e0be4b83e9dac571ad728ef3c38c00c26f87d6b67a85ef08f81a","ruler_category":null,"seed":null,"selection_rank":60,"selection_sha256":"28dac04649a15d3243c93bc4206efd4f0acd9190fdbbdfa2ab5345af799d2bf1","sequence_length":164,"sequence_token_ids_sha256":"db9853f0252606fe408f3f653940737ffa69d88b087eaf4b282f42551da66ed9","source_content_sha256":"2e61b0813b9f940f7d87c5121e73ed7a0b8b2d507012929e153f7b120c8c94ea","target_token_ids_sha256":"4282f90a8268a9ed7bc92f471b79152055c095686127816dd9ad5e82abe8893d","token_span":{"cache_exposed_start":164,"cache_exposed_stop":164,"prefill_start":0,"prefill_stop":87,"scored_start":87,"scored_stop":164},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1de6107371ef2883413c366d370b6b037d94c76ec7232ca2b34bb6d3efeed728","anchor_positions":[13,27,41,55,69,83,97,112,126,140,154,168,182,196,210,225],"anchor_positions_sha256":"87cf1b468f68c1b9ae4a445f5ca6ddac019379b906bcb9569e18bc77f8dab19a","canonical_id":"894","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[13,27,41,55,69,83,97,111,125,139,153,167,181,195,209,223],"fisher_boundary_sha256":"50541ef032b9ebd546fb5146735c5dc646a875ccea81f95e3135cfb092f777e8","horizon":1,"input_positions":[14,28,42,56,70,84,98,112,126,140,154,168,182,196,210,224],"input_token_ids_sha256":"29b3d6ebd87eeec326946bda12535cdd931e5c75481c27025e5044b0cf8d5ae0","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[15,29,43,57,71,85,99,113,127,141,155,169,183,197,211,225],"target_token_ids_sha256":"468a710354f96e062d0f534f2df0e373bceff5193c12fd9ae67b05776f4cb12d"},"formatted_content_sha256":"85fe4f431f987f9bcccff54e9fff464b3411229d14fbb421bec0c42bf02159f6","generator_receipt_sha256":null,"identity_record_sha256":"78c4954da5b1f2cd2dea1777612b390c51dd7f053d98f84f8dd93e4c16182ef1","prompt_token_ids_sha256":"366bcf6fff0a082cca6c159ef4ee352857b3adb5dc747a33fbe9d5a187cf1a70","ruler_category":null,"seed":null,"selection_rank":61,"selection_sha256":"2a07715411e7dcd5d13fd6bdab1eedf96aa272627f4dcec5410c9273f87bda59","sequence_length":226,"sequence_token_ids_sha256":"060aa83ed0ef449b43aebe0dba81de96624ea81a91b9f2bbf5346ec7ebba5d2d","source_content_sha256":"25f8c228a65fe0501ff126cf8215ef8c40f73ba446adccaac8696c0592d9cb99","target_token_ids_sha256":"5d62950c056f097d29c50fc4fa74da37364692566b85b25d8801ce7fd0cfae6e","token_span":{"cache_exposed_start":226,"cache_exposed_stop":226,"prefill_start":0,"prefill_stop":198,"scored_start":198,"scored_stop":226},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e9fc3e673c65e510fa4e60bee32295e16ef953843d59dc679325a149cb63954e","anchor_positions":[13,28,43,58,72,87,102,117,131,146,161,176,190,205,220,235],"anchor_positions_sha256":"a5ffcefdcb22ed36cff0992ac04165c50a214c58858b64309c2292bae4ad2df4","canonical_id":"911","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[13,28,42,57,72,86,101,116,130,145,159,174,189,203,218,233],"fisher_boundary_sha256":"a343b73549c45e626b75cab9d67da7ec20d01d1df92c5ea43c27940cd5ac7b0d","horizon":1,"input_positions":[14,29,43,58,73,87,102,117,131,146,160,175,190,204,219,234],"input_token_ids_sha256":"fd590f614ed1c54400b44a8923440159f0d548ea06103f72a56c7794eef8d355","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[15,30,44,59,74,88,103,118,132,147,161,176,191,205,220,235],"target_token_ids_sha256":"84f73363046430aa4e0c6c4dcd11b1cf627b0582b1c34fc1e0b65e35a6e9f9db"},"formatted_content_sha256":"3af453e7ae9642b7b3a6fcb1a5eeaa953e64e0f8b2a19e654ef7c76e72f1ff54","generator_receipt_sha256":null,"identity_record_sha256":"f4ea17bad40bae1caf3eba8695e7ad325112bd00019e2511ff3614fb4f3a37f4","prompt_token_ids_sha256":"0b618b4457d4e22f6ca61e219aee86cfa88f0ae704ed487315c854f83ad3c11b","ruler_category":null,"seed":null,"selection_rank":62,"selection_sha256":"2a3d19972c5985a3c0f494d32fc1cf627ec69e3ceffcf0239862d2c14a69ee16","sequence_length":236,"sequence_token_ids_sha256":"4b476aa638606f6f73086d180b8d59e8f59f03a7f0df8b934e603b7a121b0987","source_content_sha256":"fdcefafb28d63d4587a84cb4bad47b369563ee74a482a1216341f7f6dba5f2d6","target_token_ids_sha256":"de9d6a1e7f34b8648d9cc024467db18a0100561d5d78da9dd043028f0c4b1058","token_span":{"cache_exposed_start":236,"cache_exposed_stop":236,"prefill_start":0,"prefill_stop":171,"scored_start":171,"scored_stop":236},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7b01d8ec76dac2b7f8b13a0dee07ad2225b142de86be4c08fe417f349fab06b2","anchor_positions":[11,23,35,47,59,72,84,96,108,120,133,145,157,169,181,194],"anchor_positions_sha256":"366062d114b4b8abb7d67fff75b399eddce3c91d9f37bfd07f29cdd16e658363","canonical_id":"907","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,23,35,47,59,71,83,95,107,119,131,143,155,167,179,192],"fisher_boundary_sha256":"3e6fc1c5625bb4f9b6651ced2a909d037dc5aba4278dcfc13b85144e9142af0f","horizon":1,"input_positions":[12,24,36,48,60,72,84,96,108,120,132,144,156,168,180,193],"input_token_ids_sha256":"ede325c0962debe0dc616d99ffbf44bf49b11833fab47e5f51463bdf2ffd6011","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,25,37,49,61,73,85,97,109,121,133,145,157,169,181,194],"target_token_ids_sha256":"1d7cf9fe79dc3a0f99a9c1c0dc95972f78b47a89b20d976ae513c41b67a2b4ea"},"formatted_content_sha256":"99bd7655cb5e40f7f33cb82dee142cbc78854fc3a5fe6777c50f8ee8f6340cbd","generator_receipt_sha256":null,"identity_record_sha256":"8d7818a7f7ed0b3732c51529861e8d1f37b442f565e0755ae9412f4e544cb0d8","prompt_token_ids_sha256":"1d66bbe818bc1c82dece92e787302ed8be7bcca6dd90590e6de294c4d367d8be","ruler_category":null,"seed":null,"selection_rank":63,"selection_sha256":"2a42596fa08d9ebf6f8f895ed62f34cfc00f343a1b8fb9eaba4fce50e57632ff","sequence_length":195,"sequence_token_ids_sha256":"0239420cf0bc52b4ff133acfa7d4ba2d445aa4efa7ca99f95e10b1e7fb7bff32","source_content_sha256":"523274b507fb693b2bb38a9148d6f328b15704929202f43a1eadb3f695c038b9","target_token_ids_sha256":"8225f2a151273aa3c714667358b8a7f304f775c7bed4fa3e4aececc369b6aff8","token_span":{"cache_exposed_start":195,"cache_exposed_stop":195,"prefill_start":0,"prefill_stop":139,"scored_start":139,"scored_stop":195},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"dca37f2fe51bc394708fd6c2564b0cbf119163fd665608a4d93cd495b3998ad1","anchor_positions":[7,16,24,33,41,50,58,67,76,84,93,101,110,118,127,136],"anchor_positions_sha256":"b394d1b64998869298fd2b292d4ab84e6faf0b5a40eeb811c668f2563bb4df64","canonical_id":"744","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,24,32,41,49,58,66,74,83,91,100,108,117,125,134],"fisher_boundary_sha256":"31ee43d6395a61f1005ea89235db2d9169957c4fa0ecd817d09d92731e4bfff7","horizon":1,"input_positions":[8,16,25,33,42,50,59,67,75,84,92,101,109,118,126,135],"input_token_ids_sha256":"305b211372837f28d3c09a9195ff9da8870eb18255d1a5ade3aa23447ae066d7","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,26,34,43,51,60,68,76,85,93,102,110,119,127,136],"target_token_ids_sha256":"55671dc7b5a880e5aa6b49754d8ad5916d76392b74ba47f6eb14a67a99a65dd0"},"formatted_content_sha256":"24bfd2c8acd42c24dd0175dade66586d4ef9c8fba6740ab43be8a40d2344a8f0","generator_receipt_sha256":null,"identity_record_sha256":"0d3378f11bdab707d5a5e866bf31ece77bae782c5e98264606c8669811279e26","prompt_token_ids_sha256":"45cbb79cde4efb50926321115cb572632eaa8fc91f1b03e56edb1ec881aac0d1","ruler_category":null,"seed":null,"selection_rank":64,"selection_sha256":"2b946010dfbfbf789ca965f025d6a4ef990e3f3685bc221716f664a8af147cdc","sequence_length":137,"sequence_token_ids_sha256":"27302f59061cb4d77641b855190e68d1082efe4fa19ef17f6d1886ea443d271c","source_content_sha256":"388472fd029ca253a1e8ebe0b058b306b19e8d0a99d80ac0ff0cc80f6ef6401e","target_token_ids_sha256":"f529366ede3a080401140af1ed24d8b5dc889784a050980a1baf21988730f4ef","token_span":{"cache_exposed_start":137,"cache_exposed_stop":137,"prefill_start":0,"prefill_stop":106,"scored_start":106,"scored_stop":137},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"72f1915bbcebb3cd41116e7d18a62d81645a57d6ab6aefc5dd4d833040f03e86","anchor_positions":[10,21,32,43,54,65,76,87,98,109,120,131,142,153,164,176],"anchor_positions_sha256":"1a7be41f5dd5dc6797d88e73a587747cdd29dd38e9373f3f584d8051e8e93686","canonical_id":"858","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,31,42,53,64,75,86,97,108,119,130,141,152,163,174],"fisher_boundary_sha256":"dce1d589b941aa5d007836b839a80510d6ed27d69a6bc4826bc4f9d1be58fa8a","horizon":1,"input_positions":[10,21,32,43,54,65,76,87,98,109,120,131,142,153,164,175],"input_token_ids_sha256":"1f8edd352b430cc0ec42d176428e4dc421cd81f094568d0cef527f3c289d718a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,33,44,55,66,77,88,99,110,121,132,143,154,165,176],"target_token_ids_sha256":"bd43a32b864185de3d60dd13a588ea5c44371aff58aa9439eee1749afedfd427"},"formatted_content_sha256":"ffcfea4052a70d3b74318b9aa27367efd191b6bb39153e56b0d8c6be270bf7dc","generator_receipt_sha256":null,"identity_record_sha256":"4303eb5887d9002f18afa44a53d86830e876c3605b807968cb2327574864c1ad","prompt_token_ids_sha256":"7e2722d66f52a4b3e9bc2b499fe7271693a49bfccaf075db172096a06590501c","ruler_category":null,"seed":null,"selection_rank":65,"selection_sha256":"2bdf2de5737b5e9e9c804506d9b637c0b3e4c429c713402914877db9e163698b","sequence_length":177,"sequence_token_ids_sha256":"1bd4f0e350599ac78e054ba1f3313444f17aca30b3ab8a7d1b97b0a4db36795d","source_content_sha256":"30117a9c1f5c8fa1a71aa5d47b10e7b6a6ed631272d09a212703db54dc4e2b48","target_token_ids_sha256":"8597d05e46c539c2c6176333f41f2d43081ba451957d9e6fb3fab21ad0354486","token_span":{"cache_exposed_start":177,"cache_exposed_stop":177,"prefill_start":0,"prefill_stop":161,"scored_start":161,"scored_stop":177},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"90e208ba484962acfe5a450f2238cad06fa1b1e78cb9b693ad7f8e119d289e81","anchor_positions":[9,20,31,42,53,64,75,86,97,108,119,130,141,152,163,174],"anchor_positions_sha256":"3a23db0bdf537a9cc8cf4af3c5b682f02edafbf8e11933d0867e7f5a84a5d5fc","canonical_id":"908","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,31,42,53,63,74,85,96,107,117,128,139,150,161,172],"fisher_boundary_sha256":"3ddff55453bdfe39ff17279167d0bf79d250df7e3566058fbc8400d75148e0cd","horizon":1,"input_positions":[10,21,32,43,54,64,75,86,97,108,118,129,140,151,162,173],"input_token_ids_sha256":"a57955a9e52e60a3866dc92a0785d9ec79382d7f1e9d9a6fbe931b41cbb23f05","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,33,44,55,65,76,87,98,109,119,130,141,152,163,174],"target_token_ids_sha256":"08a89b4fe30bb44a957a32e4dbff9cce9f766b22eb990ab1c15ed78cf1409870"},"formatted_content_sha256":"542026924d7c553982b9f71b589a6b03aa17c1a9071232b6b352a8d6de880b4f","generator_receipt_sha256":null,"identity_record_sha256":"9ab67c828f5ebad51f93eefcc25a6f64e7b2efa682ddcaa0b13cdfaae34bb357","prompt_token_ids_sha256":"92dc6abd8d2bcf9552cb9117b89f426545c93d4bf69ad859fe2e22691583d830","ruler_category":null,"seed":null,"selection_rank":66,"selection_sha256":"2bfb0c5069394a29410fe17d912939039348a3279be0932efb83b1a7c531f5cc","sequence_length":175,"sequence_token_ids_sha256":"470fc1e9439dda64ef76f847132fd61d9cc29b4d1eae89d0d27b45dcf9b147bc","source_content_sha256":"7f2ca7ee05543293bafe5fc5ea510834c2eadcb6c383bdf04c72ec8b68a255ba","target_token_ids_sha256":"fad64c8dc2d584216528c7db193f37eff8c35a166fa9f081fea4f50fc1596f9c","token_span":{"cache_exposed_start":175,"cache_exposed_stop":175,"prefill_start":0,"prefill_stop":143,"scored_start":143,"scored_stop":175},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"ae2a13b9227b88c92f44ed18f4a1e74a1af336a5e505f3993199f634831d84bf","anchor_positions":[9,20,30,41,51,62,72,83,93,104,114,125,135,146,156,167],"anchor_positions_sha256":"eaf3a0e814f88177633695c30e9a05b6b492aa64c90bf98da98cb9be3917c812","canonical_id":"695","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,30,40,50,61,71,82,92,102,113,123,133,144,154,165],"fisher_boundary_sha256":"00b58a22e1ec099f9f89a453c68d32a00b7de79e01353970212f4486aea7a91b","horizon":1,"input_positions":[10,20,31,41,51,62,72,83,93,103,114,124,134,145,155,166],"input_token_ids_sha256":"0d02b321a2c7a93e523c4102c0a515f78295358243305ecf63572a9b9251f793","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,32,42,52,63,73,84,94,104,115,125,135,146,156,167],"target_token_ids_sha256":"479412a3d9bc5c68516840b01dfe7229e1ffffaec904e9405309df32f66e5318"},"formatted_content_sha256":"2c108150869bff59ce632e2f1cf6126ae8d1e5449fb81fa352e6080ece8e18d8","generator_receipt_sha256":null,"identity_record_sha256":"b8d1fbbaf80e50801cbfefb607f87f2ee0be785618f9d8733e4ec1cec193070c","prompt_token_ids_sha256":"12d9523360c9b9bbbc77331eede017f4df2b2073b06793fb6caaa4264ed356eb","ruler_category":null,"seed":null,"selection_rank":67,"selection_sha256":"2c8457d859c30b49c20f86b5a6d57cb9d3f87babdd78e191666700bc6b19c2a9","sequence_length":168,"sequence_token_ids_sha256":"61a8f24e31170e6510a3fcb1c13e03df674e6054bc07e8c1b50b2869262279dc","source_content_sha256":"f26ba058dff18925efba9c51a7c584d713db98910e092b2b8efad4050d705a97","target_token_ids_sha256":"331d6876269e5c72808d45d0ff8043245793020b7f0c47f8dff86c3d5896440b","token_span":{"cache_exposed_start":168,"cache_exposed_stop":168,"prefill_start":0,"prefill_stop":124,"scored_start":124,"scored_stop":168},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2776f19ea8ad43a4ca532ba12548c47a68fb5d577db33f2001594d30daae4b47","anchor_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,127],"anchor_positions_sha256":"be2b9b89820981ce1f31651cf7733d2ed9a608d27c6984c26e1c299ac3d463c0","canonical_id":"674","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,30,38,46,54,62,69,77,85,93,101,109,117,125],"fisher_boundary_sha256":"9fb839c7596b94297977a40da9a8bdea3ac37720cc556e470eac01fad7d8fd26","horizon":1,"input_positions":[7,15,23,31,39,47,55,63,70,78,86,94,102,110,118,126],"input_token_ids_sha256":"4f531382690861c06fb87e927c47ddec1b08260da41fc61f8ade85e2dbfa2447","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,32,40,48,56,64,71,79,87,95,103,111,119,127],"target_token_ids_sha256":"58f5f25b01c19e81aae2e0327e1c394b668618a5926e174aeb8d4db536ad7b5a"},"formatted_content_sha256":"11e4332442cb1751b96af99218e5ed832e0741b4bd279519954cf8ff75df3000","generator_receipt_sha256":null,"identity_record_sha256":"91c60f95fb1de5699a93eac54f860ec4b636acdf00d4bf36d47cb7038568d0e6","prompt_token_ids_sha256":"fc30e67c09e80fb8040481e2b49df5d4de0abb2ca503e59d5f7fe81b9053ea14","ruler_category":null,"seed":null,"selection_rank":68,"selection_sha256":"2c910566f006288010eaa2a19e63636d5cdb71c4dbe532bbb2f800b8bc2d83e8","sequence_length":128,"sequence_token_ids_sha256":"c6637a5c180bd2cb7b6ae0114077150e40aa103770a4414d373677e6197d3eb0","source_content_sha256":"b085122555255bd8b48a5d7202b27dcfa83512407504a3a7d36c26ab0971701c","target_token_ids_sha256":"bf8896134a16b2defd1fd3ad3a1e041bed0c2522bb90fb8750e2d750970281f4","token_span":{"cache_exposed_start":128,"cache_exposed_stop":128,"prefill_start":0,"prefill_stop":92,"scored_start":92,"scored_stop":128},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"62ab5323d823671ba9386621641396cc44f5d6d879e1ef5cedfd73f5d42ea49d","anchor_positions":[5,12,19,26,33,40,47,54,60,67,74,81,88,95,102,109],"anchor_positions_sha256":"04b0f7fc4a0267db36244bf79b1775bc8efc674979a8a894100f0164758c7107","canonical_id":"937","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,19,26,32,39,46,53,59,66,73,80,86,93,100,107],"fisher_boundary_sha256":"d621358bf5b31a9a6e1bf8b7861671a5c3b7d8f4b833117f22528a32214d9059","horizon":1,"input_positions":[6,13,20,27,33,40,47,54,60,67,74,81,87,94,101,108],"input_token_ids_sha256":"6b99376f37916da3177e56d8dbe872f5b54a721dad5a56e3d0f7641537953001","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,21,28,34,41,48,55,61,68,75,82,88,95,102,109],"target_token_ids_sha256":"6caf9fd439e8d278a1020748e4978b74310b3608c9d071b94f12ce03d0b9be62"},"formatted_content_sha256":"3f1c5b9ff9268e2a55000da43f44ffd96c58159b9099ecb106bfc6d0e882d405","generator_receipt_sha256":null,"identity_record_sha256":"2aef9e476b0e90a8d4c54f89251fdd678232c30c0588a40c71c3ea135c3a140e","prompt_token_ids_sha256":"de6bf797fe9bca2c9736422f1027832ef62d4311eea81a92442bdf94a157a32d","ruler_category":null,"seed":null,"selection_rank":69,"selection_sha256":"2db747f786476cd53b2b76e66050c7c9524096bc2cc634dc8949c6700bbb91e9","sequence_length":110,"sequence_token_ids_sha256":"f66fae77a66de49a19604647a7a1dcef25341e73c3beeb2dfc54b05038100bfd","source_content_sha256":"9427a32773d2dda1a56f98cb0bdbee88ed5634a53a96ba4f2adf32764392e2d0","target_token_ids_sha256":"9d11355be764566fab4daa7e657723edeee80c1c0d05f8dcbfe888b93d368076","token_span":{"cache_exposed_start":110,"cache_exposed_stop":110,"prefill_start":0,"prefill_stop":73,"scored_start":73,"scored_stop":110},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"be38fbf54d72a7f68805fbe18ccbdf7981341fa3db7c90146ed5584b8a5632cd","anchor_positions":[25,52,79,106,132,159,186,213,239,266,293,320,346,373,400,427],"anchor_positions_sha256":"2c63cecba171fd759882d1a320b87c2dfaec15e4490a663f9e1099a229e8e934","canonical_id":"783","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[25,52,78,105,132,158,185,212,238,265,291,318,345,371,398,425],"fisher_boundary_sha256":"df1eb852686d1c5fa6f464d0608a8ec972c4dfdb569e90a9d61ac80e9dbbd261","horizon":1,"input_positions":[26,53,79,106,133,159,186,213,239,266,292,319,346,372,399,426],"input_token_ids_sha256":"b4cb4bc55e5f1df30d752a6409003f2a8cb96722509b10f768d46b67657faa6a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[27,54,80,107,134,160,187,214,240,267,293,320,347,373,400,427],"target_token_ids_sha256":"22fcb95368b58ec5a4c13f727526665b92519bc8752f8ab11b799ee2ec751029"},"formatted_content_sha256":"0d8b83c24ac54c8b3c1a41092dc210a7f174f870033838f69387ee0a959cb7be","generator_receipt_sha256":null,"identity_record_sha256":"fd3b5341382c42b888558f7ec9d99892ab4bc6c62b5fea552dfd262f705be6f8","prompt_token_ids_sha256":"3c17fc671c321556046900784e286f9b2efcfec63411dcd19bc16d80f81efb33","ruler_category":null,"seed":null,"selection_rank":70,"selection_sha256":"2deccaa412b35e79530ed8315fcf11416ca3ef9bf945badf8e69214d6c67f0e2","sequence_length":428,"sequence_token_ids_sha256":"923d6ae58844ab43d7c92cc96780db2f0a68359b436399f8f4b84e8bd73850df","source_content_sha256":"3bf5dd95dc2a714c277cb345634f0add55f38f0c23e30c02e2fac976cc5e3560","target_token_ids_sha256":"89d552cd5c74ddddddc518b6c4d8776915650ab6c296da75bf7d8dc8d5e8a3cc","token_span":{"cache_exposed_start":428,"cache_exposed_stop":428,"prefill_start":0,"prefill_stop":201,"scored_start":201,"scored_stop":428},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"d8cc6a91dfe0b8725032b9dbe17d00c2267d1f3903f0f46de298aef09a0eceb6","anchor_positions":[7,15,23,31,39,47,55,64,72,80,88,96,104,112,120,129],"anchor_positions_sha256":"c03e2e14b7ad2333a14fc199839382794d3c90c5842d21c3c2d26acb100d0b9a","canonical_id":"792","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,127],"fisher_boundary_sha256":"36cbcaa5f2a243260e4b704ad965c1eb8729226a615436df507566bab0a15d14","horizon":1,"input_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128],"input_token_ids_sha256":"43ea65b95cf29e7cb303d7b58a8466e49d86ff3b94f481b088aedd213565b50c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,33,41,49,57,65,73,81,89,97,105,113,121,129],"target_token_ids_sha256":"09ed1cfb8661ecaf52c064021fdaac9e5e45d85d244adc29abbadd32cbb23a1b"},"formatted_content_sha256":"84324342a9988ef788199714ffd6a7b7f6493516f982634aeb0fab6e891dca3e","generator_receipt_sha256":null,"identity_record_sha256":"9ea4728a95faf70fc41786042f177a0a108a52c817d7f798a50fa559e6bfd72f","prompt_token_ids_sha256":"e2b25c9110fe041500ee35bda9d717a5b8764c0af9684a331d986da4d503a12d","ruler_category":null,"seed":null,"selection_rank":71,"selection_sha256":"2e13ba7ae758055511fc93fd8b21f4c8279b266645abb65bad13f929bab6d4ec","sequence_length":130,"sequence_token_ids_sha256":"173382b09d1a8419bd02b4281c9d8ea5342d4cee26d9006629bdfe38b23fc3ea","source_content_sha256":"8b4ec72b56e7b421cfd5660acfeaf5323dcfcb622b00a3b4f8a55c2bb8a599ab","target_token_ids_sha256":"45b965687a80da78d6902f0642a6a41c15d5b6065995e352b178741376df5858","token_span":{"cache_exposed_start":130,"cache_exposed_stop":130,"prefill_start":0,"prefill_stop":117,"scored_start":117,"scored_stop":130},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b526b8b7bf92e5b467bf63c4c44c32a7d3baa99496c16d829dfc10c90ef8e179","anchor_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"anchor_positions_sha256":"debc2f860d51bf30c9ccfe7850d3428800e80a005f976ab2943a8642c33fb808","canonical_id":"848","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,30,38,46,54,62,70,78,86,94,102,110,118,126],"fisher_boundary_sha256":"eb5082f8394245be99a854bb9e95a0e8a24b830d5b4e12d9acf89075a71132d3","horizon":1,"input_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,127],"input_token_ids_sha256":"2488867fcffe8e28aca7f38c1051a25ba6e3576b5816a851d585a91e5d1fb9b2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128],"target_token_ids_sha256":"67aae898043f0810d7b6435c2dc11fdfee0f51cf372142272e7060f3d549bd8e"},"formatted_content_sha256":"030fb32fa54afa33ec26144f6963d266d6aa4b969906d8ea12cf708055b33d67","generator_receipt_sha256":null,"identity_record_sha256":"992345ddf332a9b3994c60bd506b547f20b7a90c821dab684435a5333bc84659","prompt_token_ids_sha256":"f4528ab0cef6a0aedb91c8fdfdd0fd4d1d19f26b5c42b52cbb5e9d55c87e496a","ruler_category":null,"seed":null,"selection_rank":72,"selection_sha256":"2e8927b105ba240d08737397b6e4ae7e731e45937512629b5d05c9e422358ce0","sequence_length":129,"sequence_token_ids_sha256":"bdb29154778fe78320ce09bc48fb722bce1a1cb08026d6843fa036de3c59c397","source_content_sha256":"2a1f6cc18c5e358f530bd92c65c306de3dce59f6ab5bf9e55ea9c27c9608156c","target_token_ids_sha256":"d4d73429c6a8b75c9e50462ee645fac5d41af8e10ac5bbeb75778385684a24e5","token_span":{"cache_exposed_start":129,"cache_exposed_stop":129,"prefill_start":0,"prefill_stop":96,"scored_start":96,"scored_stop":129},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"a8b715cff28765151d67536c865a8213c840d7a8b7b255ed28fa0f4f4d86e4c3","anchor_positions":[5,12,19,25,32,39,45,52,59,65,72,79,85,92,99,106],"anchor_positions_sha256":"1ec66db99ded67a6b74f7dd5259127f2ba61fd9a33265762c1ea04710558d7bc","canonical_id":"973","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,18,25,31,38,44,51,58,64,71,77,84,90,97,104],"fisher_boundary_sha256":"9a38f0cf45f0d18c982e6cd4e3041a5c05aaa808ae9c3cc11acd1e5bd81ba1ef","horizon":1,"input_positions":[6,13,19,26,32,39,45,52,59,65,72,78,85,91,98,105],"input_token_ids_sha256":"beda690cd5123bf4bea71d0d29b89a37efdf1c91f271681dad45a48428687cfd","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,20,27,33,40,46,53,60,66,73,79,86,92,99,106],"target_token_ids_sha256":"b2959f450541356aa81bb0e5965581933b267e9e8e2f7bff19f500f4677cd5d4"},"formatted_content_sha256":"03a49d9fc1d2e40de0d006e558a4cc366fe5fc106b78e9dcce28bf991852272b","generator_receipt_sha256":null,"identity_record_sha256":"92e9205a716392965076d0c07b2927b8ceb264553c8ddf5e51d643b2c4447675","prompt_token_ids_sha256":"a60f5de6590c6b61e5ed13b415d33c77fd42dcd611c101939112501a21966e85","ruler_category":null,"seed":null,"selection_rank":73,"selection_sha256":"2f2ec7824d30ee016341d9347bf3033c9e8baefcf7f210237aece22c50a0c793","sequence_length":107,"sequence_token_ids_sha256":"02a78c628534fda7b94bf5fce8dc4c42f06805ffe169c5e88e972974217f23cd","source_content_sha256":"535cf2dcfdda3927e31a863cc6592099702642031c1efc3a80742b6f3ea4493c","target_token_ids_sha256":"8ad5d65ce29243858983f2e234e934222f9ebfb842ba646ed2857ea2ba6d2d5a","token_span":{"cache_exposed_start":107,"cache_exposed_stop":107,"prefill_start":0,"prefill_stop":81,"scored_start":81,"scored_stop":107},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9356ecdbe5e79b88db404f8fcc1cc4066488ab817a3a8d2c38916554dca11b22","anchor_positions":[13,27,41,55,69,83,97,111,125,139,153,167,181,195,209,224],"anchor_positions_sha256":"1b59984280d92d0f5b008fda81e8c1ee99edd8adebbc3555a45f9e3b6ccc7b43","canonical_id":"648","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[12,26,40,54,68,82,96,110,124,138,152,166,180,194,208,222],"fisher_boundary_sha256":"6a4524107689341a41d25534167f49a57a6232f42670a7ccb3c3c53953ca52d7","horizon":1,"input_positions":[13,27,41,55,69,83,97,111,125,139,153,167,181,195,209,223],"input_token_ids_sha256":"5d718cfe7aa9550f8596ed8028ffa484e5dde6a58cf75d5e564d46b03ceb80e2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[14,28,42,56,70,84,98,112,126,140,154,168,182,196,210,224],"target_token_ids_sha256":"29ce1ee9257823ee60fc561f31cf4387cbb83ef9304ba18e1878faf829fa4c12"},"formatted_content_sha256":"03ce2215aec8d5a70b9b7d5d9457a3049c63a14d3fb41d268cb7d640203b379e","generator_receipt_sha256":null,"identity_record_sha256":"47caea8e58a24ad7e9189aecc0d5cd34719c3296041ecb933c3966a471cb646f","prompt_token_ids_sha256":"c856d4839a1ad8ebc26aefea0a9fb07e5436dd63c6a305c7b4112bee6193dde4","ruler_category":null,"seed":null,"selection_rank":74,"selection_sha256":"307a74054631fd4611a4637c7dcc3b5902cc9b2530ad55e1817a7b66e0244a87","sequence_length":225,"sequence_token_ids_sha256":"996568b0b6b68ffab6ac7da97e43dcd7e5fdaabcdae5248fcf638e8c03d9454e","source_content_sha256":"c078071a1981a4c9334368583a6c9b84c5bca8b86cdedb85a12e8be1b1203123","target_token_ids_sha256":"c8d5a3a6d636c7fc1ee25da058f4b6f9c67be6202458fb01fa42861b8939a7a5","token_span":{"cache_exposed_start":225,"cache_exposed_stop":225,"prefill_start":0,"prefill_stop":172,"scored_start":172,"scored_stop":225},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"806ebe5a2818da11223ca67ee7cf2265f233845126be6d099c9dcb15c55fa08c","anchor_positions":[9,19,30,40,50,61,71,82,92,102,113,123,133,144,154,165],"anchor_positions_sha256":"f50a8793906ae0ec9418f440c4efb82f7805f24bf465b189ea08ad0bcb356f8f","canonical_id":"772","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,40,50,60,70,81,91,101,111,122,132,142,152,163],"fisher_boundary_sha256":"1b7250139134ecd49bde104b1a6a2510822a0f9dfb9d2d4e9e5eecf763a8b9a8","horizon":1,"input_positions":[10,20,30,41,51,61,71,82,92,102,112,123,133,143,153,164],"input_token_ids_sha256":"3e9f5ed8311d8d669c372b59caccce107e54dd1559931f673bbb25676156cadf","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,42,52,62,72,83,93,103,113,124,134,144,154,165],"target_token_ids_sha256":"43edf81c690bf9f561ba94748611186c5eed3e9186c085b2808196492de0f6e4"},"formatted_content_sha256":"b065a8530085bc045a0533e1eeaf8aaf8188f595824a24047f8870dde69f04dc","generator_receipt_sha256":null,"identity_record_sha256":"9f12e75bacb719e1dd7725bb446586e29d53441710a6bd0bee305318116aa5ba","prompt_token_ids_sha256":"deba03b89b8f2bb57ae57611e69505484898b45ff63c512c33597cb6fcdbb1b9","ruler_category":null,"seed":null,"selection_rank":75,"selection_sha256":"3116bf2b1674d3472c0f8a996afd05f7fc073f3720ffdb297b5d89cbc3eccae5","sequence_length":166,"sequence_token_ids_sha256":"beeb3758c81c70dda2bba27ac2041d61adaa245c5d8d0edf701cdf0f91f45c4b","source_content_sha256":"1ae2ef8f6c027aabd4091686f02cddd085364dc8ca4f35da3877f375ddbecf79","target_token_ids_sha256":"920ea1a628b222d262ce367d32d278f0b295a5028fbca3295770bcff04e208d9","token_span":{"cache_exposed_start":166,"cache_exposed_stop":166,"prefill_start":0,"prefill_stop":117,"scored_start":117,"scored_stop":166},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9b187e5fe961b945d44953c99d7e41a4cbc730ca0aee47b0408c755b6bd9ad40","anchor_positions":[6,14,22,30,38,45,53,61,69,77,84,92,100,108,116,124],"anchor_positions_sha256":"fe7a10db15891cb54d23372b10af49697c0401f54d8f6b37c01417545e9ba930","canonical_id":"764","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,29,37,45,52,60,68,75,83,91,98,106,114,122],"fisher_boundary_sha256":"e6be121ec0065a1a1aaee9d9e86ba55d2577c56a8617ecadebe8c4e3e6fa2ebd","horizon":1,"input_positions":[7,15,23,30,38,46,53,61,69,76,84,92,99,107,115,123],"input_token_ids_sha256":"8b87308fede122de1697b74353b230a58de366e630c17539cb050893d296a569","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,31,39,47,54,62,70,77,85,93,100,108,116,124],"target_token_ids_sha256":"a3e5f3dfab994ff9d370511f35cb4ce1a6409b31076b0c2fbfc672290cd7dbf4"},"formatted_content_sha256":"dd3955ad349be1f3bbe53adc3231a0d62ae00e248214be3a7b5e980d326fc9fd","generator_receipt_sha256":null,"identity_record_sha256":"700a6d8a14194a1306ce6fb464aea0085a25f117d07fcf265169ed7ae1eb2406","prompt_token_ids_sha256":"66cf8821e075a67969485c8f8e2ae3f57b8384381fd08c0db59c080fc7ed3267","ruler_category":null,"seed":null,"selection_rank":76,"selection_sha256":"315964c2edf3979db8272d920dfef4d6429237f6cc495a6ee3cd3d5549a6b648","sequence_length":125,"sequence_token_ids_sha256":"516180a745f3962c6178902c00099642c3d74217bedb6c055bb189918a4cff6a","source_content_sha256":"690c9b90840fcf0cd5119cfd171717a21c533ed2145ba5803ba93e78d2229bc2","target_token_ids_sha256":"17e59e0812b0551e0a212266dfde313e3b8b138e97f059cabb2ac99726994b05","token_span":{"cache_exposed_start":125,"cache_exposed_stop":125,"prefill_start":0,"prefill_stop":75,"scored_start":75,"scored_stop":125},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e95a3332e3f7eb017e36ba83694e18fd59804b725bb678bfc239e300e0f3db3f","anchor_positions":[9,19,29,39,49,60,70,80,90,100,111,121,131,141,151,162],"anchor_positions_sha256":"937bf130ee4ebb63677095c86a2168cbc15816e3cb8813d07c1ce8340b5fe4d7","canonical_id":"710","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,160],"fisher_boundary_sha256":"6be547ce84d00ff5e76edeb90cf3d18f3f42ed8471ed55e075a934fcf78abcd8","horizon":1,"input_positions":[10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,161],"input_token_ids_sha256":"110af952e2d5d1fb75f8a65fa643ba6711e84effc65a18d37b8308133a6bfb65","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,41,51,61,71,81,91,101,111,121,131,141,151,162],"target_token_ids_sha256":"b28186d7106e163a34e3f412a706d4588d43b0357f9c57bee118d5b51160b7ee"},"formatted_content_sha256":"88240f422a8a69eea08b888c6b9b3c0bb24a6241a7a906d6c5bd4e4f75a9e655","generator_receipt_sha256":null,"identity_record_sha256":"1781d9f90e6dd08f1aba55a3d324d8fa14693f707ca4f071b3b392360bbe0408","prompt_token_ids_sha256":"be734d8f240da897ca816e1ebc35469363413920aebb0b334817102e135917d7","ruler_category":null,"seed":null,"selection_rank":77,"selection_sha256":"3214d5450dc1d7505961e5b95aa8e71c3b102461efcea184dc5d581da13805e4","sequence_length":163,"sequence_token_ids_sha256":"8b2dd644e6f00860a2723ce8ab4d5c0a9ab6aa316b27d063fc4a39356a29e075","source_content_sha256":"15fd5fe74d1087e4941f9d296c749df402bbce707b6f506e7dfb634abdac93fb","target_token_ids_sha256":"5b619991d998bbfbccb9502f7a36880ca09dab9061d34c4dd351784267f88e5c","token_span":{"cache_exposed_start":163,"cache_exposed_stop":163,"prefill_start":0,"prefill_stop":130,"scored_start":130,"scored_stop":163},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"8623504af00a9b25be7bc8e91298e9bf3d79e889943b43f81d79a76ff7b2b08e","anchor_positions":[20,41,62,83,104,125,146,168,189,210,231,252,273,294,315,337],"anchor_positions_sha256":"d86064c4e7cc49d03ee7921a0260f3727647f42da5038fce7e70f58100a9b429","canonical_id":"615","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[20,41,62,83,104,125,146,167,188,209,230,251,272,293,314,335],"fisher_boundary_sha256":"f7c95c163c354425c2add0a88c1c4185cf084e48e914cfdd216090db1b963bf8","horizon":1,"input_positions":[21,42,63,84,105,126,147,168,189,210,231,252,273,294,315,336],"input_token_ids_sha256":"9379deab7e7b61df7302e3f0d09b7e567d9ba26caa1886692eba4506c94310dd","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[22,43,64,85,106,127,148,169,190,211,232,253,274,295,316,337],"target_token_ids_sha256":"80d5c49f9ff9f2ebebef47d67d9b86438b015a070f8c6e2dc84883e9240b9ab1"},"formatted_content_sha256":"b9645bc5ec977dfe41735d4ce5d1f4a734b1f86ea78c4a5646565ebd8c15566f","generator_receipt_sha256":null,"identity_record_sha256":"9b128354be7b44adac95032db55fe4b66adf4ee89cf51ce82d87f3159b662f3a","prompt_token_ids_sha256":"51676de173a112846b34d4fba7305f57b81fb3f9d18e638cc2fa74f8e3a3e293","ruler_category":null,"seed":null,"selection_rank":78,"selection_sha256":"3285b6f67988631b43e377c409844a0c984cfd47e40b19a98584df2d8f61b362","sequence_length":338,"sequence_token_ids_sha256":"6bebf94cb834d1fcfe47dc541cf15d6fc87989ddf47fecdd7be2a90d0663d24c","source_content_sha256":"fdb94f9efd3d2f52a851821fe1708964cb54b4e59675eb6fbb5b37edf3910703","target_token_ids_sha256":"9714d922e9b518657f61cc75eda60ded6a3f1c50ebe5b59d1d6800481296fdd7","token_span":{"cache_exposed_start":338,"cache_exposed_stop":338,"prefill_start":0,"prefill_stop":310,"scored_start":310,"scored_stop":338},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"96cee3de971a921588b61de2e2cee579332da466f101701e569463c8c6a8fe9f","anchor_positions":[9,20,30,41,51,62,72,83,93,104,114,125,135,146,156,167],"anchor_positions_sha256":"eaf3a0e814f88177633695c30e9a05b6b492aa64c90bf98da98cb9be3917c812","canonical_id":"929","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,30,40,50,61,71,82,92,102,113,123,133,144,154,165],"fisher_boundary_sha256":"a4409da62e8624d4a36ab120ae46d80d5347e0949ddb59fa84433d595ab39c86","horizon":1,"input_positions":[10,20,31,41,51,62,72,83,93,103,114,124,134,145,155,166],"input_token_ids_sha256":"ef3502c0e2121d90870bb103716d7830c424bfec40f477dca54ebfeac256169a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,32,42,52,63,73,84,94,104,115,125,135,146,156,167],"target_token_ids_sha256":"0afc3393efde3a0d92114c3ef929fe4bb20b15a93254a54991f7854ac5d8fb1c"},"formatted_content_sha256":"8720c8227a224ac2e19aea8120b6280befcba03b75052032de9bf8e95ca77b04","generator_receipt_sha256":null,"identity_record_sha256":"1386ffcc0e108e711ebccb2df6fc089ab088b8bb5bf75534b414141f41a74a75","prompt_token_ids_sha256":"d3e5efc050ed2ac9f9ae235e854ba1d481168daa7af573882848337902db8cbf","ruler_category":null,"seed":null,"selection_rank":79,"selection_sha256":"3296d51e868118ed66c05092fa54df28b0ca574c48a0cba377e10c955ebf1545","sequence_length":168,"sequence_token_ids_sha256":"e74df24c78d5f48aa647690ed6d124d47ca43f1d31e9c157bbbd81fae882c552","source_content_sha256":"bc0b73708502c4d4ef970ea6606ef292432812c718cd0e89507be65055cb1c76","target_token_ids_sha256":"e0d8a0f88aa5d40291247005bc203563ae5853bbdc6122d0bc21fb681b59aab3","token_span":{"cache_exposed_start":168,"cache_exposed_stop":168,"prefill_start":0,"prefill_stop":145,"scored_start":145,"scored_stop":168},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9209ac953f696c6753bd12b7e3aee86341dec2daa6e7890b20b1f6ca9c923fa2","anchor_positions":[11,24,37,50,63,75,88,101,114,127,139,152,165,178,191,204],"anchor_positions_sha256":"48082390de232d2e911509462169a4b3ebe8a03329ea579a66fe38623c52b3bd","canonical_id":"872","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,49,62,75,87,100,113,125,138,151,163,176,189,202],"fisher_boundary_sha256":"099898e807c53f06b4fefe6112588ccaf699892b69781fb1af96a8f953dceff3","horizon":1,"input_positions":[12,25,38,50,63,76,88,101,114,126,139,152,164,177,190,203],"input_token_ids_sha256":"e90e1cdb40861c5ce546f0f793eca7fb22ff9a2d1fcd28ac90c1c3c41b2a9a8a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,51,64,77,89,102,115,127,140,153,165,178,191,204],"target_token_ids_sha256":"dfe340d203294bbebece21c27b39e76b677b6c6f99f69c75f1c05d25a052a6b0"},"formatted_content_sha256":"ae7cccb0f44aa7dd8ae1d59a7bb423feb46060df6bccc538c0d729941cf62f09","generator_receipt_sha256":null,"identity_record_sha256":"33a68c486b6fb35b93a963a3c1ab2bc5ff188b6005bdce71a974210943d77edf","prompt_token_ids_sha256":"c587ed25a2f2bfc478fbc770f969490260f64e578d5a442189b63d9a9743cf1f","ruler_category":null,"seed":null,"selection_rank":80,"selection_sha256":"32eea1bbd2a2735f50e8ed7192ad280642b4f864877e2e12c74229735f3eb364","sequence_length":205,"sequence_token_ids_sha256":"5cd1de46f6a6a54a60046c16ee57b387b99886733c7d130b2e28f1ae13d4ae9a","source_content_sha256":"7a15fb9218af2f77deb400a4399d8ac77545eab7979f345c0ea992388693aa5f","target_token_ids_sha256":"27908577789a6b8b4d0224aecb42ddeb2609c5ec146dca642424eb2a076e6e7f","token_span":{"cache_exposed_start":205,"cache_exposed_stop":205,"prefill_start":0,"prefill_stop":183,"scored_start":183,"scored_stop":205},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9e18bfe18f9c4da802617b4dee6331466e0e733317487a86f7aef33753525ee9","anchor_positions":[7,15,24,32,40,49,57,66,74,82,91,99,107,116,124,133],"anchor_positions_sha256":"6c1029515cd9fd840ee9baeeb3bbf16de22d914596dc66d6c0b00a017e82aef5","canonical_id":"804","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,32,40,48,56,65,73,81,89,98,106,114,122,131],"fisher_boundary_sha256":"0654213b540ddd6054c3a7a54b4ed68fe2ce52d54601d589e8dd8432c5e03c9b","horizon":1,"input_positions":[8,16,24,33,41,49,57,66,74,82,90,99,107,115,123,132],"input_token_ids_sha256":"e6dbcef9e02b4ece7b578fe3e362efe755dbd02afcae77d6d6f4edcd4de4f4cc","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,34,42,50,58,67,75,83,91,100,108,116,124,133],"target_token_ids_sha256":"0b20024476a22a5a23bff385afa2fb325aef7aecc21280ce17cd65007c86f1e0"},"formatted_content_sha256":"58c227c113ff5dfa4a446ad44a803eb54659e0fbefb3f5f80dbef45d0994bdfd","generator_receipt_sha256":null,"identity_record_sha256":"01f7ec235052c97847150a68e3de4b9c2d022df8fafdb46a04092b4a7ee18c27","prompt_token_ids_sha256":"927da8d69d8b88559b5811091f4fd329f0e47607f28c399059242149e3c7336a","ruler_category":null,"seed":null,"selection_rank":81,"selection_sha256":"3379395523a918b0a3ecf21510bcc8704655d52083f91c7d3bd323ce8ea3e272","sequence_length":134,"sequence_token_ids_sha256":"bc6201b3951727d59c9b6c88db4d0722a0b56896e2812e646426b637a86b085c","source_content_sha256":"0b30c40b1ec355476196d7e4b579e4c837c30947a206b5da87928bc338b51aac","target_token_ids_sha256":"ce0bec16e6ba57588fa2e620c280ea4fdf1f2da0b33b6d8c1bf55efd9f79a60e","token_span":{"cache_exposed_start":134,"cache_exposed_stop":134,"prefill_start":0,"prefill_stop":93,"scored_start":93,"scored_stop":134},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3078f2a2b5f48560ec6b5e149abdc2ec94224715ce8b8b558250fe31cd4dbf52","anchor_positions":[6,14,21,29,36,44,51,59,67,74,82,89,97,104,112,120],"anchor_positions_sha256":"7f8463e6889d5cbbd872abcc6f675ca04c3c0d8d69d266000933478b69489b8c","canonical_id":"713","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,13,21,28,36,43,51,58,65,73,80,88,95,103,110,118],"fisher_boundary_sha256":"12bd3ed950f8f6e2d1ea75538e2472a2bb7fcef26eeb014dce2f68e39c0ef080","horizon":1,"input_positions":[7,14,22,29,37,44,52,59,66,74,81,89,96,104,111,119],"input_token_ids_sha256":"8bf93fb6683667ca762c7e68944d126e5f2c2b6c44cd212a8000cdc09acb27e1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,15,23,30,38,45,53,60,67,75,82,90,97,105,112,120],"target_token_ids_sha256":"bb0137c2b21eefda36fe32ac45aa551c534cfa2ebce4a392aed849f5496ed9bc"},"formatted_content_sha256":"681d439ccf9ce8b1da99801aad0074b3790087d6efe044ae79d7e49571d76ac7","generator_receipt_sha256":null,"identity_record_sha256":"bf5c49f9a99a125e0728cdbdb1103ef16d8f41d27323feb6d66db4c28cbdf8e3","prompt_token_ids_sha256":"689f474ef4fa8ccced0bd23ab2314dc97972bacba54c2a54dc4d440098402b38","ruler_category":null,"seed":null,"selection_rank":82,"selection_sha256":"36a09fe9c145944020ea5abafade01ab7f4a1c538f55c32b2daf644698464fd3","sequence_length":121,"sequence_token_ids_sha256":"df03ea9ab5af6fe2ad270a5fbae4db0bf61e07b1919c60f124a8070bb517a052","source_content_sha256":"9197df3cdc3a76aced1eefeae0cf812e2f43b1c25afc3bd73c6e27fa82f1aeee","target_token_ids_sha256":"6b06cbab3e1a4e869733d23ddc800cba139bbf3020eda6b3ab38843880a60fa3","token_span":{"cache_exposed_start":121,"cache_exposed_stop":121,"prefill_start":0,"prefill_stop":90,"scored_start":90,"scored_stop":121},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"63b9e82e307a72c058464d1b1d8a7f3c58d7fb5152977d0a46c053ff527a82dc","anchor_positions":[23,47,72,96,120,145,169,194,218,242,267,291,315,340,364,389],"anchor_positions_sha256":"8e2b268c6192e66f463da8f9c055e0c9184253267e4f791d54da3db3eed553a6","canonical_id":"709","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[23,47,71,96,120,144,168,193,217,241,265,290,314,338,362,387],"fisher_boundary_sha256":"967b90e282da8354335e13a711b1db1fc3dc1cd2d5f3733153901609f0b7284b","horizon":1,"input_positions":[24,48,72,97,121,145,169,194,218,242,266,291,315,339,363,388],"input_token_ids_sha256":"14ec48de9ee49ea137494da470356e32fd2626ad45fbf1122fb508edd8791526","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[25,49,73,98,122,146,170,195,219,243,267,292,316,340,364,389],"target_token_ids_sha256":"eee798bbc46c0f86ea4c32017a079b8d80703d9f4c5b22ed2e2e6a9edc5c6096"},"formatted_content_sha256":"377241d6ba849220fead3711b06a3790ec65647bdb509adac60123fd052d34e6","generator_receipt_sha256":null,"identity_record_sha256":"1d437dc4e8346fe37a0eee9d9a423bd60824e5834a88120acefb46d1be5ecaf8","prompt_token_ids_sha256":"fd90d23dea7f7557e683b5a171cd89091ccc9e500b8c27afb3c9221f5d9788c4","ruler_category":null,"seed":null,"selection_rank":83,"selection_sha256":"372cfe33288b920e01911b67999a2534620fe6fd180024291e1b86f7f221909d","sequence_length":390,"sequence_token_ids_sha256":"c1e45e96c9091049c070bbc66c111935f57c6b5469213a4d3ca91d3d27085868","source_content_sha256":"1bd1bf4c9a1573325504ef4695c6a08c9641c60e85f7cf68942681af59f0c13d","target_token_ids_sha256":"2c233f19acde716535f8aed74a2b4c7a6ff031e2df4d090d3f77cfd880649eac","token_span":{"cache_exposed_start":390,"cache_exposed_stop":390,"prefill_start":0,"prefill_stop":309,"scored_start":309,"scored_stop":390},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"755396a4616427092f5857ae9f027899125cc2f2dc290df40ac96b2609861e3d","anchor_positions":[13,28,43,57,72,87,101,116,131,145,160,175,189,204,219,234],"anchor_positions_sha256":"019a1d4f16a491584b0b921a03e82e05ab8f21be64e2ef381494510440a2dbba","canonical_id":"854","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[13,28,42,57,71,86,100,115,130,144,159,173,188,202,217,232],"fisher_boundary_sha256":"3978f984679fc70aed874ca972bca2f58c59e79999bd8b5dbaaf51586a103e4f","horizon":1,"input_positions":[14,29,43,58,72,87,101,116,131,145,160,174,189,203,218,233],"input_token_ids_sha256":"f50bcffcfe9946342cfe243e55c6a21e7d36fb6cc50a570f25fa3180e679507d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[15,30,44,59,73,88,102,117,132,146,161,175,190,204,219,234],"target_token_ids_sha256":"d2eb23c855ed7281d5d4f214ea2ec6aa0309bff553e222cc09435b17b3a89dc7"},"formatted_content_sha256":"ad6817ab94e23314f897ae56c55d36b1bfcbf69a7b8d35fb818c81c03d39b3c0","generator_receipt_sha256":null,"identity_record_sha256":"341e83cf4392f6688e220933fa083c53bd3f177ce67aba89005be759ab1ef581","prompt_token_ids_sha256":"c469e78789638c0e407409fe9aebd074eba88c7033481e56ef2dec96fd271eaa","ruler_category":null,"seed":null,"selection_rank":84,"selection_sha256":"374c60a9450ed67d1bb35eb2406c9e00aaf02021508333ed279cecdacc65f059","sequence_length":235,"sequence_token_ids_sha256":"3434905939b518880cc623791311c79432ac4833d7f9739c8a2fd83b9590d85f","source_content_sha256":"2f5b2b2d153d57ad5deccfd519cfcd97185b0cedd8a8e32fbcc6bb3429ff8755","target_token_ids_sha256":"8109cfc76d8f3759529fc35b4e397ce62b4a87dd10005164d11c77adb9a1a048","token_span":{"cache_exposed_start":235,"cache_exposed_stop":235,"prefill_start":0,"prefill_stop":209,"scored_start":209,"scored_stop":235},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3125b8323a121faaca1072275076c923a67557e8121f55b5373de0a2d7742432","anchor_positions":[21,43,66,88,111,133,156,178,200,223,245,268,290,313,335,358],"anchor_positions_sha256":"8cbf29aa79890d46e276ccafa41f46a9c68b76575359288a1ba31b9a687111bc","canonical_id":"698","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[21,43,65,88,110,132,155,177,199,222,244,266,289,311,333,356],"fisher_boundary_sha256":"c83dfd1738f12889d5c8d8fa14927532b8398e8d2ad099c912c8a526d1c14c23","horizon":1,"input_positions":[22,44,66,89,111,133,156,178,200,223,245,267,290,312,334,357],"input_token_ids_sha256":"b4fa8aaa429dfdeb479beb6db13cf536dad62e221419e87d8cd4cc6e49562547","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[23,45,67,90,112,134,157,179,201,224,246,268,291,313,335,358],"target_token_ids_sha256":"0e14956d01089b21589756a95abe69218dcb1a764bacd1f5efde70f9667949f1"},"formatted_content_sha256":"829bb0068e2180b28903791519d5fd2ef9c4873981556f5c4fb97915943ceec4","generator_receipt_sha256":null,"identity_record_sha256":"92efe3c0141538bc1fe1ae06136691ba5bfdf72bae549e16e258bdad5bc9a4ca","prompt_token_ids_sha256":"e985782bb265579625ef2c2a88762e93185a635ff5c9104060000f4dffeff366","ruler_category":null,"seed":null,"selection_rank":85,"selection_sha256":"383631dc03a19eb73d617831ef3de08685e43b584ca707ed72bff063479a80f3","sequence_length":359,"sequence_token_ids_sha256":"4f8db022db57407107163fbccfb9fa14d20b6c9ebb4167358a92b76de52a8cf6","source_content_sha256":"040369c5072294811f7d2b172d9db244cffb410fb58a0ff0ecfd4ac76e9ecb95","target_token_ids_sha256":"df7cdff2b03251bf8483757a1fe829176f9debe6a93f970241f7d8a5e45d2923","token_span":{"cache_exposed_start":359,"cache_exposed_stop":359,"prefill_start":0,"prefill_stop":310,"scored_start":310,"scored_stop":359},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"27e366c7bd906286182fb5657b877a3d983450e230b7e29de32c33658d233999","anchor_positions":[7,16,25,34,43,52,61,70,79,88,97,106,115,124,133,142],"anchor_positions_sha256":"231f3d432eea6372c0ecf521691b115566b5045b38a4ebc776666c6712167443","canonical_id":"847","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,25,34,43,51,60,69,78,87,95,104,113,122,131,140],"fisher_boundary_sha256":"8329a9007c7d4b358ba485f2d16e86dbb70099e16057eba7d09ee5bccd1208a2","horizon":1,"input_positions":[8,17,26,35,44,52,61,70,79,88,96,105,114,123,132,141],"input_token_ids_sha256":"c5de0c4ff62e358587b0368de1f213abd9ae23772b8a469ddfe3d5703c9d1012","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,27,36,45,53,62,71,80,89,97,106,115,124,133,142],"target_token_ids_sha256":"98c5bf9c6ce5ebcc0886bb8c1a84fb66bffb85cf9331303d2fd8e801f62f2549"},"formatted_content_sha256":"072b0e2d3b510043865ddae77cb50d35becf17f9e8114b0d23ee65976876342f","generator_receipt_sha256":null,"identity_record_sha256":"45013aa8aca9961b02b0a4a7cb8344d4f74b6e33cd7e17bde68f4bcc9830f5e6","prompt_token_ids_sha256":"cbc2d13d695e49c699eabbb0c37174336cdd76b1c47e1f754455737973e861a6","ruler_category":null,"seed":null,"selection_rank":86,"selection_sha256":"3856a5bb20d2c0009500867054a1e894dca05be3fb5db8c3123f8dcc626b3326","sequence_length":143,"sequence_token_ids_sha256":"d3b45de44d96d01b589d50a86f386609c7543385f067532a7174592d3748d795","source_content_sha256":"06648ba18a8fa484b8f21f1e5f0581005f1c25c15cbc72488833b9a12475640e","target_token_ids_sha256":"0fec8926851dfdf2d26e890a92aff900619119eb83bb757d608676dac4ded5d9","token_span":{"cache_exposed_start":143,"cache_exposed_stop":143,"prefill_start":0,"prefill_stop":132,"scored_start":132,"scored_stop":143},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"a71bad1c8e08458e094f4e03eef72e9b22e929c4ebde165642691319a2f08b8b","anchor_positions":[11,23,36,48,60,73,85,98,110,122,135,147,159,172,184,197],"anchor_positions_sha256":"7e13e3a1913a1aa40d4138d69b9cd0e10357d1735e6198f7373eda076b3142df","canonical_id":"627","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,23,35,48,60,72,84,97,109,121,133,146,158,170,182,195],"fisher_boundary_sha256":"fa4d10a3491f2cbd7a778d5d9cca07773dcb82334929e2637a6782e847dbba69","horizon":1,"input_positions":[12,24,36,49,61,73,85,98,110,122,134,147,159,171,183,196],"input_token_ids_sha256":"44dc69fff0abd93231a72100f67a65e70f884191a50e851eec9626b5f6d3ecea","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,25,37,50,62,74,86,99,111,123,135,148,160,172,184,197],"target_token_ids_sha256":"737d45bbae07d3bc02b3df44539a526e40a1a65dfb9bbd68a8552b4e3b8a3396"},"formatted_content_sha256":"0a0b695997f5097537b328213722620e421fc13a6da1c75ddb049d61551d4ed8","generator_receipt_sha256":null,"identity_record_sha256":"a708abf4f23637f8abca094ec7198056051b374088f997be873b42a6e20807ab","prompt_token_ids_sha256":"f19ff189894482dc79997369f05a455e1a3ef3d68bf8187d519557d6d6c6ab5c","ruler_category":null,"seed":null,"selection_rank":87,"selection_sha256":"3878b1b7a110136224b32091f63168693baa7e86e3c51b5a5536e13dad73f9d5","sequence_length":198,"sequence_token_ids_sha256":"0f9fcb1d27f0fb0d37209deaa4fa835902344d189e823d011638a2011ede20bd","source_content_sha256":"12ac418a9b4a0d9ae374ce607cfe72460e1c30e426106ea273a46ac4ba95ccac","target_token_ids_sha256":"cf5bb6259eaf6bc72ef3d80bf3723b99a1c4a093c8a643a5b9a663a259d73de4","token_span":{"cache_exposed_start":198,"cache_exposed_stop":198,"prefill_start":0,"prefill_stop":111,"scored_start":111,"scored_stop":198},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9c95bca00064ff088717e40df85e1ee14f4f14ac42b57c865b2a27f9d95db37b","anchor_positions":[17,36,54,73,91,110,128,147,166,184,203,221,240,258,277,296],"anchor_positions_sha256":"92fd03c25ec622578164ea2f5bba6632c7a3ee726d6e1c9caa74a3fe54646d5d","canonical_id":"936","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,35,54,72,91,109,128,146,164,183,201,220,238,257,275,294],"fisher_boundary_sha256":"e2680422efa4a11ce200109baaec769ad06a6773088de15eee3df9953e7958f3","horizon":1,"input_positions":[18,36,55,73,92,110,129,147,165,184,202,221,239,258,276,295],"input_token_ids_sha256":"06925d4de4cf692d9d7ae2fb8defd1d78ee16de18017fea89fc095caf7b42551","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,37,56,74,93,111,130,148,166,185,203,222,240,259,277,296],"target_token_ids_sha256":"a529ab740b332e7c438cb708428ab5907c7a59afc2ac4ac51145f549dfdc5580"},"formatted_content_sha256":"afa72e90063431bd6756ace794a9050cc1610077201aea2fd9b4bda86c662ffb","generator_receipt_sha256":null,"identity_record_sha256":"92f30344725d406c50725f35a3f5b576e19bbd592073bffc0ff773be3cd5ef1c","prompt_token_ids_sha256":"6a725c770525d60d682f99ffbb259336c6f4946dc97f839cd4ca68ee7d0cf425","ruler_category":null,"seed":null,"selection_rank":88,"selection_sha256":"38fb5a0b645f175f62d3e012cf15aec73e7982d5713c59591a4c1d1a08561d18","sequence_length":297,"sequence_token_ids_sha256":"9f51cb1048ef1942bb0af6b3e3d4087006b31ee6c48ec3a0f524b1e08cc51f15","source_content_sha256":"1effae4b4c21d10e095a0a5f927b7cffd241b64be8df419dc05137e160edabc0","target_token_ids_sha256":"6a6840a3339ec51d008e341f8b430ed9e10a11c573b431942795dcfdb1e62c77","token_span":{"cache_exposed_start":297,"cache_exposed_stop":297,"prefill_start":0,"prefill_stop":255,"scored_start":255,"scored_stop":297},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3b13375846101ec060f2a2ecefd25b66c836c8f9c53225732689232378b5cc44","anchor_positions":[9,20,31,42,53,64,75,86,96,107,118,129,140,151,162,173],"anchor_positions_sha256":"5eb476494a95db7e5ff69d760daae6c1b59a438e7d071342cc2f8648e535c378","canonical_id":"827","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,31,42,52,63,74,85,95,106,117,128,138,149,160,171],"fisher_boundary_sha256":"d2950d5bb22979ced35eeb6c35b7e0f442e13245822cf644ec5b7300565507ee","horizon":1,"input_positions":[10,21,32,43,53,64,75,86,96,107,118,129,139,150,161,172],"input_token_ids_sha256":"995c75651110d6eeaf5f7790a1e5988aa866d3ec3f148b43040167cca7bd163f","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,33,44,54,65,76,87,97,108,119,130,140,151,162,173],"target_token_ids_sha256":"4d18d212a638d29f456dfa535bd673c510d66ec32e43594822d9bb9241ca20e6"},"formatted_content_sha256":"4790f2d60d6a241d89f28294cf7110652356521e1216cc3a9e2ce7e0b065bbe2","generator_receipt_sha256":null,"identity_record_sha256":"e93e8a672f73ac122851223c0114eb0974c76a960f1a880f4d4f64c9eb90f40c","prompt_token_ids_sha256":"9cc3fc3392444e349135b1065a5227e3391859f3554be674981d424cbdfa9fbe","ruler_category":null,"seed":null,"selection_rank":89,"selection_sha256":"39165caf08bb54ef81c9f6ea0a6ae1eac6a1c3f0b6f9748dcd2e2d953b5a44c6","sequence_length":174,"sequence_token_ids_sha256":"f5f5521cd3bd481ca316528bee26f7f9355182d73550ff14da89d91f6558600e","source_content_sha256":"c60d7f94f875619bd70591ab5f80b6e8fb8d336938e07fbeed1b32564c1a0b38","target_token_ids_sha256":"fac10609f7b6c41ca9b4bbbf7495988b94cc3b4df19e5dac66d84079bd3d4b1c","token_span":{"cache_exposed_start":174,"cache_exposed_stop":174,"prefill_start":0,"prefill_stop":148,"scored_start":148,"scored_stop":174},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"ad1b36a9a3ef4aa1d78cc9a2babd28dfdd79a75b4a7e69563454a05c10b6e2bf","anchor_positions":[7,16,24,33,42,50,59,68,76,85,93,102,111,119,128,137],"anchor_positions_sha256":"451653062c723e6fe2316470c66684b5f06f6d96d14e7b3bb2fb0c9e7d88845d","canonical_id":"930","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,24,33,41,50,58,67,75,84,92,101,109,118,126,135],"fisher_boundary_sha256":"a46d5960cd9fe3147cb09c59211821b555240915a05f4d76e9a91b70e253bf77","horizon":1,"input_positions":[8,17,25,34,42,51,59,68,76,85,93,102,110,119,127,136],"input_token_ids_sha256":"e59435b4be9b10629e72f1905eae42cfc9f3fd8b77cbff58f09164e96872cde6","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,26,35,43,52,60,69,77,86,94,103,111,120,128,137],"target_token_ids_sha256":"e841c04edad63a6a7f6c695115859daa7a4c3eeb9fe4f2f7094561aff448ee28"},"formatted_content_sha256":"dcb79d95145dcc945bd4f4ac20fe07c33528e7e8d8ef4cfea0312e5b02e817b5","generator_receipt_sha256":null,"identity_record_sha256":"5b49551d1068cd3633b13af41d2b4e4ab38c2383efee9d8c5370832e1a78e65e","prompt_token_ids_sha256":"2eeaa907d87be72e61b726a9ed4672463343033ccb6bf2376754766f298a566d","ruler_category":null,"seed":null,"selection_rank":90,"selection_sha256":"3934218a3cdeb9fe23fe0820bbaf2003e6426a07af797c8318d3df525a7b9d57","sequence_length":138,"sequence_token_ids_sha256":"5c11b18fe521da6db909428adfb48961064e310ff957a9d2a1ade8b72d391175","source_content_sha256":"435474a45f03a3373a0a7f5f78526f3621d01a4daf6907075965fc04440e323f","target_token_ids_sha256":"3fa7542edfba5ed95e02424207a7fc3d1bdc29ef1bb3fb055507740f8a0860d6","token_span":{"cache_exposed_start":138,"cache_exposed_stop":138,"prefill_start":0,"prefill_stop":90,"scored_start":90,"scored_stop":138},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"926ef081a870c670e181e6f21a6dbb6e67418bbe21df0f8841a9da25ad315d93","anchor_positions":[14,29,44,59,74,90,105,120,135,150,166,181,196,211,226,242],"anchor_positions_sha256":"3859b527bfcde30950aa1fd4919bae6af3d771f9dcd051365cd8ca16d468c677","canonical_id":"751","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[14,29,44,59,74,89,104,119,134,149,164,179,194,209,224,240],"fisher_boundary_sha256":"af0bf62935327f2467d479d54e944e5f444b6c444118efcf42d997ed4da7cc37","horizon":1,"input_positions":[15,30,45,60,75,90,105,120,135,150,165,180,195,210,225,241],"input_token_ids_sha256":"f52f3b35b365e905e8af0a0912e656ae9d1e3ad2906698dfa15eff817cdc47e1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[16,31,46,61,76,91,106,121,136,151,166,181,196,211,226,242],"target_token_ids_sha256":"c71c870c1fd6b3ebc3243a03cd21567b7db14cc45512773ebc666af0317504a3"},"formatted_content_sha256":"b9749edd133bc0cefb8007a7358f767d918467015bff6d845fa78b29bdd5b3c2","generator_receipt_sha256":null,"identity_record_sha256":"bb8bf4056e66c85336deaf5c6bf0614c6c6eea646b46a3dcc0b05e265ed2293b","prompt_token_ids_sha256":"4d217c659514cbec1da37d7ca1cf02b1de4d40d095c106f7fd89d4517b8e1800","ruler_category":null,"seed":null,"selection_rank":91,"selection_sha256":"3972ee7b79e05ce309d4c65459acc2d264f7821184dc4baf556f892f50507741","sequence_length":243,"sequence_token_ids_sha256":"22a137779b1349921b5784c3f3a7da9f3700715398d21e5c69536216ef7c5d82","source_content_sha256":"04de6155b488684214357c82595e2a1b0b4162588221bfc1f959c3ad02c64302","target_token_ids_sha256":"4a34e467583136f16ea6deeb64a30a0f625052cbead11e0c2460872fb5a60d9d","token_span":{"cache_exposed_start":243,"cache_exposed_stop":243,"prefill_start":0,"prefill_stop":129,"scored_start":129,"scored_stop":243},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"bc742a938d14512badf1f6f2dc626345160bae405ccb62cc78f8fcf2fbef198b","anchor_positions":[11,24,36,49,61,74,86,99,111,124,136,149,161,174,186,199],"anchor_positions_sha256":"cbd5b119373faf027bdb11d72340904f9e8568f13150612cfe8c0d28d26f0a32","canonical_id":"707","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,23,36,48,60,73,85,98,110,122,135,147,159,172,184,197],"fisher_boundary_sha256":"f539aaf9624257146284435f180cb1ef2b1c3d2448b93f936510956f316e60d0","horizon":1,"input_positions":[12,24,37,49,61,74,86,99,111,123,136,148,160,173,185,198],"input_token_ids_sha256":"53699344c41ad6728511aaab294416c9cb8ab020e2cce05973c6abf72fe3830a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,25,38,50,62,75,87,100,112,124,137,149,161,174,186,199],"target_token_ids_sha256":"c13ae7cca2c350903d6e54179df99c8f0ca7fdd40021679bdcd12e59ab7d3f00"},"formatted_content_sha256":"b7943712ec68b060c30f63f9c1e79415e5d98891bad66512dd84aae0ea33779a","generator_receipt_sha256":null,"identity_record_sha256":"7e8098446b3e22dff19a34b8e6cb18c072889a0914269f5e6c46eda3e278f916","prompt_token_ids_sha256":"aedaaa76edf70a4335d76e3753a2e74b6358828bd7fd27089a9349b82dbc4f04","ruler_category":null,"seed":null,"selection_rank":92,"selection_sha256":"39d4884c3293b83c7da73a4192c68ac6ed7dd2a2b8cb6535c8bc27dabefb2152","sequence_length":200,"sequence_token_ids_sha256":"f84c8958f4ae3235716092ff6d50ea9f9eeeed151677fd0291eefd3f4c046e46","source_content_sha256":"835bcc97198da2e9313e8115ac8c70845ef11e086ca2f36a6cbb1bc74afb15e6","target_token_ids_sha256":"63a1e5e3435f4582095fe5bf45dca9f77c0635e287377e887030050e9576dfbe","token_span":{"cache_exposed_start":200,"cache_exposed_stop":200,"prefill_start":0,"prefill_stop":82,"scored_start":82,"scored_stop":200},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b368a5e3bf6a4d6a9d65ed071a6ae293e6addf944c619d41b4db74fd479ea1c6","anchor_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,208],"anchor_positions_sha256":"e3c9aaa391385675ab3475417e405da92d531052d937e046ddc609185dc2c7d4","canonical_id":"946","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,50,63,76,89,102,115,128,141,154,167,180,193,206],"fisher_boundary_sha256":"6717c7a65af61cfd12e44550f289c401161af90d68d57dd17e3c519d22a1613c","horizon":1,"input_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,207],"input_token_ids_sha256":"35c588f0b227da0babb6432ce2d27c1f6f0ddf842b8da0c7b8c5def4f5567635","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,52,65,78,91,104,117,130,143,156,169,182,195,208],"target_token_ids_sha256":"3cc14e363f767b18bf8fb37c1c7a1681237d5f34c0d8244fad4f8ea124be7de9"},"formatted_content_sha256":"6672cb487be13a90610c98af3babace38c6413c484e2a543e785501685eea0d6","generator_receipt_sha256":null,"identity_record_sha256":"f4090de3fc3464141d40f4b023f18db5c2e540193c894d2b56d314001a2cc540","prompt_token_ids_sha256":"23c086ef1490fd86a6b3f23899489b782e5aef4d38689af88dbd19f7fb7b1d0a","ruler_category":null,"seed":null,"selection_rank":93,"selection_sha256":"3a31bfe17e95e3f782d9b40a9bad5670a4dbab98abaa87ce2414402b4d4c9d3e","sequence_length":209,"sequence_token_ids_sha256":"8002749920d9aa312887fb27e1c8d185977ea0ce5f145e2a0dc684e46b759834","source_content_sha256":"bd64b5ce22599f2fceb61fda2a89a9caed49e3e0b63694d888f83b548c800738","target_token_ids_sha256":"d1d65dd1bea87ea89e2cb46bc0c015ee3a4e8df03fe4a02a29bb30c5549b96d0","token_span":{"cache_exposed_start":209,"cache_exposed_stop":209,"prefill_start":0,"prefill_stop":178,"scored_start":178,"scored_stop":209},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"122dd2605483f8c364752af43aece851c5117635a2c390bbecc2d2c38ee203cc","anchor_positions":[12,26,40,53,67,81,94,108,122,135,149,163,176,190,204,218],"anchor_positions_sha256":"34e0f2438d49fed8c871a33d3818a40fc035b239ea8639ec66d64d1e59a9bb66","canonical_id":"853","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[12,26,39,53,66,80,93,107,121,134,148,161,175,188,202,216],"fisher_boundary_sha256":"41be834ada2a5acf9617b5b43955804d2e6a4ebe3234bb01e8e5dad38bb0a1a3","horizon":1,"input_positions":[13,27,40,54,67,81,94,108,122,135,149,162,176,189,203,217],"input_token_ids_sha256":"afd2edb984710f6fc20bd321b1036090c05a37d38b4b24a9acf50e7bd4160e2a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[14,28,41,55,68,82,95,109,123,136,150,163,177,190,204,218],"target_token_ids_sha256":"b5093288c3e5810d5c3d19010a1acdd8082bcba55f9489bdabb69e3196c53db6"},"formatted_content_sha256":"abf4e533a3c9388da04e8ab3ba22c031aaaeb56e90338a4793f35ac968218c95","generator_receipt_sha256":null,"identity_record_sha256":"e80d081ddd92bc2a2b3317fc55c504dc53c4973ed5fdd77bb04345fc653a731e","prompt_token_ids_sha256":"54d2e4ab6ff2869f39d935df89b9c87e44df50e0fe4ecf67a377211040ceb3dd","ruler_category":null,"seed":null,"selection_rank":94,"selection_sha256":"3a95db7ccef488fa57c922c5fa561a269f79fc7c743e4d7a0ba136629ed14154","sequence_length":219,"sequence_token_ids_sha256":"7c11f008ccc4d7cd53ee721463f7a84f45653c8215e85fce7deab133839b9b0e","source_content_sha256":"6e23a95d9b3ebcb264ca654a288e4f63ec19e20f70c8c2cde95ec7ec6f5b65bb","target_token_ids_sha256":"5da4da15ffc85692e7f49aaf3e35fa21d41d697277e633358e0285f47b164471","token_span":{"cache_exposed_start":219,"cache_exposed_stop":219,"prefill_start":0,"prefill_stop":83,"scored_start":83,"scored_stop":219},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4475867faff977bd5c0d96f97b251d767fa8320ea73f71a38a236d7f33bd9db2","anchor_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"anchor_positions_sha256":"debc2f860d51bf30c9ccfe7850d3428800e80a005f976ab2943a8642c33fb808","canonical_id":"824","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,30,38,46,54,62,70,78,86,94,102,110,118,126],"fisher_boundary_sha256":"652add26cddc5b2b11115550fc5858cab1d3a2bf00d5a3de3dd88ed6f9ee6844","horizon":1,"input_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,127],"input_token_ids_sha256":"f5e28e53573eb8607c244c66e29a9057150911f5b5348a120bb3a756c686e14e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128],"target_token_ids_sha256":"f2aa9adbe82c7f97119740cd60b33102619d0355f0db44358231d88ca624f304"},"formatted_content_sha256":"37af9d0afbb100c1787445b2ccb87b0a15ea15c7b2fac0560e9fc296bf6fe512","generator_receipt_sha256":null,"identity_record_sha256":"3933b2a829550c994be7ec3c13432e7fcc7eaa7429de8d86f09f2fdd0d54c382","prompt_token_ids_sha256":"e63b26e32e2d08ba0e1878bb31f438a8b4f0754fac3b288bf8ff4bc11ebc726b","ruler_category":null,"seed":null,"selection_rank":95,"selection_sha256":"3ae7adc084daf636812058b087ea0414390f1d765f67235a978166a970420f8e","sequence_length":129,"sequence_token_ids_sha256":"325eb23014b521a2ed7488ed8be2e2022f039e91aa7939745a5ea7decd8357c3","source_content_sha256":"f3f9375971e245fb7d4b9ecb4368b1469f96e8826d3d290349822de215c9524f","target_token_ids_sha256":"98ee353303b490751f3b0bf4a9db8ee9f144fd843e84aee5b61e6b8b237bdca6","token_span":{"cache_exposed_start":129,"cache_exposed_stop":129,"prefill_start":0,"prefill_stop":96,"scored_start":96,"scored_stop":129},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"95c07a4ae2c995c71644eb87f395b0201ab61998c92c88acd0deec2bdf52eb7f","anchor_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,208],"anchor_positions_sha256":"e3c9aaa391385675ab3475417e405da92d531052d937e046ddc609185dc2c7d4","canonical_id":"755","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,50,63,76,89,102,115,128,141,154,167,180,193,206],"fisher_boundary_sha256":"630fdfb50def6a0e9810e25416f083ca388d391614da06d1ea421eee032302e5","horizon":1,"input_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,207],"input_token_ids_sha256":"211b86c0afe55079edbfe3277616d686ed0a24152163e4ad422e5c2071214472","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,52,65,78,91,104,117,130,143,156,169,182,195,208],"target_token_ids_sha256":"279608d1d01474978b20236572dd00bca3894556f73197ecfd6a080fdab2cea0"},"formatted_content_sha256":"d7d00e78af67fa7b8d4f1123bc771c6c1838a42ca7940f09be045fa8e0258d3f","generator_receipt_sha256":null,"identity_record_sha256":"163bb04292002d23bbb74050efcec0fe6cb3a616a2ed635b0e83b30ad3599dbc","prompt_token_ids_sha256":"649e3b31340e844cf9aca294f09bbfb6ea6f59b8fa4339754a8040e2939a14ab","ruler_category":null,"seed":null,"selection_rank":96,"selection_sha256":"3b77b38c7af3c05a391dfb6e4427c1d6c8e1408b314a52ec893944d4dba7a59d","sequence_length":209,"sequence_token_ids_sha256":"5eadaf66f4c2a5b1419e30b0e9445bdcf5e76923985d3015fb3da1ae81a6112f","source_content_sha256":"f132344866b31da506e676ecd7f787c4f0b9c30ce84dbcf234d1bec372a4a3bf","target_token_ids_sha256":"b0d0429e68642809416cf5dc819cdacda9612ad5ec72ede704a6165361970f31","token_span":{"cache_exposed_start":209,"cache_exposed_stop":209,"prefill_start":0,"prefill_stop":107,"scored_start":107,"scored_stop":209},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e588a1ee6a1182055ad85a99122b37dca3e494260dd21a56a3847938dd6650d8","anchor_positions":[15,31,47,63,79,95,111,128,144,160,176,192,208,224,240,257],"anchor_positions_sha256":"58b00b7c1a9bcbca3b1a752019d06b8217b207235aef4a33f8862dbb76f94f4d","canonical_id":"705","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[15,31,47,63,79,95,111,127,143,159,175,191,207,223,239,255],"fisher_boundary_sha256":"7ddf7690ae9f4e218fef449a93f7970eefb48a98743e2cdab0de23b0a0dec118","horizon":1,"input_positions":[16,32,48,64,80,96,112,128,144,160,176,192,208,224,240,256],"input_token_ids_sha256":"fce5f6a7ef57e2e29ac18c5c07d311d066b45dcd8cc0d5feede90040805886a4","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[17,33,49,65,81,97,113,129,145,161,177,193,209,225,241,257],"target_token_ids_sha256":"7e886f55d5a1f767632ff6ce8330179e78261592f007a8d785bcacf263c6b5b2"},"formatted_content_sha256":"79b5d499f16fb4783be74bd200b7cdbb4738662932b42d91a8cb390646bd7282","generator_receipt_sha256":null,"identity_record_sha256":"fabe611b8a7c9e97b13d178a8020c782eeb92caeab58708b2acab9e36c7c9739","prompt_token_ids_sha256":"0906a739ae0536ce684360464813702b15ce64ce3043e3fad9895d8c7a910e81","ruler_category":null,"seed":null,"selection_rank":97,"selection_sha256":"3b7d0fef73308384ee4b6d98b48fefed969da32df7231a61067116bb7b9b77e5","sequence_length":258,"sequence_token_ids_sha256":"c89f8002d735b00177a7842bc2ce8332555e0cb94e7fc1f651c7d9306085ae48","source_content_sha256":"249e819abcbbc94c533d4966c357933213dbe7b58282534d182cfdd719ab0b02","target_token_ids_sha256":"f81a0d07e6c10d8a36f2da0ff7490f0c0aadc3fe1411b385e79f96372bff8f0b","token_span":{"cache_exposed_start":258,"cache_exposed_stop":258,"prefill_start":0,"prefill_stop":231,"scored_start":231,"scored_stop":258},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"21ba94b504ab5ded4ffa601325cf7df425e7f81789df808b86d2e08ba2117dcb","anchor_positions":[9,19,29,40,50,60,71,81,91,102,112,122,133,143,153,164],"anchor_positions_sha256":"e70afdac88bd21ff5ffd057462f0868a1aaa408a52296b58ecd4900f7f13759c","canonical_id":"614","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,39,49,60,70,80,90,100,111,121,131,141,151,162],"fisher_boundary_sha256":"b4910baa9b90efc3b7ffe399e1e19b050f46dd444b4bfa0faaadd835b9a3173c","horizon":1,"input_positions":[10,20,30,40,50,61,71,81,91,101,112,122,132,142,152,163],"input_token_ids_sha256":"09f0051a5092a284f02162790ff187253c2a8be10b80c2ff7b64476a0924f7b8","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,41,51,62,72,82,92,102,113,123,133,143,153,164],"target_token_ids_sha256":"9c3d0858143570f2b17fd9c211bbddf1beedb863bd39887f1982c64284451830"},"formatted_content_sha256":"b0e77c735851949333aa5a07a80654370a25c6f4c29b682d0a7296c28db21615","generator_receipt_sha256":null,"identity_record_sha256":"9e60edb916d1cba4b8692f32df62fc1c57894b2aa815769ff9a17da063de91f8","prompt_token_ids_sha256":"591d12041b2684415e1d05a0ea815d006597cc26105e19c6a0769dc0ea26a8bc","ruler_category":null,"seed":null,"selection_rank":98,"selection_sha256":"3c33175d44e2cc571159f6914ce6a7c57a1949ccdaa58585114ece582b46e4a7","sequence_length":165,"sequence_token_ids_sha256":"964232f668fe81da4ef5e6945e2164c11b771718b74fc029dbc71ae25a213e61","source_content_sha256":"10dc2964ee279825eee885f98c2883dcf718ce9958fca29aabfd2057996b5a19","target_token_ids_sha256":"fc3e58f7036836c2a7f90f009d9d954dbc547bc6a412334c087a495ec6166136","token_span":{"cache_exposed_start":165,"cache_exposed_stop":165,"prefill_start":0,"prefill_stop":140,"scored_start":140,"scored_stop":165},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"41e7124eadf2c8d65f12d91df35d39b81b0af62f8f0339d10d48d23001bed1f7","anchor_positions":[6,13,20,27,34,41,48,55,62,69,76,83,90,97,104,112],"anchor_positions_sha256":"311e7702507c904e70997e0b842193b8c519ddd45b6861c3e52fdd12fcbe662f","canonical_id":"835","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,19,26,33,40,47,54,61,68,75,82,89,96,103,110],"fisher_boundary_sha256":"ef27b3360779ff829870ac1205d40a423bbd1857f9d29564cf1a4bf273508b37","horizon":1,"input_positions":[6,13,20,27,34,41,48,55,62,69,76,83,90,97,104,111],"input_token_ids_sha256":"622f2d82715555a3a862da50633d47a53da0f6d05afb23f037425ec2460ebf97","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112],"target_token_ids_sha256":"e4f44986dfe9f6afb783e79bc0788f586424b03e794cc20ab1129e7a75ca4b91"},"formatted_content_sha256":"16149587d9ee5b76bb9036394de7cc48051ba6deeced83ecf9552046bdf3d6fd","generator_receipt_sha256":null,"identity_record_sha256":"bcd976206ac1782d1fd75247a66068233d32764eef93e916f4303ac7216894e6","prompt_token_ids_sha256":"defa46cd3ab312e32247db6a9f87d5520f8ebbaa993a9af787a0d44ecc31a91b","ruler_category":null,"seed":null,"selection_rank":99,"selection_sha256":"3e545f645fe778609ea47f9d9d70db3a2d6fbf7c1b704f5c4b892a732ca4870c","sequence_length":113,"sequence_token_ids_sha256":"d87292b8689b41a713b9f23d42d1936617675fed9a389358046c37dee67270a9","source_content_sha256":"a2d12e2a125b7130ed036625cab03ca87319d5c2bc2f78d56e5b06e5d48967bb","target_token_ids_sha256":"d18a4e331e8bc3c665aeec8b4e8173eeaf216f691cec5deedcce9b89d1eabcf1","token_span":{"cache_exposed_start":113,"cache_exposed_stop":113,"prefill_start":0,"prefill_stop":85,"scored_start":85,"scored_stop":113},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4428369199e6df8c36558c477449442560d7dbefe1b1a34029eb0d3a540e8850","anchor_positions":[8,18,27,37,46,56,65,75,84,94,103,113,122,132,141,151],"anchor_positions_sha256":"99b598ab1b26514b0793ce6e9d9b8fd5fa21cfd91c15cf4ba944b02280eccc00","canonical_id":"785","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,27,36,45,55,64,74,83,92,102,111,120,130,139,149],"fisher_boundary_sha256":"224129c3bde0a7dc09aba43d89bfcf9df160c62a8d883b2421405de57da83148","horizon":1,"input_positions":[9,18,28,37,46,56,65,75,84,93,103,112,121,131,140,150],"input_token_ids_sha256":"d1cac9f469f540d35dd51eee4d84dc6489bdfbcb831d778ca3b910593caad5ba","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,29,38,47,57,66,76,85,94,104,113,122,132,141,151],"target_token_ids_sha256":"293d996a68091856a066009f8299d24def6c9ecd1505302fd1289596b4ce5929"},"formatted_content_sha256":"55cc81d66bd2973ef60996d3a13b658ddb8e10ba036a3a91808a8c4ae225b19c","generator_receipt_sha256":null,"identity_record_sha256":"70a4011b9bb314496b859d408f1205aaa618d08b8ca5504e4b0b7a8b3a794737","prompt_token_ids_sha256":"b5c378f6ed8ac42eb8be052f6d123a663ddfcd36c4bbcfc5decae012cf1c1926","ruler_category":null,"seed":null,"selection_rank":100,"selection_sha256":"3e7cbaec0d7be5d2de99697b23da0c61163c5bc4acecf283c4d9167e7b780eae","sequence_length":152,"sequence_token_ids_sha256":"ea3cdc2b1f803377a8cdf0588f013cebc2f019e5ad60fdf01294bf2d04d429ca","source_content_sha256":"06e8fb9b807385f5950f7335cb79a3b8105f621f1f5ac389155c9e737ff13d4a","target_token_ids_sha256":"a21eb422039b5c07e6a0e5f71858bbc8eaa8c315d06b39437977061a7cfdd253","token_span":{"cache_exposed_start":152,"cache_exposed_stop":152,"prefill_start":0,"prefill_stop":109,"scored_start":109,"scored_stop":152},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1cad8beb17ffee15f4de17ab4607544233d3683ae14b0cba6d063ccc9f11ff77","anchor_positions":[20,42,64,86,108,130,152,174,195,217,239,261,283,305,327,349],"anchor_positions_sha256":"3e58027edb04b4c749c71fe1c1753ad8300a599facd28b3d327dffaa160dd92e","canonical_id":"662","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[20,42,64,86,107,129,151,173,194,216,238,260,281,303,325,347],"fisher_boundary_sha256":"a261daa2db8ba7f05e7ff89074fdc6a0533b335ee67208179b8229ca3e70980e","horizon":1,"input_positions":[21,43,65,87,108,130,152,174,195,217,239,261,282,304,326,348],"input_token_ids_sha256":"d7e08d7e4f81bf68a6cf07dae075daaf7f79121f48d39ad9dba80fb057031e6d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[22,44,66,88,109,131,153,175,196,218,240,262,283,305,327,349],"target_token_ids_sha256":"8501644795ff2f929a4924fb2a3a1644ea894aa81320b17182c421175240b4b9"},"formatted_content_sha256":"91184e28eff6c3890010723ccfc61b5b4736a602d677efdffe59a220b3bac9b5","generator_receipt_sha256":null,"identity_record_sha256":"45b6269ce5d7d2ccefccb9d3aff10c4cc786d4482f036ebc1e0837d56e6c1923","prompt_token_ids_sha256":"ee07e3808c4e1142a8b3f904c8582078b899979e541c1ebe2297a3670112077c","ruler_category":null,"seed":null,"selection_rank":101,"selection_sha256":"41900ba952796bb2c024b96db8f08d2e63a757587e61fd96909dc934d50da156","sequence_length":350,"sequence_token_ids_sha256":"6890b69767a6c590160f23954c0338831fa0e3912f06dd6514c5b1552317c006","source_content_sha256":"3cf31926336b0e44d3630a63a486831d0b79632309d78e0b65f28e1c6f37353c","target_token_ids_sha256":"38e5b68f608a60d1fc61bff0194cd835196d4f8e8acb129c9e0744850d8641bc","token_span":{"cache_exposed_start":350,"cache_exposed_stop":350,"prefill_start":0,"prefill_stop":319,"scored_start":319,"scored_stop":350},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7d64f9e8b46b0ba25c21f37dee8e110df8862303d52cb3c2697353097a97db76","anchor_positions":[7,15,24,32,40,49,57,66,74,82,91,99,107,116,124,133],"anchor_positions_sha256":"6c1029515cd9fd840ee9baeeb3bbf16de22d914596dc66d6c0b00a017e82aef5","canonical_id":"732","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,32,40,48,56,65,73,81,89,98,106,114,122,131],"fisher_boundary_sha256":"2ea117a68a1f560fe1206940c2b26cdf6971aeeecd14dde580d013efe50b498e","horizon":1,"input_positions":[8,16,24,33,41,49,57,66,74,82,90,99,107,115,123,132],"input_token_ids_sha256":"46da8cb7bee3ade0de904115c405d0a4e2c1e215d9814066bfc5c50885b092d7","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,34,42,50,58,67,75,83,91,100,108,116,124,133],"target_token_ids_sha256":"3dd8aae9f6aff8d199075c33306d8bb5bd3b5f73666f2d236bd066a6d55fcbe4"},"formatted_content_sha256":"cf5f521e053dcaa96c55370f299514c9de27ccc98b8d846b0a51c4d2450341e3","generator_receipt_sha256":null,"identity_record_sha256":"1ea4e6a64df86bf2e3771dfc41f1716146cfac365a658d3864c030013a4cd84b","prompt_token_ids_sha256":"d705f66b234374474a57207de30809b5832cf7196bccb9dd73addbb879937264","ruler_category":null,"seed":null,"selection_rank":102,"selection_sha256":"41c7c18c717892ef0cbe99444d2f1d3d16142bd3128dc408df64987e27a8531f","sequence_length":134,"sequence_token_ids_sha256":"c98483cae5d714b2a8de61cd02257cc0ab084c30a0673971a4460e6d07f59d2c","source_content_sha256":"e5aeda9a803b146c76d557a6c8dc84720e938726b6f35982a5e61465a9a9d11c","target_token_ids_sha256":"15d06316804c670aad3848c3d01e3abb336656533c4feb76f93e90a0399aa760","token_span":{"cache_exposed_start":134,"cache_exposed_stop":134,"prefill_start":0,"prefill_stop":111,"scored_start":111,"scored_stop":134},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"a1bc1d7610d52bcb926390555c1cab77e3b88be528f3e256ea9e9a8106902d2a","anchor_positions":[7,16,24,33,41,50,58,67,75,84,92,101,109,118,126,135],"anchor_positions_sha256":"9058652c46869f6bf5cde3a730d23fba0f016461c119f6bbc8af3fb1c3031545","canonical_id":"679","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,24,32,40,49,57,66,74,82,91,99,107,116,124,133],"fisher_boundary_sha256":"810b5e7562357db03cc4e0193687f43bf7f4a08839cf3425a51f4e629ca7f5f6","horizon":1,"input_positions":[8,16,25,33,41,50,58,67,75,83,92,100,108,117,125,134],"input_token_ids_sha256":"03872be7640939a2770876b1cc144c05f59d3e06a6418658db2bceeecdbb5f11","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,26,34,42,51,59,68,76,84,93,101,109,118,126,135],"target_token_ids_sha256":"5c71155464943a602fc92f92c7507287819905cfa27f665ae40e126b898cdc40"},"formatted_content_sha256":"ffbf833b467a2b56b26bc50bcb320d893f41f60eed6341ca4c61813965020c7c","generator_receipt_sha256":null,"identity_record_sha256":"13c42ddbb25f335f5eaa4ef5df7c2cecd1ea19af1f8972ade37312c4fce78802","prompt_token_ids_sha256":"df73b549576d1784c30bb45b9c873ebb80267ac6c56afb4335eadca3c7c56849","ruler_category":null,"seed":null,"selection_rank":103,"selection_sha256":"41da3df61a258a1897eeac98083550d328825fdeec35836584dc0b7a76b060b4","sequence_length":136,"sequence_token_ids_sha256":"fb871500f94a5409b4acab60ab3e2aa6955924b9ceb87214232c4099d302dab0","source_content_sha256":"39571a4165c49d0610e00d19d3f5b5e1879232f2d0429100e2aaf915eaca9ab9","target_token_ids_sha256":"d6eaabcd23781ac4a910aadc33823fd08ab066d01331a124767f6e84f181ffba","token_span":{"cache_exposed_start":136,"cache_exposed_stop":136,"prefill_start":0,"prefill_stop":118,"scored_start":118,"scored_stop":136},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b8947897cdec3fc049ba3461c75adcd685b2264e6810778decb68c1db034f7b8","anchor_positions":[17,35,53,72,90,108,127,145,163,182,200,218,237,255,273,292],"anchor_positions_sha256":"c72256a78f816fa7bf8e540cc3aaf3fc1795369a3859efe48a9c4a8cce0d2489","canonical_id":"895","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,35,53,71,89,108,126,144,162,180,199,217,235,253,271,290],"fisher_boundary_sha256":"d12370caffbc875b01d6a59c8cf4c99d8fb4c93c519041b5500142fdac3a8340","horizon":1,"input_positions":[18,36,54,72,90,109,127,145,163,181,200,218,236,254,272,291],"input_token_ids_sha256":"a316e2b2888cea8097ab8420cf22051694e1d976c3b5b32f83f5207f4007d89f","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,37,55,73,91,110,128,146,164,182,201,219,237,255,273,292],"target_token_ids_sha256":"ea285c83c7e7736e7a436adedf9c66203844bbe2521ff7ad148dd51b419e98f5"},"formatted_content_sha256":"9b9360fb9e789a22496cf1d9c91e89cf7895942447ca7e16f15acbf2fe6515b8","generator_receipt_sha256":null,"identity_record_sha256":"080db8e8894bfd652b320dbf19c1d4e53c8b6ed2626bcb8f8d3cdf87268005f8","prompt_token_ids_sha256":"3ed029adce95edd3406c2359cabf621905d3ecaceb70e519fb1cc89a66899110","ruler_category":null,"seed":null,"selection_rank":104,"selection_sha256":"43113137520f4bdb68e26d41a096fa093a2f995386d5473618f0bbe4b2a0f73e","sequence_length":293,"sequence_token_ids_sha256":"24d56709934330dbe30c61a49434a07def76fdd2b31e47f84f71b050fc7aff39","source_content_sha256":"d1edefa8e37752a9da687c2a2e51086bdfe6401714de0bd593a892ecd4fe7bcb","target_token_ids_sha256":"e894e28b0e66238fd8988d8b0b978628f3354d87898164223cb7c7dc2ce5a596","token_span":{"cache_exposed_start":293,"cache_exposed_stop":293,"prefill_start":0,"prefill_stop":161,"scored_start":161,"scored_stop":293},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7d9d79586d3fd7a1376390bd1f085ee9ec35cb401fa471d99ddf45417339ce49","anchor_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,160],"anchor_positions_sha256":"cea919870992245499fb8d8a858d848e1e76f02a4e2230ada935442533c03198","canonical_id":"727","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,18,28,38,48,58,68,78,88,98,108,118,128,138,148,158],"fisher_boundary_sha256":"768624e8d0cb29118f002fce04ec89945eebe14e7587f370283fa04b2d90cf95","horizon":1,"input_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,159],"input_token_ids_sha256":"93e60eaabf27cffd7d1fee3301e42b3f62e183c9e40317db494ede3450b1ea7c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,160],"target_token_ids_sha256":"507cebbcf35bc57eb21aeff74f69ab933d22c31de894e77b573193730da7662a"},"formatted_content_sha256":"934eb88ee283eea8ac09e46d9333c08ae4e41a6910aa75e0638bc73403184864","generator_receipt_sha256":null,"identity_record_sha256":"aac6297b0c851ed51271119b880a93cb272776311e2b9e97f498c76785255100","prompt_token_ids_sha256":"ad50cbf08781e9a1429766f1a21e3ed4a76d9fc30e43b5b8d968dcda018da4d4","ruler_category":null,"seed":null,"selection_rank":105,"selection_sha256":"4338156536b7925e6044f19142d418110ae27757c47306d15815155b2d682917","sequence_length":161,"sequence_token_ids_sha256":"422567bd8e19fbdf69ab47274fe1bda49c8ac4eae311d7232ee13e0888b0fa23","source_content_sha256":"400a362fb6664f9084e588aa605b6e7ad1505849ab7c769574e0996829f6827f","target_token_ids_sha256":"c8e9d40bce2584a16cdb8ee232a9e1f86dd9c58c6fee7999125c37073c1f5bb5","token_span":{"cache_exposed_start":161,"cache_exposed_stop":161,"prefill_start":0,"prefill_stop":134,"scored_start":134,"scored_stop":161},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b50fd03bddce1923b315763ae4ef228d2b6c9cce2b80954554946c98c92a3760","anchor_positions":[8,17,27,36,45,55,64,74,83,92,102,111,120,130,139,149],"anchor_positions_sha256":"27ff593f69b04d82cc98f1ee839db50560b1d26eb9ecd645cc99a4e74dade2d0","canonical_id":"629","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,26,36,45,54,63,73,82,91,100,110,119,128,137,147],"fisher_boundary_sha256":"d8f311b4a33da5b28927ed11e2b7efbb2a1812456e5cd54c884287cfc1ce634d","horizon":1,"input_positions":[9,18,27,37,46,55,64,74,83,92,101,111,120,129,138,148],"input_token_ids_sha256":"f56a22b67052689f706d00c90e6fc5541583c78a589b0b3a26c6b93eda4162d8","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,28,38,47,56,65,75,84,93,102,112,121,130,139,149],"target_token_ids_sha256":"93c8a6c8bdc5c1ed56f7db15ae4424e87803630ba80e3845d421f97169235f34"},"formatted_content_sha256":"9c29f28fcb8fb63c6faa67311c4d67889b21525c18b625ca3fbd69bb6611342e","generator_receipt_sha256":null,"identity_record_sha256":"dd6a317d0ffc82ff3c3b71c470d6f7f4ee8c3276d55ea892d77526d03f80719a","prompt_token_ids_sha256":"ae80eeafd124153f30025382491abc9552e2fb6ac10aa0e25acea39bb6cab5a7","ruler_category":null,"seed":null,"selection_rank":106,"selection_sha256":"4357903439ea78adb2c1be88172806c1a5e5d57f1bfa26229be4d3d56bf15b5f","sequence_length":150,"sequence_token_ids_sha256":"58d577039503bfee517e7ec9f595d3378f0520d7b10780457232e42ac1f8b9e0","source_content_sha256":"d75386bde56ac72aa78e5b13019f0613da914d25303ec7ae96f618caf713bf8d","target_token_ids_sha256":"25dde1ab97315ca73c3e47f45297c03f015715e1dbf87dc40a90d1fff93faa1d","token_span":{"cache_exposed_start":150,"cache_exposed_stop":150,"prefill_start":0,"prefill_stop":109,"scored_start":109,"scored_stop":150},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"743e09d0b63e0d1d69cb1f4d92a19f1b612ce8d5ed318965c7cbc5ef10da4ad0","anchor_positions":[5,11,17,23,29,35,41,48,54,60,66,72,78,84,90,97],"anchor_positions_sha256":"3a7e2557ca5cb4f3e315ceb26b0546b57f9a7cdbff08065541d63492d4059513","canonical_id":"789","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,11,17,23,29,35,41,47,53,59,65,71,77,83,89,95],"fisher_boundary_sha256":"03a7963bc281465bc3ed237e9329c2d675fc5a2743cb29e2366d18c053e4354e","horizon":1,"input_positions":[6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96],"input_token_ids_sha256":"51ddc5b200c5449da711c0cb31fd958274770a19f7df1e0a919bfde5c62fa46c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,13,19,25,31,37,43,49,55,61,67,73,79,85,91,97],"target_token_ids_sha256":"5b9e09d236094c93cd27152a8362d6fc0907405ed836f4c806eb6a49ce662637"},"formatted_content_sha256":"95cda84841f0ed6748d2aa3eab9b3f2a1d602bef0648c2354cb235fbc49eca99","generator_receipt_sha256":null,"identity_record_sha256":"1b904dd9e776f5697d87583519b28c5869436344070e3c267f5d2f9cbe24f679","prompt_token_ids_sha256":"59d140fe1bf3dedbe286b918f769e76d9062fc0a8677694f8a29a4264c3dcfc6","ruler_category":null,"seed":null,"selection_rank":107,"selection_sha256":"435d524443c963ac0697492db5d755ed96c21825c6c12686e14ad1216a9d98d3","sequence_length":98,"sequence_token_ids_sha256":"f843f9ee0ba37389203b844c5d69910ebcaaa1266a694030f5cd56fa427be6b8","source_content_sha256":"528cefc12ccc28d572fc35698c3e55d5fa17974dcd83ab45fe269d5d96166d59","target_token_ids_sha256":"e052b469e73f16555fd8aae9d7e12029ab50d45bb5400880bce25ee85f7c0582","token_span":{"cache_exposed_start":98,"cache_exposed_stop":98,"prefill_start":0,"prefill_stop":75,"scored_start":75,"scored_stop":98},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"56264414b48afbfd02d06a09b83695f1be6a109c5e5917043688018a40e3a3c2","anchor_positions":[23,47,71,95,119,143,167,191,215,239,263,287,311,335,359,384],"anchor_positions_sha256":"5ecef0009b1f2b5ca1c05ae59fc5945152ea0233cf7a7da1d75b9641a0da44d4","canonical_id":"819","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[22,46,70,94,118,142,166,190,214,238,262,286,310,334,358,382],"fisher_boundary_sha256":"83c8e0577de0e56d18d8697c6d54b87b15a0a615623c44993bd8b9c2163d30c8","horizon":1,"input_positions":[23,47,71,95,119,143,167,191,215,239,263,287,311,335,359,383],"input_token_ids_sha256":"e56c492befed4cd096e4eb5258b978187c18cba26036c7ea511ea7b3828aea8a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[24,48,72,96,120,144,168,192,216,240,264,288,312,336,360,384],"target_token_ids_sha256":"b5f4bffe29cf172dbb04ddcf7fd6b1f217b698f7016f31ff8f6e48c727e2c9be"},"formatted_content_sha256":"616a3535fe1ec791a293ac6da5c856a7712282d6db633cd638270d4013cec624","generator_receipt_sha256":null,"identity_record_sha256":"c33ba3115c44c968c4ff9d9c33e133ea4f975549b4e7717c6581b9f976c187f0","prompt_token_ids_sha256":"de127a7cae4376da7d66c0f3f1eae83db093b0eee64a2a21dde2ac28c80f0322","ruler_category":null,"seed":null,"selection_rank":108,"selection_sha256":"439c6f2d57f5500b6cc21860ee22a1ebce8b83eeeb2bb034371733828ffa361e","sequence_length":385,"sequence_token_ids_sha256":"5ff88ef079fd63302a6d16c9b8d2bb597b0886ed6975c9ee0d7005fd42a6852e","source_content_sha256":"1b57006191af74fc5cabf13ded0936d54e33d0eb2d92e12170ffbdf83e11e029","target_token_ids_sha256":"f0b593d02c8c7048d24a55e2360b1e42ecae542816dcf5a6d3a7a5f0d3baa103","token_span":{"cache_exposed_start":385,"cache_exposed_stop":385,"prefill_start":0,"prefill_stop":268,"scored_start":268,"scored_stop":385},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"f83dda5f76b6c77a2f8f34d32d59899a5438aa93e284ca72500f57061c684a29","anchor_positions":[8,18,27,37,46,56,65,75,84,94,103,113,122,132,141,151],"anchor_positions_sha256":"99b598ab1b26514b0793ce6e9d9b8fd5fa21cfd91c15cf4ba944b02280eccc00","canonical_id":"886","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,27,36,45,55,64,74,83,92,102,111,120,130,139,149],"fisher_boundary_sha256":"18cbe59ca514ee6ee4de7a9ff59d5f2173303011464fcc35cb15114c03969d2a","horizon":1,"input_positions":[9,18,28,37,46,56,65,75,84,93,103,112,121,131,140,150],"input_token_ids_sha256":"7e8df804a72a77f8e56f283f6f3d3e0ccf7bdd663763c7ba3cf49b6cf2c8c479","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,29,38,47,57,66,76,85,94,104,113,122,132,141,151],"target_token_ids_sha256":"fa410d4aeb53ce61e53c7bf6ae6bf9008a5152a79caa4db0b21713c71de06b6d"},"formatted_content_sha256":"1be2ddc47aaf0ef4c6ce6979f16a7daed6383578cbb0e12a913e8c76157fe911","generator_receipt_sha256":null,"identity_record_sha256":"16353e62969e446cb1fd7fc9ca7a99ffad2f73e31b73381c626a08c7293cb32b","prompt_token_ids_sha256":"fdf8cba22d99356b2668bf60f4e3c5f33e9981ae71ede27ef250e9945224d770","ruler_category":null,"seed":null,"selection_rank":109,"selection_sha256":"43d5d553cb2e28075980d0bd0920a10d225c7497df856fe96c4367fbe7e6bace","sequence_length":152,"sequence_token_ids_sha256":"8853c262834e9d1226bdfde776d4ef0e7a0e7c8778a464180efae64bfb4f3c20","source_content_sha256":"434c4f97415e9ae869612f384ad8eb1d4459b91e71891d13598de2c0ecf58ce9","target_token_ids_sha256":"c73711379fdd03655ee8104d211db2039b2bd1b1d8757202f91122e2e8800f82","token_span":{"cache_exposed_start":152,"cache_exposed_stop":152,"prefill_start":0,"prefill_stop":120,"scored_start":120,"scored_stop":152},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9bfe15074048c1b3cc4e9a64f9f82a7762f5e3fc7b6286e9505bebdfc8ce78e6","anchor_positions":[17,36,54,73,91,110,128,147,165,184,202,221,239,258,276,295],"anchor_positions_sha256":"54ac14bb1cffe66a9bd2a063df3998236174c0775660ded8a46c6b4f3b34b4be","canonical_id":"669","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,35,54,72,90,109,127,146,164,182,201,219,237,256,274,293],"fisher_boundary_sha256":"e64ce615c2670462c794e8f67c34ff9ba26d820a1d46601098cb0321f7249f65","horizon":1,"input_positions":[18,36,55,73,91,110,128,147,165,183,202,220,238,257,275,294],"input_token_ids_sha256":"2147411fed9b01bc77a54da5a575818dd51a16507fe185b59c616201c9dbe92e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,37,56,74,92,111,129,148,166,184,203,221,239,258,276,295],"target_token_ids_sha256":"d9cc42057cf2f44e2193c2fff4ec48c3d54db14f1369ce5096749eb0a37be2eb"},"formatted_content_sha256":"fe004ffe4d515aa06edc2d7a6ff8f2900da04c2c06fe055101a34ba872ba040d","generator_receipt_sha256":null,"identity_record_sha256":"5a4fdfb2bb3bdb18316e26b3524adf61c217b8661a9502ef4888ec80cf44a517","prompt_token_ids_sha256":"545545ebb1988d32125cc7c0b6360272908d9ddf46dbea3b96b9923e8f61a5da","ruler_category":null,"seed":null,"selection_rank":110,"selection_sha256":"454527a603157418a471cc4d85900b7bc64b30edc7df17206aecd43f4b706a81","sequence_length":296,"sequence_token_ids_sha256":"0c56d0026a937592d71b7773e5acad5e8774badceb5806f30884aac130918445","source_content_sha256":"76e713c3e769794fd21cc2b169d3fd25155269e05a87202fa2ac7130dd7d9534","target_token_ids_sha256":"e5afdd086f0ae8795afe5c271dd9f9a4fae2f74fda8cd0e04e3f697b2d39bb25","token_span":{"cache_exposed_start":296,"cache_exposed_stop":296,"prefill_start":0,"prefill_stop":113,"scored_start":113,"scored_stop":296},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"bba06f38d7ef46d2274fcc3c62d6b99545aea08fc9a970bb2cf4112497f86d55","anchor_positions":[18,37,56,75,94,113,132,151,170,189,208,227,246,265,284,304],"anchor_positions_sha256":"94f33a91bb15b5b174db14bd5a18a220943ce79ec8eba0ee2655b965e84d64e0","canonical_id":"846","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,36,55,74,93,112,131,150,169,188,207,226,245,264,283,302],"fisher_boundary_sha256":"e56eaa1d53ef9cac5e4ea05eac5fae5d2ff32456ec4fd16aa432bb0ba682df16","horizon":1,"input_positions":[18,37,56,75,94,113,132,151,170,189,208,227,246,265,284,303],"input_token_ids_sha256":"b57a987371cd39d03b724b08c3b6837260a5e8ad281a18e85c9901110811e452","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,38,57,76,95,114,133,152,171,190,209,228,247,266,285,304],"target_token_ids_sha256":"adb018d6d0b2d7716ff0afe479b6ee4cb1e352197e41003e0e3962d666cf8a0f"},"formatted_content_sha256":"d783edb3ae5e40760a243586ed62d48e4d1fab758d36793b9e509e3e34c74a3d","generator_receipt_sha256":null,"identity_record_sha256":"3c715e7b82c94eee9feb5fcc68ac303cd191e6ec2068a3d300d3ca8e4dd6e5de","prompt_token_ids_sha256":"604c21c3a8e0575c89ca1b7911771a2a7ef7b4a915f2b165be5a3a2c56b206e1","ruler_category":null,"seed":null,"selection_rank":111,"selection_sha256":"45dda3a343ef8f05e6b253db4cedda79669dd8cedb0368cd7ba98b9b569952cd","sequence_length":305,"sequence_token_ids_sha256":"b0ea9f94ab26fc47af030e71000bddd166cb8e1001febd6121f29acbdf792437","source_content_sha256":"e03f7274bd5dfa833b5d69f144a32a2866a4d5e2df73779e455ab0a797e8b9f6","target_token_ids_sha256":"7c55e00d0effcd9ceef682e102dd49697117fe7a162f11e2bcd35a18523b96dc","token_span":{"cache_exposed_start":305,"cache_exposed_stop":305,"prefill_start":0,"prefill_stop":182,"scored_start":182,"scored_stop":305},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2f002330a14ccd49bbdf1186b94d7f4a660f3c79ca0935f6733429506fbac4dc","anchor_positions":[9,20,31,42,53,63,74,85,96,107,117,128,139,150,161,172],"anchor_positions_sha256":"9bb32a96de9c1958e4d75c68046a769ab85020ffa742f7286cba9f5cfe66172b","canonical_id":"809","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,31,41,52,63,73,84,95,105,116,127,137,148,159,170],"fisher_boundary_sha256":"b791fc560913737b45746c61831af949c9b84432325347422a7a866d99f55180","horizon":1,"input_positions":[10,21,32,42,53,64,74,85,96,106,117,128,138,149,160,171],"input_token_ids_sha256":"35bdf9fe7875294a945c0b7fa771287b6e892aade75b41750c2e0144945a8d4f","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,33,43,54,65,75,86,97,107,118,129,139,150,161,172],"target_token_ids_sha256":"901d22acbaeea26debc37b388abaf43ac50248b0422016d07fd3a292798c4cd0"},"formatted_content_sha256":"7d79ec67027ef058aa16aa49df49541f7821cc1e06a8283c93d05218bee5c30d","generator_receipt_sha256":null,"identity_record_sha256":"00931944a29fecfc4c8c6b60832e5269dd3d431f1fc0fec8f21cfc3d5b2e56c5","prompt_token_ids_sha256":"d2c4a745f10a0a0263c72f290f8ef232ab4aafe62f10358bad7a317ba0990e25","ruler_category":null,"seed":null,"selection_rank":112,"selection_sha256":"4639d4a26db93822b4b2e18576a8169bd874398e85f3a592904ffc3f972fbb17","sequence_length":173,"sequence_token_ids_sha256":"97308fe18d6a1e87ca3681748a23734c46402fa3f1992ea007b186d80d049540","source_content_sha256":"197f56fd23a55154213be6170da63f959f105eb46b7cd9cb707820e74a06529e","target_token_ids_sha256":"c3f58b0c22170cfdfc5245c921af88598385a2623da14949ba7503f68723c427","token_span":{"cache_exposed_start":173,"cache_exposed_stop":173,"prefill_start":0,"prefill_stop":128,"scored_start":128,"scored_stop":173},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"696c3eff19f8cd3ba60be744325a381bcade3a5bde2058ce2b7057bd86a602e7","anchor_positions":[8,17,26,35,44,54,63,72,81,90,100,109,118,127,136,146],"anchor_positions_sha256":"95e4d2457eadd7ff2ebe49af36c11afbb6034a7c8d08013efa9de5325f17bb5d","canonical_id":"797","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,26,35,44,53,62,71,80,89,98,107,116,125,134,144],"fisher_boundary_sha256":"0f290f46b842b2d9555c6342ee65ad579f7f6195e2840a98edaf138b7f77e879","horizon":1,"input_positions":[9,18,27,36,45,54,63,72,81,90,99,108,117,126,135,145],"input_token_ids_sha256":"45d165859afaa71c112d2d4db0282de95319684d4d160e6d34be3255f19e378c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,28,37,46,55,64,73,82,91,100,109,118,127,136,146],"target_token_ids_sha256":"dd11e1c91e7cf97f459b2696ad0d206a7e54b2e224552b8590c9c6431e8584a1"},"formatted_content_sha256":"8e313e746cb70b0f39a63b254855866da5978faa757f114d540a3e7ae9d63cbf","generator_receipt_sha256":null,"identity_record_sha256":"6952ce6716bd8931f4ca2c58ccaf72f466062aa0785261289d59a5104678a7b5","prompt_token_ids_sha256":"04b1d2f628e08d770e64feea8b70cbb7a64b347ed50b692622c6c3e0c7c8029d","ruler_category":null,"seed":null,"selection_rank":113,"selection_sha256":"46503a4924e57013ffda3af84d33a2db536b465afca8ee866ff2a2e87961e472","sequence_length":147,"sequence_token_ids_sha256":"dbf6194f0a08074850f461642c989526f8fe9049d05306dfd6a2da9a9987f8f8","source_content_sha256":"aa80156f16bf70bd62cebda84762b9a1a8ae437dc8871c0acb7d61e9d83b1b8a","target_token_ids_sha256":"d91b2a60c8ed8407d0c17107efc9a4cc4e719d8a094196cb04d089c6d8aea011","token_span":{"cache_exposed_start":147,"cache_exposed_stop":147,"prefill_start":0,"prefill_stop":91,"scored_start":91,"scored_stop":147},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b907107ca4056a13d77b90b11cffa6104f002742e48e4d73d84701f35cda3ce3","anchor_positions":[7,16,25,34,43,52,61,70,78,87,96,105,114,123,132,141],"anchor_positions_sha256":"4a3ecd9aedde633817cc01e0837692eec5dd57faddeca3f7bcf5e18333ad7c64","canonical_id":"803","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,25,34,42,51,60,69,77,86,95,104,112,121,130,139],"fisher_boundary_sha256":"8c5dd3de4221407fb31879a8831c58de078b7bd96edff29efabeb0d4f941ca5d","horizon":1,"input_positions":[8,17,26,35,43,52,61,70,78,87,96,105,113,122,131,140],"input_token_ids_sha256":"612f42435d82b79fa57bcb8dc07f12f964553d764d942f85ac5c989daa0ac60d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,27,36,44,53,62,71,79,88,97,106,114,123,132,141],"target_token_ids_sha256":"20ecbcd0e6f4d0f38e6e34055d4306d4556676db803bddcbf4efb50dde35e518"},"formatted_content_sha256":"b80b90161540d8a9b1c0ba6896342db1e035ccefbe96663db9df9b74d82a3eae","generator_receipt_sha256":null,"identity_record_sha256":"cdbc48a4424d2515ce6141290d69310f055340010d03518c808f7666de9eecb6","prompt_token_ids_sha256":"9a94491891f327307e6c2d541dae50c594d32f7e7113f10a4ad421cd16f7efee","ruler_category":null,"seed":null,"selection_rank":114,"selection_sha256":"47a81fc27a92568642839ad9294c190dcd0b21f98ad4eca4472567609f81bac4","sequence_length":142,"sequence_token_ids_sha256":"4e6d4b6ce7a5a64ad6b5d03e8b6c0786f2a1206a63cd36a35f88e41282093685","source_content_sha256":"fcc558736fc46d07648e59309cc19de0322f28060b2265985cd5cb46f9ccac97","target_token_ids_sha256":"16fcd9ca012cf831e086d70b1c2f66e51e595f470fb2610a600125f50807abae","token_span":{"cache_exposed_start":142,"cache_exposed_stop":142,"prefill_start":0,"prefill_stop":82,"scored_start":82,"scored_stop":142},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"61f0fff09b788c1dde584ff85418eee082c77947d695e4df4e831eb33cf048eb","anchor_positions":[10,22,33,45,57,68,80,92,103,115,126,138,150,161,173,185],"anchor_positions_sha256":"fde5a3b64a487819cb5be9d572133781e4744390c11836e2f115ba2608fe7216","canonical_id":"644","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[10,22,33,45,56,68,79,91,102,114,125,137,148,160,171,183],"fisher_boundary_sha256":"3c448eaba6e976f4d8540a8018f1b6ee02f1a078e5ab97372fe15de20e37c79d","horizon":1,"input_positions":[11,23,34,46,57,69,80,92,103,115,126,138,149,161,172,184],"input_token_ids_sha256":"61a5b967b5a2460665ba8ac148a2ed8fda9b1a0e95c8d66a509db1c89e384bb5","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[12,24,35,47,58,70,81,93,104,116,127,139,150,162,173,185],"target_token_ids_sha256":"e6285297df83884b53ce8b429f39bfbc76734d91c897b1ef6b45e02c90bdc53e"},"formatted_content_sha256":"cc789f8b54535a69935595ebbc2d15ace8da8a517f3e3a766f4c9801ab975e73","generator_receipt_sha256":null,"identity_record_sha256":"add76763c425e50ca33682da8956ff29cc4547518dd84c9fea5b339de6f6125b","prompt_token_ids_sha256":"a672313ea736e1353ebd2a843db4d5ae378ddbec92e54cb81d21376a13087e88","ruler_category":null,"seed":null,"selection_rank":115,"selection_sha256":"49614c4b89fd4e064236884bd54d2b6744d30da34cae36de98a3f8d6dc005295","sequence_length":186,"sequence_token_ids_sha256":"2a629a0ceb8ed135d2d3814402c87b6b4d4925c16ec65dcf756469c7f70d28dc","source_content_sha256":"6ca1519c2cb7b633366e44afc37a261e40427c751412998cceb863343020cf4e","target_token_ids_sha256":"147bf03e5bc17953126d30a0640ae17e4b57cbaf4d65322290bee14c6f8c12a8","token_span":{"cache_exposed_start":186,"cache_exposed_stop":186,"prefill_start":0,"prefill_stop":160,"scored_start":160,"scored_stop":186},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0a09087a0815d40c6da7249de80b96629effaed44659b49eb51e112de7acf916","anchor_positions":[5,12,19,25,32,39,45,52,59,65,72,79,85,92,99,106],"anchor_positions_sha256":"1ec66db99ded67a6b74f7dd5259127f2ba61fd9a33265762c1ea04710558d7bc","canonical_id":"647","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,18,25,31,38,44,51,58,64,71,77,84,90,97,104],"fisher_boundary_sha256":"bbab335f18e2c0ccb810bd85c071826d79d9b490e590cef0c6d681341c0cc08e","horizon":1,"input_positions":[6,13,19,26,32,39,45,52,59,65,72,78,85,91,98,105],"input_token_ids_sha256":"46b16340d489ab9d5e99f73c42d4a7005f7436243eef5fc34fa92767754e2622","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,20,27,33,40,46,53,60,66,73,79,86,92,99,106],"target_token_ids_sha256":"761af83bc90049bcb9676e87032fb087e505ce54ccdace5bd3157cfe531b0e6f"},"formatted_content_sha256":"892bdd77965161d3f33ecae4371aaab245852c1a733575ea82a03f4c5162b6e9","generator_receipt_sha256":null,"identity_record_sha256":"096bf170f6323441d41cfdcd82e991e5330ce43bbca166fdaff8e2867cb3901f","prompt_token_ids_sha256":"9d1498f0e195703f3d0a9bab6ab2b95760b77762a386814e263445e614eebe8e","ruler_category":null,"seed":null,"selection_rank":116,"selection_sha256":"49bf4e9209616a951134062596b8e1762dbe5ae3d6768f1c37fc3ca1a831c8d8","sequence_length":107,"sequence_token_ids_sha256":"e9307774f416105649348d7b962aed8818afdd9670718378a8011aab53794a9f","source_content_sha256":"a4c7ff187da9b88bed24a5ce4e33b1d245e061cc2f7fb731157aeb3dfaf250ab","target_token_ids_sha256":"d3f2b417944e705153e271af9eeac8e255eb09fdbc9a8241cedc49106c05cb62","token_span":{"cache_exposed_start":107,"cache_exposed_stop":107,"prefill_start":0,"prefill_stop":82,"scored_start":82,"scored_stop":107},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4294762e8478186f4347187eec4203bd212ed4716abafee16d6c831b211a92d0","anchor_positions":[6,14,22,30,38,45,53,61,69,77,84,92,100,108,116,124],"anchor_positions_sha256":"fe7a10db15891cb54d23372b10af49697c0401f54d8f6b37c01417545e9ba930","canonical_id":"605","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,29,37,45,52,60,68,75,83,91,98,106,114,122],"fisher_boundary_sha256":"c098a5d59dd920acf5e6ecb372e0ff5df7d0b3ee1501bb011f699481cf66e206","horizon":1,"input_positions":[7,15,23,30,38,46,53,61,69,76,84,92,99,107,115,123],"input_token_ids_sha256":"faf2e5d622e9951baec003f38847af5d1319f255169b8c041a433d5b46952e3f","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,31,39,47,54,62,70,77,85,93,100,108,116,124],"target_token_ids_sha256":"26f6b36890856124d1f7f7a676b24ea8aeea3382d92cd1489d141c22982a041b"},"formatted_content_sha256":"80aa72fd69bd10f4c2e5403802378e6fb00d15a1e298252ded65c6ce760ab095","generator_receipt_sha256":null,"identity_record_sha256":"ee34d2c3691827aa084c190eddddebe9d1345888c7f48e25c87d60d765f6dfcc","prompt_token_ids_sha256":"9fd32241d4f2b36ee9f01d68e556fe7d2b96791f996bcb4bddb46b56225c1199","ruler_category":null,"seed":null,"selection_rank":117,"selection_sha256":"4c310c24967e920116a6713c985dc574fc27f51aa0e5f96102923b5848a2fea8","sequence_length":125,"sequence_token_ids_sha256":"d5c71f5ac8cca279a5091bc56ba019e4c7f2a97540c957843ce3658f1edad1ac","source_content_sha256":"dcf527a4f6986a9167a459248d6567bd87701d5fa34856a31f9be5ee02c91168","target_token_ids_sha256":"741ab7638f051c1d782ffdca7cafb68a673af7ec2773fdff58abe66e46d7eef1","token_span":{"cache_exposed_start":125,"cache_exposed_stop":125,"prefill_start":0,"prefill_stop":68,"scored_start":68,"scored_stop":125},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"05c8e3cbead3af43c5c1e6e8d87467ab8c03ec4648a4298153b82d936448e5ad","anchor_positions":[19,39,60,80,101,121,142,162,182,203,223,244,264,285,305,326],"anchor_positions_sha256":"f3e5839dc1785a2c00f87ad2ea0e816ae94270b3238251e62817f25d09a21cbc","canonical_id":"630","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[19,39,59,80,100,120,141,161,181,202,222,242,263,283,303,324],"fisher_boundary_sha256":"de76b71c7d9a5873e8f18d38d9bfe274b70bf5ccb7ad95749ffff39ac8269bce","horizon":1,"input_positions":[20,40,60,81,101,121,142,162,182,203,223,243,264,284,304,325],"input_token_ids_sha256":"b71f4ccff2a296d03b9b8fada60f1b7538853521101cb758ed05e1b6e41628d2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[21,41,61,82,102,122,143,163,183,204,224,244,265,285,305,326],"target_token_ids_sha256":"939dd654b199e687770731bddc84e26956b5c40203089d198687d9c12778ed94"},"formatted_content_sha256":"b39458254cec5c52b4b1ffb7f1dc476b0e99bab62c4d66d0998f147e9cd767cd","generator_receipt_sha256":null,"identity_record_sha256":"d20e005d97d9c08185e9a25884618ab3451741d9e00a6f7cf4842daa416fb1a1","prompt_token_ids_sha256":"92effc02a358ea378746b418ac301dfe5bacb980d7a8a9bfee9363f21a656396","ruler_category":null,"seed":null,"selection_rank":118,"selection_sha256":"4d4a3d035c7868292b96361ee3387aa0653c72bf82ff2ffbcdeee1c544f9b310","sequence_length":327,"sequence_token_ids_sha256":"11215c622904358795cd7771cb56cd3e68586e31dea00e30330fdece99fcd790","source_content_sha256":"d026e046adcf26a5522065e389d1e74a8d7ef963fd53e4f8db8952c3c3058226","target_token_ids_sha256":"59f027a42d76a9550f2a9d95174c87b9fd402df01c12dd6433c83e8bbb1834ec","token_span":{"cache_exposed_start":327,"cache_exposed_stop":327,"prefill_start":0,"prefill_stop":236,"scored_start":236,"scored_stop":327},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"865d6b59e048496b3bf7c4b287421ac71174d11943f8a146bf5f215f6e662bb5","anchor_positions":[7,16,24,33,41,50,58,67,76,84,93,101,110,118,127,136],"anchor_positions_sha256":"b394d1b64998869298fd2b292d4ab84e6faf0b5a40eeb811c668f2563bb4df64","canonical_id":"868","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,24,32,41,49,58,66,74,83,91,100,108,117,125,134],"fisher_boundary_sha256":"7f4687722337e0596bbc45f63cbf6b70e8988c03af12154439f27a9db973cb09","horizon":1,"input_positions":[8,16,25,33,42,50,59,67,75,84,92,101,109,118,126,135],"input_token_ids_sha256":"8d01b29f328a257d664952e75b1923e83d2777cbcd4b7dd6cee1855ba4bd0e23","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,26,34,43,51,60,68,76,85,93,102,110,119,127,136],"target_token_ids_sha256":"8b8ebd92cf550b5df42ff3360fbfe39d8efc120a7e745a701b2476dc01b5b305"},"formatted_content_sha256":"44d9a5afa4c9790a32cb254c2d7e1c8151d16636b7c10318c889176cec4bab1e","generator_receipt_sha256":null,"identity_record_sha256":"fa45bce52100c237b5694c601b0c6d6d61bafa11dc08e55e2816e2dc5931e145","prompt_token_ids_sha256":"b9f642149302b4aad7f0cd78c10cba633079e023b978bce75611464d15bbca52","ruler_category":null,"seed":null,"selection_rank":119,"selection_sha256":"4d7d9814d186699e85046419caf35e8bacfed3ab6650cbd36ef4705e2fe2367b","sequence_length":137,"sequence_token_ids_sha256":"b282f3d8b38678f37b26da12ae374fa98bd55719947e98f5670fa8a8530270c8","source_content_sha256":"eae1ffd3380c73bbc35bb8929b99e7549668c0ee1f54da998081ad923ebc2b98","target_token_ids_sha256":"92e8679c291f99696b3ac568510fbe994ddeef46b7ee37dd2252d2f7a9c4bf84","token_span":{"cache_exposed_start":137,"cache_exposed_stop":137,"prefill_start":0,"prefill_stop":78,"scored_start":78,"scored_stop":137},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3053356b37e0ec018499d9df09100d5d459ba083855ec3be6383459c76336c2f","anchor_positions":[18,37,57,76,95,115,134,154,173,192,212,231,250,270,289,309],"anchor_positions_sha256":"1d1fbb7874404570c3a9cce52a9bba801a787fd59ed00a041f9aca9922c13383","canonical_id":"902","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[18,37,56,76,95,114,133,153,172,191,210,230,249,268,287,307],"fisher_boundary_sha256":"b03addf9ce28714390d0c8aa72e4cf5bfc4d998e1e89125d619cc5de86bb2d27","horizon":1,"input_positions":[19,38,57,77,96,115,134,154,173,192,211,231,250,269,288,308],"input_token_ids_sha256":"a9d774c7e1c1f9d59bd7b757d0d21d9849e8eec8052afe6025037cde871559f7","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[20,39,58,78,97,116,135,155,174,193,212,232,251,270,289,309],"target_token_ids_sha256":"52cd56fb6fa5b1573d45b6aa37a594fb645c879bc5efa69d6edc441f4e348a6c"},"formatted_content_sha256":"5c1d9ec33b3a94931255d1220cce10d4ee097868c37884a8e19ae9f9a448966a","generator_receipt_sha256":null,"identity_record_sha256":"11c597e2cf580e07f77afc5041e62139cfccff5f5ab2c548565673fb0b500832","prompt_token_ids_sha256":"bf15bcbdbc30699f9cd0081bb00efafed21ea1abf4e5381da4e70c8a8ea2907e","ruler_category":null,"seed":null,"selection_rank":120,"selection_sha256":"4daa86751a582244f7d590b6ba730fc4e01e1d3eb676941a3552f1d490faba8f","sequence_length":310,"sequence_token_ids_sha256":"376f09bf804c7e1d57cc7e70ae46ecf6050c3c815f6a7cee4a95bd11199c01d1","source_content_sha256":"57ceedd084b1dac9a04f6d5be3b4a7e63c4474855a5cbb2577214912ffc07f8e","target_token_ids_sha256":"49502604cb2f2efcb1c5cee4993331ea6bb17ff438c88e61241bb8472ffbd75a","token_span":{"cache_exposed_start":310,"cache_exposed_stop":310,"prefill_start":0,"prefill_stop":278,"scored_start":278,"scored_stop":310},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"afd54f87610f85707a02f20f8b511f975ef944c51b61fa08f447b37ed40c6a78","anchor_positions":[6,13,20,28,35,42,49,57,64,71,78,86,93,100,107,115],"anchor_positions_sha256":"5af6cea1df262184ac4897d6d91d25605392e5fb08e41b5742c64172e9670d87","canonical_id":"955","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,13,20,27,34,41,48,56,63,70,77,84,91,98,105,113],"fisher_boundary_sha256":"1c62dce0ea57f80fb1a56a3ea511f2ca9f646f6413e8db3d99f5d99de6521811","horizon":1,"input_positions":[7,14,21,28,35,42,49,57,64,71,78,85,92,99,106,114],"input_token_ids_sha256":"ef01c4ac7ec6c6b186fdd4eb31a7f21ee8d3e4dc566355d91c713471887db336","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,15,22,29,36,43,50,58,65,72,79,86,93,100,107,115],"target_token_ids_sha256":"785091482f15a4ef0949a945686a1778f4de4cf07b8104861aa31a9db7b18f4a"},"formatted_content_sha256":"ef1384fcf9c059d64323dd5fa9880b686c5b7ea953ce70d245534bb1c6ed7137","generator_receipt_sha256":null,"identity_record_sha256":"f520632d9fe13e3b9ffeebdc9ea540d828dee798f54926584e9872d0fe7b87da","prompt_token_ids_sha256":"010438e2e3ae3ee5626141c874feb0f74e50fca32fd27c7d8e49a07e790ec560","ruler_category":null,"seed":null,"selection_rank":121,"selection_sha256":"4dd5c5d56c9d92dfa81aec10aa5beae38c2061e96e799b813b238234a0ce8e4a","sequence_length":116,"sequence_token_ids_sha256":"fcc6077ed67145358daf5bcf3eb1ece0e289f68f139666cb89192eec34ca4386","source_content_sha256":"8408dff25eaf845d972ce9ae510b9d76eaa3e7a6bcfb8b8c7a628d70d2dd6cf6","target_token_ids_sha256":"76411fdcb68c0c051819b5bc2c0fed5b4048dcb9000413fb60e5de16d537d568","token_span":{"cache_exposed_start":116,"cache_exposed_stop":116,"prefill_start":0,"prefill_stop":72,"scored_start":72,"scored_stop":116},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"6bfb1e634cbb65a8f9c6d30dcee64be9d19b2bf1774c4fa850c4b7b5ce5abb9f","anchor_positions":[6,13,21,28,35,43,50,58,65,72,80,87,94,102,109,117],"anchor_positions_sha256":"f40be3d1cf7064d7e86e3a810371d8f2d368583e1dcff4660d5d02fc4a37f43e","canonical_id":"640","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,13,20,28,35,42,49,57,64,71,78,86,93,100,107,115],"fisher_boundary_sha256":"261088f8bd3fe25be23146357491bf2699628f4ce493685b028cc1ddbac01eb3","horizon":1,"input_positions":[7,14,21,29,36,43,50,58,65,72,79,87,94,101,108,116],"input_token_ids_sha256":"db9064f4846367e8570c2c84fe797b835d18960e9211681a5e0daac0654b2347","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,15,22,30,37,44,51,59,66,73,80,88,95,102,109,117],"target_token_ids_sha256":"0a9ab2151f96a33137ea1ad001a0d76f499d390891517e6c83bd2ac4188c1207"},"formatted_content_sha256":"06826303346d9d86d61fc5abbffb92ebac62b2e090a9f70b82927eb026441a29","generator_receipt_sha256":null,"identity_record_sha256":"7b2ff5b497e0a82c3b0d260a880af1a82e4a4c08c8185dd3ce186597015d3a20","prompt_token_ids_sha256":"05a2a260c33e6e500d66729f3e56aa8f9ccd92ecf9825c6449d5f8fe340e67a0","ruler_category":null,"seed":null,"selection_rank":122,"selection_sha256":"4ddb9210eb7ae0fdab8792259d081bdba6eae451141c47a811a0245cec7ee62d","sequence_length":118,"sequence_token_ids_sha256":"20509fd474053e85eba21ed705bd626c6c6f337445f4daf2177bf8b506211954","source_content_sha256":"6438ac844fd844aa7880ed46075764fa684488592b7d562f4ae65218f8b35da2","target_token_ids_sha256":"fd3de534e5c7f8e743fbfaeaa6eb5d1a4c3d2acfdfd4a83cb4e9f6b0a9383173","token_span":{"cache_exposed_start":118,"cache_exposed_stop":118,"prefill_start":0,"prefill_stop":85,"scored_start":85,"scored_stop":118},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"f5ff8bcd127367dd786bcc2b0c253bbddf5577a9692a04a945dfd7611e658813","anchor_positions":[7,15,23,31,39,48,56,64,72,80,89,97,105,113,121,130],"anchor_positions_sha256":"4b59ffbfdc6171d05db7e4ce13fd81c13fb1c2704ef54cbfaede26eafe2b1382","canonical_id":"602","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"fisher_boundary_sha256":"4366416cc3edb9f1e2ee5e0a55178de659ab8d05d59a8cbbce27f688cce7b2e6","horizon":1,"input_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,129],"input_token_ids_sha256":"2db46a104b4295959d6a16c851488604e471fef800ab5243426a1d23cce719c5","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,33,41,49,57,65,73,81,89,97,105,113,121,130],"target_token_ids_sha256":"2b0416f82f51105c2f972ed61b19f300631a81a49326605f7f5bf45fc3d5957e"},"formatted_content_sha256":"48559307e617b830c4e7a2ef8b20df2f6a53a7ddd15f766c7415c7b8597ff2f9","generator_receipt_sha256":null,"identity_record_sha256":"baa5a63154617acc29a3b5192d3459d2edae4c0959c00d24e2dadeeb3837d757","prompt_token_ids_sha256":"0bb478e0d8162f79fa8ab3da8baed5b87413927171e2f2c8071c49ab30da5521","ruler_category":null,"seed":null,"selection_rank":123,"selection_sha256":"4e02d4ed1f0c6d170eb518cc4020d82ff4ab1bd18d4949c5913e319fa7cc9e83","sequence_length":131,"sequence_token_ids_sha256":"e7b30f61231d01e87b3afa4dee3faf2008c8e20ffd1b3a478e7a94932567aac3","source_content_sha256":"59b47992e5e199dff2a5b0af7ea0a02713ea803a5dcc6fc3db6ecda2c4879288","target_token_ids_sha256":"78bd9c18ec9dfb40ae35c333bd6dd45162c4cc291b415b84d5ae2a87dd1ea334","token_span":{"cache_exposed_start":131,"cache_exposed_stop":131,"prefill_start":0,"prefill_stop":86,"scored_start":86,"scored_stop":131},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"bf2d404007f3e1af973b816cfc665fdfc8257f337deb35322ac0077058cea4d9","anchor_positions":[12,26,40,53,67,81,94,108,122,135,149,163,176,190,204,218],"anchor_positions_sha256":"34e0f2438d49fed8c871a33d3818a40fc035b239ea8639ec66d64d1e59a9bb66","canonical_id":"712","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[12,26,39,53,66,80,93,107,121,134,148,161,175,188,202,216],"fisher_boundary_sha256":"957de9da750e31aa4980cb34b1fb160bb11d67e08faf67af18d3cd3e5289f959","horizon":1,"input_positions":[13,27,40,54,67,81,94,108,122,135,149,162,176,189,203,217],"input_token_ids_sha256":"f56a9ec29643c9eba2eb44791cb09e84d9f2b231c902427ad5756e74ec534316","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[14,28,41,55,68,82,95,109,123,136,150,163,177,190,204,218],"target_token_ids_sha256":"c0a62a7a97676edbafdec6adb2cfe3c67719d5ccdcfcc3a0ddba0c6331624ede"},"formatted_content_sha256":"b47cde36a934283ca86c0366ca05754f6267c0fa319aa8c9ab18bc5ff0e5d9b5","generator_receipt_sha256":null,"identity_record_sha256":"4cc5c35280b65e6893e928c4b6a5b2caa568e303f0840d0df3438ecc6b258676","prompt_token_ids_sha256":"4ef7f3aa586f42e443494b0dc4987d24f11b6cc50acb5df627b135819de32092","ruler_category":null,"seed":null,"selection_rank":124,"selection_sha256":"4e132dddf48254f73a1575b1a311842bc8036e6c5310eeac68d27fcfa9705827","sequence_length":219,"sequence_token_ids_sha256":"45c3e814d2308b690e19384d22ca303456c689d9292a9292a9974a96d2c27dfb","source_content_sha256":"5355ae6f3f743535c93a53ce5c0f05092f5c9e89585eeb219a5a424a29b6a20e","target_token_ids_sha256":"8c478cb16206b569ca9079b27faaad0f0bf99e2182d4dd8ff62cd8328afac5ca","token_span":{"cache_exposed_start":219,"cache_exposed_stop":219,"prefill_start":0,"prefill_stop":183,"scored_start":183,"scored_stop":219},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"36c3b31211a995bad7f56d142b515472a78c8c0bab6bb376f23f53c90f2b7139","anchor_positions":[8,17,26,35,44,53,62,71,80,89,98,107,116,125,134,143],"anchor_positions_sha256":"486e7f47d72e2b3d4a862156a2b41e9dbb05a70d78349fdcac0b01e09667750a","canonical_id":"606","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,25,34,43,52,61,70,78,87,96,105,114,123,132,141],"fisher_boundary_sha256":"bcfe34f783ad5a75f3e21e034bca0957d966c976fd05656202a26e596ce4d242","horizon":1,"input_positions":[8,17,26,35,44,53,62,71,79,88,97,106,115,124,133,142],"input_token_ids_sha256":"2b725b30a4b9b4a7e69480982037a0a07c57e454ad8e62b46c0f0d34a3b1774a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,27,36,45,54,63,72,80,89,98,107,116,125,134,143],"target_token_ids_sha256":"872de4a20f2b57da5f4caff92476bd417ca5b3e9d701ef7d101a9fd65f42abed"},"formatted_content_sha256":"c6fed4dd2cb64cca6e833bf23797bb7c1d862a9c875ccc65acb9b0897618cc79","generator_receipt_sha256":null,"identity_record_sha256":"a9e7062d3f002e0a39343fd4f410be18e7409cd9643a09c6dedbf4bf79332f30","prompt_token_ids_sha256":"67adef290bf71e36a6af78dc46a182ea33221b75714d4b89977dfbfb25d08ad7","ruler_category":null,"seed":null,"selection_rank":125,"selection_sha256":"4e211b7dc81626479060883502f3e73d8b09c6e4da35910a1305a8e23d6e76a7","sequence_length":144,"sequence_token_ids_sha256":"e35fd3a90e9e66af3e142095f74dbbe6c47e3a63d75cede17749f5f6d49e6dd0","source_content_sha256":"3a119437b4f93db7da36fbe2d787607dde294a4a838f8b29d327bc68a0eb0478","target_token_ids_sha256":"764d32b0ec71bb9f1d078ccc8e4a39c3ce7dd359fa503899ebfcef2b1653ed9f","token_span":{"cache_exposed_start":144,"cache_exposed_stop":144,"prefill_start":0,"prefill_stop":117,"scored_start":117,"scored_stop":144},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7d1f918958bc691d31836ac50e195b86b748a5336a8a243a1b427eb6b289303d","anchor_positions":[9,20,30,41,51,62,72,83,94,104,115,125,136,146,157,168],"anchor_positions_sha256":"a040c10bf4081d1ff5f0840fe94899e3d5f4cc8b3513ac78d8caa23e146870a7","canonical_id":"656","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,30,40,51,61,72,82,92,103,113,124,134,145,155,166],"fisher_boundary_sha256":"c9142db1a8573b215599b5dc489eb1c6d88615d3cbf8ad1c104742ef7bfc5041","horizon":1,"input_positions":[10,20,31,41,52,62,73,83,93,104,114,125,135,146,156,167],"input_token_ids_sha256":"c32bee20d7de9784fc14e7f94e2b1e165c75b8c09339f3ecc44be1cbad01f8e1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,32,42,53,63,74,84,94,105,115,126,136,147,157,168],"target_token_ids_sha256":"abafaed5c5aabca6d515ea1a705691d431f604b08089f1f0b25c6df8b45c8305"},"formatted_content_sha256":"374f6e67c0234e75d236ea8f2069dcdf8a631ce916aaed870d7810e3347cde44","generator_receipt_sha256":null,"identity_record_sha256":"770edcf1d6f1f10e0e3b4c54ae7a7e8a532a07b1a3d17003f426dc095bed8679","prompt_token_ids_sha256":"b45a43c9c890b88c42ea8fe0a8f0a5d71c55ec6f01a772f30a952fad7f8431f5","ruler_category":null,"seed":null,"selection_rank":126,"selection_sha256":"4e7a2318f515292e8fc2e6b77f63c13218916891ac0b3f71650ab59f87fc1a5c","sequence_length":169,"sequence_token_ids_sha256":"c61b8bdf63e3f2d95107fee666460e20eb1e4a5d5420d41d523360fc8916c990","source_content_sha256":"7164e42f57848f0ec61d9fcc2a78bec0f09e6a54082574820bd69a7607d84721","target_token_ids_sha256":"52ef9360d90d7b7b754a31958af231bec9dfcb171dd416721cfb316918352b68","token_span":{"cache_exposed_start":169,"cache_exposed_stop":169,"prefill_start":0,"prefill_stop":118,"scored_start":118,"scored_stop":169},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9613ae532d1ac6180936e714bd20e0522fe6966715f1926f3a3b8b7af31b80be","anchor_positions":[11,23,35,47,59,71,83,95,107,119,131,143,155,167,179,192],"anchor_positions_sha256":"97c036645b9b6a78e53e848879a7e3ee08a70ff938ed6e98aa7098b21ab4319c","canonical_id":"671","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[10,22,34,46,58,70,82,94,106,118,130,142,154,166,178,190],"fisher_boundary_sha256":"730eb3d8f69aea1c1666e7307d73b83daf05f241bec613d1628b92a50ec3bcaf","horizon":1,"input_positions":[11,23,35,47,59,71,83,95,107,119,131,143,155,167,179,191],"input_token_ids_sha256":"1100089821e0f1740660a914b8b15a1db0534566177c6e09bcf34f670905693c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[12,24,36,48,60,72,84,96,108,120,132,144,156,168,180,192],"target_token_ids_sha256":"8d49d123f2607a365571c71836e7acda4676f3df261e0a4254afd127a53299ec"},"formatted_content_sha256":"22bd9c0d68e48a76c31dfe254036ce6008c9abd1f71c4c290e76fb4135d09ad0","generator_receipt_sha256":null,"identity_record_sha256":"4cb68a3b9440b00f53a5e655ff7024c46d2ecdc277ae03326b75f41268b173cd","prompt_token_ids_sha256":"63bd2370a08ace97ff440307492d1bee8479e8e14cd40ba9b40b1487a9ad53fa","ruler_category":null,"seed":null,"selection_rank":127,"selection_sha256":"4f4ba031f801028fbf78f774438c1e6ef3c15a58f65dccc230632c90a7fe6b5b","sequence_length":193,"sequence_token_ids_sha256":"d653c8bb051679b00cd66c5370bff1e217b990629e9ca260192942d1541dc892","source_content_sha256":"85295da947527bd6a8cb51dbe0f81aed64a26a245e12c289dab42daa0830ecdf","target_token_ids_sha256":"5233a54b205f9eef3872026a1b044403750ba40d368571e658bd36b909a1a85d","token_span":{"cache_exposed_start":193,"cache_exposed_stop":193,"prefill_start":0,"prefill_stop":89,"scored_start":89,"scored_stop":193},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"797956c3bb60ce0f54962ed315c69368b4d9168e36abfd675d14ccda36e8b2fc","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/36808","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"7e362e252ec4500305135b0f2ef134e2fe129c787fb0a31326c35439cc9ff4fc","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"d0e5311a6d6bde4f201973a4428fa08ba4fa97dc6cf2580fce9e2a4604922291","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"5b2da07174c560f02d4940468e335ad6777df3f471b2be9a3550cae36b1fd62d"},"formatted_content_sha256":"f6e3c61d5723ac6698db28d6198db5b270f51e83e2c8800f5ab2cd181a993a21","generator_receipt_sha256":null,"identity_record_sha256":"e7274663db91b5411d8af35ca6979c6b6038b747658e034486c7161db217a12d","prompt_token_ids_sha256":"5233822ab87654bb7050a9eff9f77a58c8ca4a74c176420316866641587b5ac1","ruler_category":null,"seed":null,"selection_rank":0,"selection_sha256":"0000d64fc1c2791eedccba1b0a014ae30d36a69ee95be69a7d2dc5541d9ff914","sequence_length":2304,"sequence_token_ids_sha256":"5233822ab87654bb7050a9eff9f77a58c8ca4a74c176420316866641587b5ac1","source_content_sha256":"ec7b13f2bb2766533058782bbf50f269e8c0743722581ba193e8dcf47008ada9","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"5ca98bf769dab4ebd4692e9fbb560efcc0f0f474c5a49e0732d605a2683da881","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/32024","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"2b3d247e247806b4dc881d8599ac43f4212a4abfdf27c4a4984cc7b6e286bcde","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"231119f3847a33472d36eeacd144d75fbf1c16d0971ecebd977dec5f84685ed7","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"bcf8ceb1b07052b45eaec59a1762d47a3fd14770e2014af6c7462da3ae912a8b"},"formatted_content_sha256":"056228535dc5d8d08c833c37eb05eb57a524d567c7137e2d8314c181aeca5137","generator_receipt_sha256":null,"identity_record_sha256":"48c93886d09d16bbd4b1c300f10b64a7a5908445969fd866133d7dbd65c13133","prompt_token_ids_sha256":"20d38634a5307135f41953611e0e9d2a8e22a29a1f912d78af0414bcca3e0dc2","ruler_category":null,"seed":null,"selection_rank":1,"selection_sha256":"0002ed283941f3da5106f697d697735d5064de6f22aa84e512fd5fb4792c49a8","sequence_length":2304,"sequence_token_ids_sha256":"20d38634a5307135f41953611e0e9d2a8e22a29a1f912d78af0414bcca3e0dc2","source_content_sha256":"eb06d835db68d2ccee63052eac9d025e120ca9331c84186892dbfca0b8c7b5e9","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"870cf6deead4c720c1131fda048c46f654bb50af3e403058e9370e06d21978fa","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/20279","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"9a2ec9cafe47a0a911539d7cf75996655cafccb2071b17063c4d74a9a215b6bf","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"b6c9a50b23af8aa52a00688ce9f3e2f1b86afb30dba305513f107d56499b7109","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"484ad2459b1ac7e91b8a406d092192990e1eb1f2ef4295cbc9ae9b0d8db832fe"},"formatted_content_sha256":"ad22fff4026303176b09131a60d9a8a6d1a6385ccaa59768e5f6a72ff8117f2b","generator_receipt_sha256":null,"identity_record_sha256":"01c99745bb7d41ee57ea15450fc853f25c79f09d2f8bc063c0bdd3bb66dcc3da","prompt_token_ids_sha256":"aa29883de5d02657b8eddaf1e1caece900a2d6c56fcf63a868f35f285132dee9","ruler_category":null,"seed":null,"selection_rank":2,"selection_sha256":"000d6a7d93865e2955a912d7fcafdf0e79cc24fdca9b059114ada982813dedf6","sequence_length":2304,"sequence_token_ids_sha256":"aa29883de5d02657b8eddaf1e1caece900a2d6c56fcf63a868f35f285132dee9","source_content_sha256":"b899a1b6c5b3ba61252a07cb53ea7d0f15f0c63315df7b872a9c7cacc58017fd","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"c3a77259d6b11a251eaddefe696ec8873dac79b38e498e4c19008e424264366d","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/35786","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"049cec2caddadd6c5a22d5eebe0f37af9a7d660e1f8a0134c85404ef60e34076","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"34af22d248242c1ad9044c3b1cc5988e257371a5867a4bc4f686e53596cf32a7","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"e400ca0c9462a4d8e94a16cb560f6a54f948fead3ee81377ae34f4b435d27bf6"},"formatted_content_sha256":"7e0c728729a3f6e258bf44c314097e48e5bd7a3d2ed6a9f7c33b0dc333de0830","generator_receipt_sha256":null,"identity_record_sha256":"182e8520c6e45d8da1ae7d54afd83292ddcbb68a76cccbeb96d22ffdde1b9464","prompt_token_ids_sha256":"a074a6e0e6624af64b673167b490ce4de6ff1fc5bc6a5a0429a482706d5047aa","ruler_category":null,"seed":null,"selection_rank":3,"selection_sha256":"000f8dbba0aa87b15dcf3bb9e900e6a1c95b96affc23284c0aad2871def7d41d","sequence_length":2304,"sequence_token_ids_sha256":"a074a6e0e6624af64b673167b490ce4de6ff1fc5bc6a5a0429a482706d5047aa","source_content_sha256":"d99d106567585d3e0dbdbd545794876258507c655b4b92af1dd705f6e4c984ce","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"6e96a1becbdeb74006e814bfe6a5fb240a5de9f6134d609b13117dbe7211fb78","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/29413","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"f68c33e180124100446c92def52974f274452469befeff1421363223dfd67323","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"9c4a80f3b25c53840a44de74f52c06907a16b90bcc7a8c077dc4eea1fccba752","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"85a930c5f1ed93345b9259991c2d9c7a90faace00b3ca7248e61b28fbbf27372"},"formatted_content_sha256":"aa37e0852c2a9d938e44d1f6050bbe8db2ca60ee6d89c2e40ed5ddc5f9c36a77","generator_receipt_sha256":null,"identity_record_sha256":"c8f11e6e9703b0eb65e3a6f5f79135730de277373c9e3a94e68eab94f5054d7b","prompt_token_ids_sha256":"5f2026b82faac8988768fc7330d60a392741c3a5f7b364d0b87bf8d596e74bdb","ruler_category":null,"seed":null,"selection_rank":4,"selection_sha256":"001145473f2b3b765745639cd14dac4b421dffa50f63df360f9059d362dc9b5a","sequence_length":2304,"sequence_token_ids_sha256":"5f2026b82faac8988768fc7330d60a392741c3a5f7b364d0b87bf8d596e74bdb","source_content_sha256":"12b0ce7832facf193492ad1f568ff0a0b25bc5e52381a9690df089bc8a9b1db8","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0244917a01282995b95e9ac9304e280f4253bfbab1626b6c1c57bf69a558b519","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/24903","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"d50289a8f2b5eec9977321d118785e94090fabd5a2f4938b5e6f1c63045b237e","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"ee472d2e8db03638388106e721863599f643316a3422a5245d0628a1edb69010","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"4b9781c37044a00962c0018332d1f3f966c9e3b9ff49f267b124cbe927314707"},"formatted_content_sha256":"32ac15efb7073af0ae11d3032d1da75babff6e7bd6406334879047edbd6e821a","generator_receipt_sha256":null,"identity_record_sha256":"b26e48a4638561a3f7f57dc2f6915accfeabce22cb26f87ddd65919ddf7a653f","prompt_token_ids_sha256":"7b786394510a818db1c521f6d71eb3b4052359b0d28d0a57d166057d80326905","ruler_category":null,"seed":null,"selection_rank":5,"selection_sha256":"001280ff6af9eba470ab76ff48bc2983cffe1995efb051d68cffd40cbaac4d3e","sequence_length":2304,"sequence_token_ids_sha256":"7b786394510a818db1c521f6d71eb3b4052359b0d28d0a57d166057d80326905","source_content_sha256":"14074cbd779ec8e06e0e5e91c6c6e6c02ebe2d185bb5310ab0dfdf64230019d7","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"67b1ede620a9bdb86c619e3124eef21332be86df3795cb4169fdf93c1bb4c46c","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/10609","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"51f6c6102f8fa772b2f8e1665ab91ad76a5f4f11205009bb3a8af2e9c9d4c06e","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"3d06c4debf0e8c528b85d65d243bbe561bcd13afdcb83d09b2d604fed1853965","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"9bdcf5cea67426ee4ade841fed2e6833ccbede63b7b5497449fa2279f5e54182"},"formatted_content_sha256":"8639254a29be3154bbc110c673738678316be09fe6f456878dac249bdeb1f4e0","generator_receipt_sha256":null,"identity_record_sha256":"a21cb3bd2e34a22b91569c2942d19d8bafaed2f11827ee9c58af3cbe366d479a","prompt_token_ids_sha256":"0697f0eea8df7d85bbeb6a688ab15d4294212694298ccd864b429ddac3ed7a4e","ruler_category":null,"seed":null,"selection_rank":6,"selection_sha256":"00131239a626272348186654735088a9b4c813c3bd83827262f7891e6056d43f","sequence_length":2304,"sequence_token_ids_sha256":"0697f0eea8df7d85bbeb6a688ab15d4294212694298ccd864b429ddac3ed7a4e","source_content_sha256":"49ff1f749aab76c1df7c47e6c79fdee9dd3fd191d073320f0ab1f5f361059fc3","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0d6376f0009c55e4f4efbf0d650cfe974856c049c8b14c303c991dd3f6c05c19","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/39447","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"7914f39477c00d3c1516c2e43c441114314ef252fe82526ad293b346ab359284","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"7d6865c318074116ceae5d7735c27fed5066ae326d49cbc3280f9f0289cfb7cc","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"ec95c049c76add2ddc5fa9d5301e2bddb64906cc8e42e8438a5c2aeeffb46499"},"formatted_content_sha256":"033b6ca5d9572a314560961826373bc34657c08759a1919296e81a4cb84c6471","generator_receipt_sha256":null,"identity_record_sha256":"f59cda6a20ec53af40493bf6be8ad0f898023c32531b81fe0ff0fb3b13340ea9","prompt_token_ids_sha256":"81890a7b8319487c5f4c312ca211202afd2d039e46e4931b65e09c6e446fa634","ruler_category":null,"seed":null,"selection_rank":7,"selection_sha256":"00166693ce38f4c5185c40c1258a1a602a290506112c351fa067daa039c2d960","sequence_length":2304,"sequence_token_ids_sha256":"81890a7b8319487c5f4c312ca211202afd2d039e46e4931b65e09c6e446fa634","source_content_sha256":"2d19f7657219986df6eb9bcf188544ab7660f95cb5521668082b48da38ac52bf","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"6f5f79cc3c1180f2cf00501aeafbc8d0f1dd6e6e7ccc69a8648c032e3469df45","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/33622","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"9b4c1f8e6fd129beb4bc474e6514815a95af657b4e2bc5fe7a7f32362dfb40bb","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"dcd27f6003c74d9c8a85711934161ee0f2636d2f0d27c4137e156f0d5064649e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"536e8bc4cb2331a406028d335233022e273cb3c563ac84e3db83762692164b14"},"formatted_content_sha256":"de944eeb38c1a6f2dfea1cb1657adb820f3d66ebe3aa361c72a39823aad3fa73","generator_receipt_sha256":null,"identity_record_sha256":"344ad881a15a994684e1ad39c738fc64b472325c7c46c63c837f255455d9f5e0","prompt_token_ids_sha256":"93be74d13c5df9550e8923cb9af9086891766e59060bc9997c00a1c14458302a","ruler_category":null,"seed":null,"selection_rank":8,"selection_sha256":"00191187cdc6cab1097f558bd88f30baeb7d1f08e9eaffd47828d8a1345ed53b","sequence_length":2304,"sequence_token_ids_sha256":"93be74d13c5df9550e8923cb9af9086891766e59060bc9997c00a1c14458302a","source_content_sha256":"dcf794cb44d57515b528594025b879dcacdc8e06e892b1a0e62d26319aa8d8d7","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"f602f879f02588bab48170a79cd2e407d35c19e5a4e05337a2615b62c1d0c64f","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/28065","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"3ab0b82eda5a33b4272654c9de3e352bda591b27e5499e28d3642e5a138ec7f8","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"f29080ea5125a83d114344a4897dcb5e6cf6135a3d28a1b55374cdb2836dd1db","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"34dd2e61b1ba68a0bd87b1f9e8650c6869d66f35aa98c549feab516d17c3afd8"},"formatted_content_sha256":"f500fe5cd15c60256f64965b23d5784b6badce6cc0da7c4792ac3017524b173a","generator_receipt_sha256":null,"identity_record_sha256":"73be35853dc6427eeab216a511cbdeba7dbfbab63feb81bd2642b1d99bbe5f6e","prompt_token_ids_sha256":"1c701cd7ff6691ab6d74de0874349dc37eac3d7dd6450cdb43b876a5aa7ea76b","ruler_category":null,"seed":null,"selection_rank":9,"selection_sha256":"001ac4d7e72067b598bc36c7582913c54df342db338498dd73a106eec9c8ffb6","sequence_length":2304,"sequence_token_ids_sha256":"1c701cd7ff6691ab6d74de0874349dc37eac3d7dd6450cdb43b876a5aa7ea76b","source_content_sha256":"0beeb0285771e4bc2cee42c31f1b35d1570cda724294b426b193cdac54b83393","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0d2186f26771b127e5a2350edc81554ef21f6eb9da3e69d77f7377a307b8a2ec","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/1253","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"4db1f2865c58e3a6faa8055be8d46b6c3f57be277f07fb5e39670a879cc599df","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"e20502727d9945c19d9f78c314ab8a1be19622e5a4316dd86e9823225d38135d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"be880e0cd3b16a5b9d737b8b2b7cd71353c9aad69f7dd469e024c5803d69f411"},"formatted_content_sha256":"a6e5b442be8b8fd1b45487216940754b214f379ae9a2021d30f82598049ca5f2","generator_receipt_sha256":null,"identity_record_sha256":"02492d4c2ba2a4f9435fd461ed91ea404250adb616636b5c5f6930fc4c350595","prompt_token_ids_sha256":"c55c7896671653ced092d5fc45ea5527e9457780b57375667e77dd586f161da4","ruler_category":null,"seed":null,"selection_rank":10,"selection_sha256":"00223684c9261647b075a573c94999d1d273100f31524ec62c5e6c73b4787787","sequence_length":2304,"sequence_token_ids_sha256":"c55c7896671653ced092d5fc45ea5527e9457780b57375667e77dd586f161da4","source_content_sha256":"6d1b9548166f4f9d82806a3af5ceb3ef3a1f644c2c946deb8aa86ec78d2c2fe5","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2608f932273743963113f1c10bf0e1c74703b9252284d4ab16394eea6ee00a1e","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/17774","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"ea92971f2ed12679772d304c1af6ce6e5d20a0a07f9c6e29ab3b8b95eb0c9c7a","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"de2670faad91b518e2bdc4f93edd92e3cf3b6d9e12fe24803a9dbb2d6ee7d8c3","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"d1ca5e91572227a6a1cdbeac2c3e0023a33d11172238246640c98d613aeb61a8"},"formatted_content_sha256":"2a4bbc348f9010bdfc1eec2d9ac77a487ad6f51468cfdedc35c10316e95564da","generator_receipt_sha256":null,"identity_record_sha256":"aba94b34901cf76209e4f0517738794062a4e8978f75d8d8e46bd007b32f2d61","prompt_token_ids_sha256":"25656de4135fa58df4d27bf5e1bb9122210cd21754c03b8b5f83f9412a6e2f5e","ruler_category":null,"seed":null,"selection_rank":11,"selection_sha256":"002494cd7b130c2e8101df6704ee2628630640a6ea6d8c21e9a0419c810d8f25","sequence_length":2304,"sequence_token_ids_sha256":"25656de4135fa58df4d27bf5e1bb9122210cd21754c03b8b5f83f9412a6e2f5e","source_content_sha256":"c71f6b5999e91ecf1857fb9dfb3f2573827e4ae4c5c3c4af2b08e3ba6a349e07","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"112223ea4e8c7a17eec0768b0bef76c663105f97785440668c062c5080b3859e","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/26008","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"2eb3fc663ac2fcca42c750d5cba63ad3d0d118921b38a48382c1cac4734faa8b","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"ecda06dc336cf17705f48dcde65a60f0b7ff432af083c07918dd0f1dbf09d79c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"e7eb1385302da9031a1227b768aaabecafafdb5614b3a0e2873e45ccfa3f7325"},"formatted_content_sha256":"fb608858dedf19c2f0a8e14d8710aa1d1c393be5a72ac8554731f0af33714904","generator_receipt_sha256":null,"identity_record_sha256":"af695a09abd9495d4e87989cbe7b352aebdfea88272863b3cade1adbd9d9c1a1","prompt_token_ids_sha256":"8eac346c569d9163ec8a9bde2355446690ca3c72a9a9f5c9bc00b745b6648f63","ruler_category":null,"seed":null,"selection_rank":12,"selection_sha256":"0027603c9d271c580c43673b3431be3adf061650ddff27e01b7869ea18673d9b","sequence_length":2304,"sequence_token_ids_sha256":"8eac346c569d9163ec8a9bde2355446690ca3c72a9a9f5c9bc00b745b6648f63","source_content_sha256":"469e817fab0a0502150988f96b4b16d5c4de03fb7f7bd185e1cff0715aee4b26","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"c447c8e41acc4393a51daf3ef28ed95971628266812cc78c05b887e341a38f02","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/38160","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"849f6474e4eb97b56439c66b9aaf2ac1fca275e49c29e33c48b2de0e806d1193","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"d11cb3a40f35a5034111c7bbbf1a8ff895d15396fea10ddb0cdbd97e8a47a5bb","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"ae53490b8b58ad6da63a300819db55d4a14e7b1bf25c07bbcd54b06469cc8b62"},"formatted_content_sha256":"b87156fd6d95b79ffe144b1b7448682cc84e8c2d06709db21214d82439b58315","generator_receipt_sha256":null,"identity_record_sha256":"f629ef55f22acb269a1825e56238e5770bf1bffc327d1d71d03370911d984632","prompt_token_ids_sha256":"55fb1408407a280bc2c65e78161f5cefb0ceaa4ceb04073817e0973d6a982b53","ruler_category":null,"seed":null,"selection_rank":13,"selection_sha256":"002a54c1b2c80d534795a77a68dfa9f3411107c3896ba33d31a868339d813293","sequence_length":2304,"sequence_token_ids_sha256":"55fb1408407a280bc2c65e78161f5cefb0ceaa4ceb04073817e0973d6a982b53","source_content_sha256":"f59885bfc6d88898f97c7bda7625006b3be0651da5929a1e272907e85a9772e3","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"14e443d1ba9ca076a4a9d065c90f0b3ccb2c24defaced1591fe7cc77f71e7527","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/2773","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"5a0b47a49833f2e6114f9b3e2d475b69a889ed7945db09507580c9536ef76448","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"f0dbd8116db99645f9271fcd040a856c59b111c6243748f64ddc1422928c64ff","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"78a78a32fdab2f986b289b32d0a4612fb33ea2014dccbbce3a60d65c16e765ac"},"formatted_content_sha256":"68c11a24ba674cdac7966cbaf0f71a9d2fc30a62a0d90a649a6554907ef00125","generator_receipt_sha256":null,"identity_record_sha256":"ddbf6728a3fc2c07defaa4546c34ea401f4fd05c829819301903562cd04a9519","prompt_token_ids_sha256":"8b935a7676a8137367066b0b1af7a39f635e0e3592fe3c1982a28bac24eb96cc","ruler_category":null,"seed":null,"selection_rank":14,"selection_sha256":"002d0ca3ff81dfc79dea1f54c4f28547f367149283e13dc83a14c114322353bd","sequence_length":2304,"sequence_token_ids_sha256":"8b935a7676a8137367066b0b1af7a39f635e0e3592fe3c1982a28bac24eb96cc","source_content_sha256":"be1775efe2cd9be13c65cbd65200b0cd27ae7d29b52ff493b27ed85e4b549c61","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1fb610bd6210663f10a294d4b45df223b7ee4e3b6a8c0de390f3e50612ea1219","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/1219","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"92e822869a8cdca6f8c09e9495406404c8a345c0a99d06dc98a948b7bcd010d6","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"7ea3f80cfbc818f9210b87b0153e90af5720869a1fe3f26b0d5f723bfa9eead2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"c1725ca6a384a182c8f164ec35f4faded719127f4cef2baec3d812545eff3e6f"},"formatted_content_sha256":"cd30711364e668a3275113e8a3f92ecee4b268935f5a095cdb0f0c1f11205c1c","generator_receipt_sha256":null,"identity_record_sha256":"2d205b87fc59eabeed0bda29df3e0789eef230026c46ad11796d41b866d18b2e","prompt_token_ids_sha256":"65e443512311bce75a506b694195aa105b7831541ecb92c1768fe5a0ce285ef2","ruler_category":null,"seed":null,"selection_rank":15,"selection_sha256":"0032c753604107e647b4ccd7ef877562bc0ca9c0e232a26a761388befd002fea","sequence_length":2304,"sequence_token_ids_sha256":"65e443512311bce75a506b694195aa105b7831541ecb92c1768fe5a0ce285ef2","source_content_sha256":"6cb2a106560ad166c025fb7a83708e519f95bca68cecb9a7bff61e261a0b8384","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"52b78dddc7c06fdc5e5ab3937802742ef3b3414f9743427d2bd71994c5cb1671","anchor_positions":[113,227,341,456,570,684,798,913,1027,1141,1255,1370,1484,1598,1712,1827],"anchor_positions_sha256":"9289ece2ebec2ce8ec20865a50beecbecea8ea6a3dc4b5c5c8ed9f6ad0d784ac","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multikey_2:length=2048:seed=12340:sample=0","config":"niah_multikey_2","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[113,227,341,455,569,683,797,912,1026,1140,1254,1368,1482,1596,1710,1825],"fisher_boundary_sha256":"a9016c04cfbd31eb6f72106eb0693c1545bbaa0451e228e6e447a7814d921eb7","horizon":1,"input_positions":[114,228,342,456,570,684,798,913,1027,1141,1255,1369,1483,1597,1711,1826],"input_token_ids_sha256":"dfd87b8cdcdc45c3aed78576c14b5dd53c5f3a8de7a4a69358857943c1f8ba1a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[115,229,343,457,571,685,799,914,1028,1142,1256,1370,1484,1598,1712,1827],"target_token_ids_sha256":"4ed30d7e42a7866174c94f532bdcc6947a15ad58245c18f3aba71a4612f05990"},"formatted_content_sha256":"d1e07461677a5fe92f27926505552cc8a343cd60a5b5be86c2fa00090fae27f0","generator_receipt_sha256":"801e44bdb609773d985e8fd74f1d853f1afb30cf0a66d2cee0f7013ae8749c9c","identity_record_sha256":"a1cb4798036f5d2c63f1da08e09141e23d4fdec224e854eeb2d770ac8ba11721","prompt_token_ids_sha256":"793e38d71d7e91af366df94fb359ffb203e20cafb77f7afbb412db3ac5cfc5d2","ruler_category":"retrieval","seed":12340,"selection_rank":0,"selection_sha256":"26538bdeb4a99f8904324b26add0826a13744b51ec94fd9a004c72a2ba68c287","sequence_length":1828,"sequence_token_ids_sha256":"793e38d71d7e91af366df94fb359ffb203e20cafb77f7afbb412db3ac5cfc5d2","source_content_sha256":"801e44bdb609773d985e8fd74f1d853f1afb30cf0a66d2cee0f7013ae8749c9c","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1828,"cache_exposed_stop":1828,"prefill_start":0,"prefill_stop":1828,"scored_start":1828,"scored_stop":1828},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"75123434e18fce7bfe0e4b995fd12769a05a08d35fc315c68b8438e286e23f77","anchor_positions":[246,494,742,990,1237,1485,1733,1981,2228,2476,2724,2972,3219,3467,3715,3963],"anchor_positions_sha256":"dff504ff5227ef8cb24123e368d13dfb8cef71f15a3e715472122831820e3e93","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multivalue:length=4096:seed=12340:sample=0","config":"niah_multivalue","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[246,494,741,989,1237,1484,1732,1980,2227,2475,2722,2970,3218,3465,3713,3961],"fisher_boundary_sha256":"b36b4cfcb1258559a6eeafb235f1efb9a3bc5d2d3ec31389b39290f7e7209d82","horizon":1,"input_positions":[247,495,742,990,1238,1485,1733,1981,2228,2476,2723,2971,3219,3466,3714,3962],"input_token_ids_sha256":"dd9ed3a5d8b1bac29e9c251660e29d7a61b1d0cbb9846d333768dd85128cd708","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[248,496,743,991,1239,1486,1734,1982,2229,2477,2724,2972,3220,3467,3715,3963],"target_token_ids_sha256":"80f466fa56b4c9b1c24b8b51bb8cacb08c7290a7b0f1fd9ceeb119534231b14d"},"formatted_content_sha256":"f6bd8b2f8c5eb25a8ea8e08f2f4fb8e581a8d2080ff40194bce0a86f5deccc0e","generator_receipt_sha256":"fc75f4bdae8929325589536065fb7c8f5bb1fd737001da616d339365aa94920a","identity_record_sha256":"8091f9201ab1213aa14972869be942841763c3897bee5b30b77fb3d34877dd86","prompt_token_ids_sha256":"d498355b8380354e768abc57a79afe8cc73c5849b98f01081fd945329db5e77a","ruler_category":"retrieval","seed":12340,"selection_rank":1,"selection_sha256":"34e79c93add91edaaf2f631ffbc9e4c5350f61dc96769cf971f03ae113ccd1b5","sequence_length":3964,"sequence_token_ids_sha256":"d498355b8380354e768abc57a79afe8cc73c5849b98f01081fd945329db5e77a","source_content_sha256":"fc75f4bdae8929325589536065fb7c8f5bb1fd737001da616d339365aa94920a","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":3964,"cache_exposed_stop":3964,"prefill_start":0,"prefill_stop":3964,"scored_start":3964,"scored_stop":3964},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"532ac060a0cb86e868a3d0d5f0e7217475df142f3480655a5f9cc923addbf97e","anchor_positions":[124,250,376,501,627,753,878,1004,1130,1255,1381,1507,1632,1758,1884,2010],"anchor_positions_sha256":"baf07dbe6e81bfe17433570a4cb75b5ed267ca19947bc93119fe6c078f612806","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=2048:seed=12339:sample=0","config":"vt","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[124,250,375,501,626,752,877,1003,1129,1254,1380,1505,1631,1756,1882,2008],"fisher_boundary_sha256":"9a1bd7b90a51362eb876693f97e6c20ec0fa6bff1077ed5acd1237a823804b3a","horizon":1,"input_positions":[125,251,376,502,627,753,878,1004,1130,1255,1381,1506,1632,1757,1883,2009],"input_token_ids_sha256":"d43a3eb571f5bfc8ea7dc04191b8336f22f066f154918abe65e057791d13a43e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[126,252,377,503,628,754,879,1005,1131,1256,1382,1507,1633,1758,1884,2010],"target_token_ids_sha256":"453114002e40922d3386b14ea8e4915c4aa872297dae554dfbf91fe2f3b3e9ec"},"formatted_content_sha256":"221e10f0973c46ef1f20a688046e4bffae24656998c1059a831d7b20726fede0","generator_receipt_sha256":"b9a9b27500644e9aa2532304d577b85ab988c4359dd5edf7630fc8a8bee4b60e","identity_record_sha256":"3c519c0d4df294b9bb4d57dfaf44f3fe4934ae8e697dc924d17dbf39644026e8","prompt_token_ids_sha256":"cd95e7b2f95f4b705b1b43dc6b201b8307b182fe379566cb2e900ef21733b21a","ruler_category":"multi_hop_tracing","seed":12339,"selection_rank":2,"selection_sha256":"48145a1ddcbd5ae2492d753c9ffdf69047346dcba3d4c88e131c036be6968d9e","sequence_length":2011,"sequence_token_ids_sha256":"cd95e7b2f95f4b705b1b43dc6b201b8307b182fe379566cb2e900ef21733b21a","source_content_sha256":"b9a9b27500644e9aa2532304d577b85ab988c4359dd5edf7630fc8a8bee4b60e","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2011,"cache_exposed_stop":2011,"prefill_start":0,"prefill_stop":2011,"scored_start":2011,"scored_stop":2011},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"deb02a9e929ebd4d409deb018ab87063b47401d9993356ae1689a17e44d9121f","anchor_positions":[248,497,746,995,1244,1494,1743,1992,2241,2490,2740,2989,3238,3487,3736,3986],"anchor_positions_sha256":"a5ad1bc8d8afa780351af0b99deecbf3b31c05390a133ecb9438c32584b13b51","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:fwe:length=4096:seed=12339:sample=0","config":"fwe","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[248,497,746,995,1244,1493,1742,1991,2240,2489,2738,2987,3236,3485,3734,3984],"fisher_boundary_sha256":"3ce3ab1d00aff5ae9a8f93284d14aaa6f1744a79acc88d0772877941983042b6","horizon":1,"input_positions":[249,498,747,996,1245,1494,1743,1992,2241,2490,2739,2988,3237,3486,3735,3985],"input_token_ids_sha256":"87a331f7d1b5d5467dcce6012238bb065bb86881942f6c071315f3911a04e910","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[250,499,748,997,1246,1495,1744,1993,2242,2491,2740,2989,3238,3487,3736,3986],"target_token_ids_sha256":"0c3be397f087a6670d953bee273680288fc932be2e83d55d3086f5f6090398bb"},"formatted_content_sha256":"0fead2f4a48fe6c038641241dc6133e35cf0290088d4b06bf388bbf505180d9d","generator_receipt_sha256":"7fe1d0a912ee13f2dc1aae6839a7f8e87a623d5df30a3408ac5c6059f88aed13","identity_record_sha256":"bef04af160238768f27800f450d76a76013d80a3c96282a8a684900ba5807d4d","prompt_token_ids_sha256":"1610d294ca00f9745a960e213dd10784d8a523eca88bdaded2c7bde663d09a1c","ruler_category":"aggregation","seed":12339,"selection_rank":3,"selection_sha256":"51065012a292ff5d1577ac8988306f3ebc15a9c955ac9c104f135c18f7ed7cc9","sequence_length":3987,"sequence_token_ids_sha256":"1610d294ca00f9745a960e213dd10784d8a523eca88bdaded2c7bde663d09a1c","source_content_sha256":"7fe1d0a912ee13f2dc1aae6839a7f8e87a623d5df30a3408ac5c6059f88aed13","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":3987,"cache_exposed_stop":3987,"prefill_start":0,"prefill_stop":3987,"scored_start":3987,"scored_stop":3987},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"8a6b31f4f399feceed15eefe68a896dcd67ad7e19c3225837e50ada41fc0a847","anchor_positions":[252,506,759,1013,1266,1520,1773,2027,2280,2534,2787,3041,3294,3548,3801,4055],"anchor_positions_sha256":"e3a8a2871a86f2cf789e76ba6df6ede1df61047b14b69480ff1217c0a911147f","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=4096:seed=12339:sample=0","config":"vt","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[252,505,759,1012,1265,1519,1772,2026,2279,2532,2786,3039,3292,3546,3799,4053],"fisher_boundary_sha256":"159f8ad58c26093c2f75744da0f9502f7a2906b033bd5fce2665cf525fefb38b","horizon":1,"input_positions":[253,506,760,1013,1266,1520,1773,2027,2280,2533,2787,3040,3293,3547,3800,4054],"input_token_ids_sha256":"e8a2ff60bb805b206d15a2e0bf12885fedc537e7082f756e2c445a6bf70c326d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[254,507,761,1014,1267,1521,1774,2028,2281,2534,2788,3041,3294,3548,3801,4055],"target_token_ids_sha256":"49fc01abb0da9ff437c1e5a9aefc9158a1f8db110f275f2946581b8be9491118"},"formatted_content_sha256":"946e2f3a8214ecc488a65888ebeb1dc5b42d3027357523816a5ce8d3414b899c","generator_receipt_sha256":"8ea9a21c91b1b5203697a120d21824bea8dc810455d228e58a7ae319dc72dd82","identity_record_sha256":"5556eeb12c7c154e817d9c46859eaeb64f6a54512b787f1b1f38bf9fcf284542","prompt_token_ids_sha256":"9973907d38c28c89e9a53bec04ecb4c695cc4c94f35050f705a22d2e6ab2c897","ruler_category":"multi_hop_tracing","seed":12339,"selection_rank":4,"selection_sha256":"5c5a0c61d80b5c3a76e5bbf1062b35594e4a4995d482c429634e381eaaadcf21","sequence_length":4056,"sequence_token_ids_sha256":"9973907d38c28c89e9a53bec04ecb4c695cc4c94f35050f705a22d2e6ab2c897","source_content_sha256":"8ea9a21c91b1b5203697a120d21824bea8dc810455d228e58a7ae319dc72dd82","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":4056,"cache_exposed_stop":4056,"prefill_start":0,"prefill_stop":4056,"scored_start":4056,"scored_stop":4056},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e6965757239ac98f811d9377515f2e5110de3926f0390ca3416c9f2e3ef74bd3","anchor_positions":[79,160,240,321,401,482,562,643,724,804,885,965,1046,1126,1207,1288],"anchor_positions_sha256":"a02a7b889641d261dbd8927a54b0e9bda873159838da4973760162d67a2adbcd","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multiquery:length=2048:seed=12339:sample=0","config":"niah_multiquery","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[79,159,240,320,401,481,562,642,722,803,883,964,1044,1125,1205,1286],"fisher_boundary_sha256":"57d7fe986325f9a24836dcf6f4afdba3cd327289b6aa8077b19ec4aaff566ce9","horizon":1,"input_positions":[80,160,241,321,402,482,563,643,723,804,884,965,1045,1126,1206,1287],"input_token_ids_sha256":"0a50d970537d979d05cfd20781db749fecbfb1eb42f1426875d990cadb4a07ee","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[81,161,242,322,403,483,564,644,724,805,885,966,1046,1127,1207,1288],"target_token_ids_sha256":"88866037b9a7485b03068ce42d898b0e5484251bc087329d421a95fea8993da1"},"formatted_content_sha256":"bfb4cc6fdedde1a2578a09daf8fa4e779725a810722080b59de46588e1d5ab2b","generator_receipt_sha256":"0e8034964c18899e91a435a5737bfa29f7791fc41c29f42e4f2d9d75fa3a98ff","identity_record_sha256":"6bb7156902c5f36fc7c3e3c92666c510e1a2d572d3c6c557cc98ab440dd94db2","prompt_token_ids_sha256":"c68672f5e746622c7ef9c8d45f9312057365f04182c7af0576723d8089244ca1","ruler_category":"retrieval","seed":12339,"selection_rank":5,"selection_sha256":"65f813250e251ff50bf38bf6b227abf56f6e43ea5dedef6b18e6cf94874e510d","sequence_length":1289,"sequence_token_ids_sha256":"c68672f5e746622c7ef9c8d45f9312057365f04182c7af0576723d8089244ca1","source_content_sha256":"0e8034964c18899e91a435a5737bfa29f7791fc41c29f42e4f2d9d75fa3a98ff","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1289,"cache_exposed_stop":1289,"prefill_start":0,"prefill_stop":1289,"scored_start":1289,"scored_stop":1289},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"128a207210316772af37d86a91c97448a5eec69418db26c573a15daa57fff7db","anchor_positions":[252,506,759,1013,1266,1520,1773,2027,2281,2534,2788,3041,3295,3548,3802,4056],"anchor_positions_sha256":"ab52ebab779359ace36255ddbc6dfccb71733c90923120903537310d833e1be5","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=4096:seed=12340:sample=0","config":"vt","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[252,505,759,1012,1266,1519,1773,2026,2279,2533,2786,3040,3293,3547,3800,4054],"fisher_boundary_sha256":"c02daef8e0847ca8ef8f6795ed064d750cbfb33070533ceaf84d3f199f95bdf4","horizon":1,"input_positions":[253,506,760,1013,1267,1520,1774,2027,2280,2534,2787,3041,3294,3548,3801,4055],"input_token_ids_sha256":"758c575c40dc855f729daac2fb4c427f22776d59517e67e7dcd8ad13b9015cd3","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[254,507,761,1014,1268,1521,1775,2028,2281,2535,2788,3042,3295,3549,3802,4056],"target_token_ids_sha256":"e7ee891d8494cf6174e6a03c1da26e912c52473297bcba994663141abda1e73e"},"formatted_content_sha256":"84db9b2d31c7c5cfb2df3227d24ed8fa03d50cc474dbaf560e4f7aee2e52bb97","generator_receipt_sha256":"5b6cd316a368db4420845994f9423aaeedc0fd75802aba9eafb7deb4f4eddc73","identity_record_sha256":"ce937620a8ea8bbf5af1e1b522055853a1ec44ac92520d74486cc8e8f0bae348","prompt_token_ids_sha256":"42167ea36f4f43f21c1eadf1bd9fb391cb7d16904f553e91eac1401c360e902c","ruler_category":"multi_hop_tracing","seed":12340,"selection_rank":6,"selection_sha256":"79c3cb58b42a4b44fb604b28cd22ae1aa71bc49e05a3eb8399ca698c0044ee51","sequence_length":4057,"sequence_token_ids_sha256":"42167ea36f4f43f21c1eadf1bd9fb391cb7d16904f553e91eac1401c360e902c","source_content_sha256":"5b6cd316a368db4420845994f9423aaeedc0fd75802aba9eafb7deb4f4eddc73","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":4057,"cache_exposed_stop":4057,"prefill_start":0,"prefill_stop":4057,"scored_start":4057,"scored_stop":4057},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0fe15004ffd8e39b3cd00a623cb86c9ae2e0570216811c075442338f6c8c6963","anchor_positions":[141,284,426,569,711,854,996,1139,1282,1424,1567,1709,1852,1994,2137,2280],"anchor_positions_sha256":"9ea473fa5cae3aef9908a7ca4fa168866d469f3650f18f07e5dce4ee02714d7c","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_2:length=4096:seed=12340:sample=0","config":"qa_2","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[141,283,426,568,711,853,996,1138,1280,1423,1565,1708,1850,1993,2135,2278],"fisher_boundary_sha256":"12c58ae371c707d325c0b7db5868000c046d213a4b92b2344f6482266e812bec","horizon":1,"input_positions":[142,284,427,569,712,854,997,1139,1281,1424,1566,1709,1851,1994,2136,2279],"input_token_ids_sha256":"e6b8490ddd4e543ee4bbe0a2096b3a34397a5c6e4ee16c9efb438fba25a387d5","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[143,285,428,570,713,855,998,1140,1282,1425,1567,1710,1852,1995,2137,2280],"target_token_ids_sha256":"644343be1688f80c9846724beff4e6d47423e4ced93613951f85b480fb1aa6bf"},"formatted_content_sha256":"1d3faed8d004df01dbd7309b8dd1ed0e326c1bd1c5fa54306abe22855487372d","generator_receipt_sha256":"fba640bf322cb8c1f6175cc1a73e6267b18618e8555d9601e202268427ebabe8","identity_record_sha256":"1e32be0f57d7bfa1580c73b4f1c5e8a06ff8f429ff4611e766534940463ad32e","prompt_token_ids_sha256":"dab144649219fd1b15323997ba14f284492db0e725562192c789c0f766c8de3c","ruler_category":"question_answering","seed":12340,"selection_rank":7,"selection_sha256":"7a527495d7e22511d3058e2b2a4a846cd8b244d93a021ee51622ee975c2530ea","sequence_length":2281,"sequence_token_ids_sha256":"dab144649219fd1b15323997ba14f284492db0e725562192c789c0f766c8de3c","source_content_sha256":"fba640bf322cb8c1f6175cc1a73e6267b18618e8555d9601e202268427ebabe8","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2281,"cache_exposed_stop":2281,"prefill_start":0,"prefill_stop":2281,"scored_start":2281,"scored_stop":2281},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4a35e619dc657f107bad1efd0356762a6276d17f1c44fe7c843454ce15350f14","anchor_positions":[84,170,255,341,426,512,597,683,768,854,939,1025,1110,1196,1281,1367],"anchor_positions_sha256":"52762b5ae990ebf341f9864370415ddc0be98c34b1415f0fb53c8f4bb5237780","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_1:length=2048:seed=12339:sample=0","config":"qa_1","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[84,169,255,340,425,511,596,682,767,852,938,1023,1108,1194,1279,1365],"fisher_boundary_sha256":"71d2d40de65a4fd2d9d9e691302e52d3a9fc059b1629b02c3b6543f8201e463f","horizon":1,"input_positions":[85,170,256,341,426,512,597,683,768,853,939,1024,1109,1195,1280,1366],"input_token_ids_sha256":"673327b5691ed16299c8dd34b55f88b87d1fe642535708588295ca0f00f89422","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[86,171,257,342,427,513,598,684,769,854,940,1025,1110,1196,1281,1367],"target_token_ids_sha256":"aefebb06df48a7ba94a6d5de230228ea289d46553d344edc66924e98dea4b336"},"formatted_content_sha256":"9463b4927878ab07df8a32bd0b0584b76f5d42182dc21d90193275757cadc022","generator_receipt_sha256":"8d5f10bad274c66452a465f737da20099210de1d161c27a36c6331783da5f4f2","identity_record_sha256":"9198d8dbcec2c86f25636e37bc394d9d761cea2bb93559b1786b527f69e7fd7c","prompt_token_ids_sha256":"1102e50da7352084f75b9fa23ab6cc2d70708530625fc296126f13ff696d77e9","ruler_category":"question_answering","seed":12339,"selection_rank":8,"selection_sha256":"8bfbb4cad87302a003026059836508ab6e2d89448865d0217704654cd5b9bc8e","sequence_length":1368,"sequence_token_ids_sha256":"1102e50da7352084f75b9fa23ab6cc2d70708530625fc296126f13ff696d77e9","source_content_sha256":"8d5f10bad274c66452a465f737da20099210de1d161c27a36c6331783da5f4f2","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1368,"cache_exposed_stop":1368,"prefill_start":0,"prefill_stop":1368,"scored_start":1368,"scored_stop":1368},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"56eb209f76f5b84c51f3a6dfa390051c3e6179a589094e42a630548e33c79ab6","anchor_positions":[117,235,353,472,590,708,827,945,1063,1182,1300,1418,1537,1655,1773,1892],"anchor_positions_sha256":"0f267c5222d70787fd8238bd3de2e5bcbdfa19d30f6519cbc9abe6eb49aff8f8","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_2:length=2048:seed=12340:sample=0","config":"qa_2","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[117,235,353,471,589,708,826,944,1062,1180,1299,1417,1535,1653,1771,1890],"fisher_boundary_sha256":"433027cb3aa35bbaaa74d9db1d137366f2b08b50e51181e868b602a2a55b6952","horizon":1,"input_positions":[118,236,354,472,590,709,827,945,1063,1181,1300,1418,1536,1654,1772,1891],"input_token_ids_sha256":"eef4188a93a14820c4b8529c64e375c3dc36f29c2314d289c81684b9e3d1db9d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[119,237,355,473,591,710,828,946,1064,1182,1301,1419,1537,1655,1773,1892],"target_token_ids_sha256":"828ccb325b0d5b2cbe71be59f39c620de4093e298de9f0dda10d92926f2263fe"},"formatted_content_sha256":"696b7eb132f4a3d5f49052ac7680047c4a0a1ad1fc05f0bfa2bb25b4ddd67353","generator_receipt_sha256":"e58aa114cbdb8502b1dff17517a049edca17022eacfe90cd5afc0e2eda097c3e","identity_record_sha256":"67d8a257b752e3b5c57062870f7fde1fc0efbf2427ed7c5aaa4b1f1b94419ddc","prompt_token_ids_sha256":"ca3e9180e70700d36dc0d15f1ca4e7e4febee3a9058814b3f575e89f58f04daf","ruler_category":"question_answering","seed":12340,"selection_rank":9,"selection_sha256":"8fd992ea1ad818ee9cb171e5d9070ef59e24c95c1dfb3549cdc3963f904c96d3","sequence_length":1893,"sequence_token_ids_sha256":"ca3e9180e70700d36dc0d15f1ca4e7e4febee3a9058814b3f575e89f58f04daf","source_content_sha256":"e58aa114cbdb8502b1dff17517a049edca17022eacfe90cd5afc0e2eda097c3e","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1893,"cache_exposed_stop":1893,"prefill_start":0,"prefill_stop":1893,"scored_start":1893,"scored_stop":1893},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"cc7b46a7e50bc99f0cdb680c8631375304bb7983eedd6e6bec9dbbd2fbcfbdf5","anchor_positions":[251,504,756,1009,1261,1514,1766,2019,2271,2524,2776,3029,3281,3534,3786,4039],"anchor_positions_sha256":"7360c0b207ce24b236c422e4e664bff53fcafb7e524bd09aaebed6ea7c3ec7f8","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_1:length=4096:seed=12339:sample=0","config":"qa_1","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[251,503,756,1008,1260,1513,1765,2018,2270,2522,2775,3027,3279,3532,3784,4037],"fisher_boundary_sha256":"e551a79a205e6b53d8d809cc13da1c1d6f98faf520fcf9e40ed135aee7b9bb8e","horizon":1,"input_positions":[252,504,757,1009,1261,1514,1766,2019,2271,2523,2776,3028,3280,3533,3785,4038],"input_token_ids_sha256":"eb303aed866e00a6665d3e56e5ccdf9775fbd1e8a9bb7f5652c3fd7bd9707424","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[253,505,758,1010,1262,1515,1767,2020,2272,2524,2777,3029,3281,3534,3786,4039],"target_token_ids_sha256":"fdd7b7b8e4d53d09f5fa32c50c573deb2dbc0fe9474bed57416b8fac4ca54c1e"},"formatted_content_sha256":"9c7b681fe9e8d8f416aceaac08bdfd92d885e97dc140320d76ff0d9fefb803db","generator_receipt_sha256":"2dd908d09753092d8a358262f5073246919dd0069dfb0a60eca406aba764caa5","identity_record_sha256":"de00cb025c394a25eaa32bd222daf198e16413a112f3bccd5a72b628d593cd51","prompt_token_ids_sha256":"a089b295fbea580bdf8d050e97c37b242dfd30da2c3c7a3e620b76a54ed9c79f","ruler_category":"question_answering","seed":12339,"selection_rank":10,"selection_sha256":"a114fee36328bb536916462de6cc07e89d3dd7b3809c5295bc63e2663d7973f7","sequence_length":4040,"sequence_token_ids_sha256":"a089b295fbea580bdf8d050e97c37b242dfd30da2c3c7a3e620b76a54ed9c79f","source_content_sha256":"2dd908d09753092d8a358262f5073246919dd0069dfb0a60eca406aba764caa5","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":4040,"cache_exposed_stop":4040,"prefill_start":0,"prefill_stop":4040,"scored_start":4040,"scored_stop":4040},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"271a989874c4cc17ca1a739347bba23628a9e467e85d3219bb363f9ed7b07bf0","anchor_positions":[115,231,348,464,580,697,813,930,1046,1162,1279,1395,1511,1628,1744,1861],"anchor_positions_sha256":"cf6b395abd6929080fc35256e6179d2e3c657bc70c02f16093cb0533b0035bcc","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:cwe:length=2048:seed=12340:sample=0","config":"cwe","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[115,231,347,464,580,696,812,929,1045,1161,1277,1394,1510,1626,1742,1859],"fisher_boundary_sha256":"4015c686931b188bc2932b79e3bfa9ac30c8393959c5972c66fb8b82816a3bfd","horizon":1,"input_positions":[116,232,348,465,581,697,813,930,1046,1162,1278,1395,1511,1627,1743,1860],"input_token_ids_sha256":"7a7df1462084ce8ab589c85b605a1f4311c0ec91dc306a02673a84285b192131","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[117,233,349,466,582,698,814,931,1047,1163,1279,1396,1512,1628,1744,1861],"target_token_ids_sha256":"77743f96c0e111a7c8ad0cb12178be0abbe535b25edec949d8812bb2522c07a0"},"formatted_content_sha256":"3b5a450e8c283afa9103e3393bb8414b1d833a42c88cc5d7d4bd3f26a6120467","generator_receipt_sha256":"1c23f946536f8b05ad81d768edf4b1a5cc74ce39a3e9048124593635eb45ad57","identity_record_sha256":"21e8e431c773084606e1bb52807ebb59255d6a84e9bff884ebb333b51b0833f7","prompt_token_ids_sha256":"52bbc9b8942f395aaf5ad8a6b712b521a6a64d665cf085cb5fba76caccdf5b8f","ruler_category":"aggregation","seed":12340,"selection_rank":11,"selection_sha256":"a4f7cd62de6cc19b84e61c4bbedb2f095811b5b15e0045c070a67a003ff2110a","sequence_length":1862,"sequence_token_ids_sha256":"52bbc9b8942f395aaf5ad8a6b712b521a6a64d665cf085cb5fba76caccdf5b8f","source_content_sha256":"1c23f946536f8b05ad81d768edf4b1a5cc74ce39a3e9048124593635eb45ad57","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1862,"cache_exposed_stop":1862,"prefill_start":0,"prefill_stop":1862,"scored_start":1862,"scored_stop":1862},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"134c9730064b61e43d80673f0cb931058bd1b3199bca91ad233138c0226ce11c","anchor_positions":[120,242,364,486,608,730,852,974,1096,1218,1340,1462,1584,1706,1828,1950],"anchor_positions_sha256":"1e64ef366c8307e3a3410c30274175d81a4400831d9e53231d196d85e95f722d","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:fwe:length=2048:seed=12339:sample=0","config":"fwe","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[120,242,364,486,608,729,851,973,1095,1217,1338,1460,1582,1704,1826,1948],"fisher_boundary_sha256":"6f240a219eecca56c401dc3488a6be62114cf717fb5d3354698e9b9bd01cb16c","horizon":1,"input_positions":[121,243,365,487,609,730,852,974,1096,1218,1339,1461,1583,1705,1827,1949],"input_token_ids_sha256":"cb97fb7a28acb3307ed6ee8d8c153283de017a8ff009aa8791bb6bcb457ee5f6","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[122,244,366,488,610,731,853,975,1097,1219,1340,1462,1584,1706,1828,1950],"target_token_ids_sha256":"317374ba87916932cb7eb42091c9f2e9f86931a246150c2f199b1253ca4deba1"},"formatted_content_sha256":"6133db426215f641d319c7f7a251c50725f112b597bacfa38b5b397f559b9bc6","generator_receipt_sha256":"bb2518d8cd4ac6e41fa7aafc28a2951da762de64e8c2843d63b20f731b44ed69","identity_record_sha256":"fda1b358c53c26d14ec53f9f1307c3a894b7bd9b2265f34dc4835ad657df108d","prompt_token_ids_sha256":"5dc1507b9a08d033e3d20d61fa912a594f04959a9046d7a45a38e8828444ba30","ruler_category":"aggregation","seed":12339,"selection_rank":12,"selection_sha256":"a802c5ae87d368a5b0da4639128597266434ded4c54487021d47309dd1c7d305","sequence_length":1951,"sequence_token_ids_sha256":"5dc1507b9a08d033e3d20d61fa912a594f04959a9046d7a45a38e8828444ba30","source_content_sha256":"bb2518d8cd4ac6e41fa7aafc28a2951da762de64e8c2843d63b20f731b44ed69","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1951,"cache_exposed_stop":1951,"prefill_start":0,"prefill_stop":1951,"scored_start":1951,"scored_stop":1951},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3a9bc7a5fff3d80eb04fe1bfebb5c1e9543f99c72342a46af4069af857693b87","anchor_positions":[222,446,669,893,1116,1340,1563,1787,2010,2234,2457,2681,2904,3128,3351,3575],"anchor_positions_sha256":"f02061347ae4b281dab9fe2e3493b73bfb716ffe8fbf61cda899a3009674e03c","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:cwe:length=4096:seed=12340:sample=0","config":"cwe","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[222,445,669,892,1115,1339,1562,1786,2009,2232,2456,2679,2902,3126,3349,3573],"fisher_boundary_sha256":"44f51387dfbe3723bf9be8b472cc53ae9a75aa3358aacbe82ce774bca098b167","horizon":1,"input_positions":[223,446,670,893,1116,1340,1563,1787,2010,2233,2457,2680,2903,3127,3350,3574],"input_token_ids_sha256":"7d7dec828b0e79f43e590036098693e354e401b432f63c491d9b74ef31439c57","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[224,447,671,894,1117,1341,1564,1788,2011,2234,2458,2681,2904,3128,3351,3575],"target_token_ids_sha256":"8a65c1ab28b83d19798e54204b1019dbe72fa0be387f3d6e56ba27d82d13e7fa"},"formatted_content_sha256":"bbb3633bd44e955d1f33e80a2c1f71f0ae0e778f671a8a282b12b48eb76e3e53","generator_receipt_sha256":"3a6698b2ea721d388d72f28e1c4d4646616d6952fe01ed8038c01dc53dee3a72","identity_record_sha256":"cad2c6009d9d8080acff02b0fc66babc81f232bac219039d597c2889f240a957","prompt_token_ids_sha256":"679f814d93763a3c0db75fbc0cbf70d684bfd0ccde3a62fe15ff9d09dbc1a014","ruler_category":"aggregation","seed":12340,"selection_rank":13,"selection_sha256":"e09c4998af87e1e33e97059c684d3b0558e6e1e107295e2ccf3655bc3d70e7c8","sequence_length":3576,"sequence_token_ids_sha256":"679f814d93763a3c0db75fbc0cbf70d684bfd0ccde3a62fe15ff9d09dbc1a014","source_content_sha256":"3a6698b2ea721d388d72f28e1c4d4646616d6952fe01ed8038c01dc53dee3a72","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":3576,"cache_exposed_stop":3576,"prefill_start":0,"prefill_stop":3576,"scored_start":3576,"scored_stop":3576},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"8046d74f592f47985b3070756d1e18a618db723abbd079cab7a5a30d4efcf596","anchor_positions":[124,250,375,501,626,752,877,1003,1129,1254,1380,1505,1631,1756,1882,2008],"anchor_positions_sha256":"d296c9d479bcd5abca26602f7c8ff6f9e1668dd7bcce7512dc2b7a9f6612803c","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=2048:seed=12340:sample=0","config":"vt","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[124,249,375,500,626,751,877,1002,1127,1253,1378,1504,1629,1755,1880,2006],"fisher_boundary_sha256":"bddd9058cc4355edbe7a8c638de06d849c2134d8813b5401ee6957f2d201a918","horizon":1,"input_positions":[125,250,376,501,627,752,878,1003,1128,1254,1379,1505,1630,1756,1881,2007],"input_token_ids_sha256":"56e2b2e1f3b426b0a06e44e2cbb64564b09850f5d87d64bf62b5cea5466bcee1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[126,251,377,502,628,753,879,1004,1129,1255,1380,1506,1631,1757,1882,2008],"target_token_ids_sha256":"bd6a1f6ec03f5d3adcfb37a0b1ac11dadaa147d56888994f667aa7841dc1257b"},"formatted_content_sha256":"b17100d16d1771ca1122c2b9937cacb984da6216ccf38bef7be0228979e43248","generator_receipt_sha256":"e8b50ab62299b2f7306b66c83a8c6f2b471e1198bc02aba090f1ab406da24110","identity_record_sha256":"ab943760500ed5247a613c9eb87ac98c621b16c8919c9b1bcf5bebb2f1da258f","prompt_token_ids_sha256":"c6dfdc419e73d825ef29b21a8e2369076e96964690bfab85b8262a2299d05ce2","ruler_category":"multi_hop_tracing","seed":12340,"selection_rank":14,"selection_sha256":"f833f67c79e0d9e87d7d872ac8220d32c02483b56c47dbcd2385976fff75fe5f","sequence_length":2009,"sequence_token_ids_sha256":"c6dfdc419e73d825ef29b21a8e2369076e96964690bfab85b8262a2299d05ce2","source_content_sha256":"e8b50ab62299b2f7306b66c83a8c6f2b471e1198bc02aba090f1ab406da24110","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2009,"cache_exposed_stop":2009,"prefill_start":0,"prefill_stop":2009,"scored_start":2009,"scored_stop":2009},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"163e62428e8cb8f598c255e524c87ac6fe75f5451263fe61445eb8cafaa35b23","anchor_positions":[246,493,740,987,1234,1482,1729,1976,2223,2470,2718,2965,3212,3459,3706,3954],"anchor_positions_sha256":"c09190aec7245025981cb99d1fb2f1b133b5cd4db4f1f1baa5e107f0f2266b39","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_single_1:length=4096:seed=12339:sample=0","config":"niah_single_1","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[246,493,740,987,1234,1481,1728,1975,2222,2469,2716,2963,3210,3457,3704,3952],"fisher_boundary_sha256":"5d449cbf648406313b9dd1abfb6d827d86ea00767627eae59437398324207334","horizon":1,"input_positions":[247,494,741,988,1235,1482,1729,1976,2223,2470,2717,2964,3211,3458,3705,3953],"input_token_ids_sha256":"bc6efe8e70dc84d76b6d63607de536e9dad39e3e692c1153b17d01d1f948f5b4","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[248,495,742,989,1236,1483,1730,1977,2224,2471,2718,2965,3212,3459,3706,3954],"target_token_ids_sha256":"8b9cd390ee896f79de83be50c222152af4aff55c3013e4e55bd32df875a3182e"},"formatted_content_sha256":"5044e6b8cb4a5c93592f73b69fc42041fad9c54b2ba7d53216fc486266f9f8ba","generator_receipt_sha256":"bd8f1387bc9aca3118ced5d9287c08f72c670bfbe6ef88729d36d6940cc2a22f","identity_record_sha256":"9209f85027779d40b0ad2a15080aef80e042f89c3c96760320f1648c2f18b461","prompt_token_ids_sha256":"786505e0d874cafdc0c5a41e3e0922fe96e048b0fb0ed0d4c098605d9bf54be8","ruler_category":"retrieval","seed":12339,"selection_rank":15,"selection_sha256":"ffbb6f30b667ab8eb83eff25cc9f46717b4a36965d1f9c15e08b6633fe46e13d","sequence_length":3955,"sequence_token_ids_sha256":"786505e0d874cafdc0c5a41e3e0922fe96e048b0fb0ed0d4c098605d9bf54be8","source_content_sha256":"bd8f1387bc9aca3118ced5d9287c08f72c670bfbe6ef88729d36d6940cc2a22f","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":3955,"cache_exposed_stop":3955,"prefill_start":0,"prefill_stop":3955,"scored_start":3955,"scored_stop":3955},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"}],"resolver_version":6,"schema_version":5,"selection":{"humaneval_plus_stage_a_b_namespace":"recurquant.experiment013.humaneval-plus.stage-a-b.v1\u0000","humaneval_plus_stage_c_namespace":"recurquant.experiment013.humaneval-plus.stage-c.v1\u0000","pg19_test_namespace":"recurquant.experiment013.pg19.test.v1\u0000","pg19_train_namespace":"recurquant.experiment013.pg19.train.v1\u0000","pg19_validation_namespace":"recurquant.experiment013.pg19.validation.v1\u0000"},"source_manifest_sha256":"fcdfca04cf2de0d6beef8ae6418bda13fe9db033d839b139bbd788843f827c30","status":"frozen","tokenizer":{"class":"Qwen2Tokenizer","file_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef","files":[{"name":"merges.txt","sha256":"a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d","size_bytes":3353259},{"name":"tokenizer.json","sha256":"fe000e3ed39ed12b8d2481d527d44f93c65d37e87645d2dcc80d1bf9d50d2927","size_bytes":12807196},{"name":"tokenizer_config.json","sha256":"e611fbccc7c29ef3b1cafb1cb7ea548d189968632901d678fd62be68c47885de","size_bytes":16712},{"name":"vocab.json","sha256":"ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003","size_bytes":6722759}],"revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68","source_id":"Qwen/Qwen3.5-0.8B-Base","transformers_version":"5.14.1"},"upstream_tool_contracts":{"evalplus_formatter":{"id":"evalplus/evalplus","revision":"26d6d00bb1fd0fa37f39c99d5290da67891d1c5e"},"ruler_generator":{"id":"NVIDIA/RULER","revision":"c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a"}}}} From 85625a5c4e4d7c6d1b015c0f3cccffea5c3d71c3 Mon Sep 17 00:00:00 2001 From: HiddenTrojan Date: Sat, 15 Aug 2026 01:53:05 +0800 Subject: [PATCH 18/24] Repair Experiment 013 identity authorization contract --- ...EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md | 106 ++- scripts/run_static_q468_calibration.py | 226 ++++- tests/test_run_static_q468_calibration.py | 834 +++++++++++++++++- 3 files changed, 1119 insertions(+), 47 deletions(-) diff --git a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md index 1ee621f..92692cd 100644 --- a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md +++ b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md @@ -1,22 +1,24 @@ # Experiment 013: static RHT-Q468 packed-native adoption protocol -> **Status: pre-freeze H0 candidate; not yet preregistered.** +> **Status: replacement-H0 candidate after a preserved pre-model authorization +> failure; not yet re-preregistered.** > -> This working copy becomes the frozen Experiment 013 preregistration only when -> its exact bytes and dependencies are committed in a clean source commit H0 and -> that H0 is bound by the pre-access seal before identity resolution, policy -> fitting, any protected materialization, model-weight loading, or quality -> measurement. A dirty or unbound working copy is not a frozen protocol. +> This replacement working copy becomes the next frozen Experiment 013 +> preregistration only when its exact bytes and dependencies are committed in a +> clean source commit H0 and that H0 is bound before any further identity +> resolution, policy fitting, protected materialization, model-weight loading, +> or quality measurement. A dirty or unbound working copy is not a frozen +> protocol. > > The amendment history is retained below, but its rules remain candidate rules -> until that H0 binding. Canonical row identities, token spans, tokenizer-file -> hashes, and content hashes remain unresolved placeholders until a separate -> identity candidate is resolved and explicitly promoted. An identity candidate -> is not authorization to load model weights. +> until that H0 binding. Prior identities, token spans, tokenizer-file hashes, +> and content hashes remain preserved as superseded evidence under the disclosed +> reuse rule; the replacement identity has not yet been resolved or promoted. +> An identity candidate is not authorization to load model weights. Protocol draft initiated: 2026-08-02 -Current pre-H0 candidate amended: 2026-08-14 +Current replacement-H0 candidate amended: 2026-08-15 Pre-resolution audit amendment: 2026-08-02. The amendment corrects a cache-exposed-span off-by-one, binds the Stage-A calibration chain by exact @@ -267,6 +269,59 @@ interruption. This incident does not require new PG19 or HumanEval+ Stage-A identities: the complete-RULER verifier ran before the phase-specific PG19 and HumanEval+ reads. +Thirteenth pre-resolution execution-contract amendment: 2026-08-15. Under +source commit `447295e5f705a74a85ad74a74b68985914096357` and tag +`experiment013-h0-447295e`, the authenticated v7 RULER batch was produced and a +calibration identity was promoted at the identity-only descendant +`de4b8d8b514a331bcc8f4ab5b039f4c2b12473ef`. The first `stage-model` +authorization stopped before cache or output creation, model-metadata manifest +read, Hub import or download, model-payload access, adapter construction, model +loading, calibration, or quality measurement. The resolver correctly returned +recursively immutable sequence views, but calibration-runner v2 incorrectly +required mutable `list` values for the three Fisher-boundary position arrays. +This was a producer-to-consumer execution-contract defect, not an experimental +result. Both commits and frozen identity file SHA-256 +`9ad6afe4a8513b8cc0cd467e75cb23bea72ee3be85c8aeebb8ed3b6e7772f260` +remain unchanged as superseded evidence and authorize no further model staging +or experiment execution. + +Calibration-runner v3 accepts an authenticated non-text sequence for each +position array, still requires a nonempty sequence of exact nonnegative +integers, and immediately normalizes it to an ordinary list before the existing +evidence-equality, length, `H=1`, exact `B(T)`, and self-hash checks. The repair +does not change resolver immutability, frozen JSON, record selection, token +span, Fisher boundary, quantization policy, metric, or gate. A producer-to- +consumer regression uses the resolver's recursively frozen DTO, while text, +bytes, empty, Boolean, and negative-position variants fail closed. Runner +revision advances from v2 to v3. Identity schema v5, capture and resolver +procedure v6, RULER launcher v7, RULER generation-manifest v2, RULER runtime- +manifest v3, and source-manifest schema/profile v2 remain unchanged. The new +frozen-identity-contract and model-staging-authorization canonical stdout +documents both use schema version one. A new clean H0 and source manifest plus a +newly promoted H1 are required; the old commits must not be amended, moved, or +relabelled. + +The replacement identity chain may reuse only the already fixed v7 RULER batch +whose generation-manifest file SHA-256 is exactly +`979f91848b6c0692160419c3e5e9ee555aa94d9e7add3092067f003ea0543e80`. +Regeneration is not required for this runner-only defect and would not restore +blindness after deterministic calibration materialization; exact replay +authentication is the relevant integrity check. Before reuse, exactly 20 +receipt files plus one generation manifest and 20 raw sibling roots containing +100 raw files must be replay-authenticated without modification. The generator, +capture, resolver, RULER requirements, launcher revision, and their relevant +source blobs must be byte-identical to the batch-producing H0, including Git +blob OIDs `b981f693a248dbe870d27bb1d5d22a8fb09042c2` for the generator, +`43e64f3f4f72256de8eb58f3f4cd9068ef3fe305` for capture, +`dd579415f694d5900e1abdc0f46af358b2a8628b` for the resolver, and +`680c107636cc27be06652b2cfea18e0c0b82df0b` for the RULER requirements. The +replacement identity must match the superseded identity's content-manifest +commitment, records, datasets, selection, calibration split halves, tokenizer and model +contracts, and upstream revisions; only the H0/source-binding and consequent +promotion-hash cascade may differ. Any inventory, byte, hash, version, or +semantic-identity mismatch stops reuse and requires a separately preregistered +fresh complete batch after the replacement H0. + ## Question Can a calibration-frozen, static Q4/Q6/Q8 recurrent-state layout satisfy the @@ -658,6 +713,35 @@ tree, H1/index, and worktree bytes, and the worktree is otherwise clean. Reports and policy artifacts continue to record H0 as implementation provenance; H1 is the identity authorization commit and may not be relabelled as source commit. +Before committing H1, the exact promoted identity bytes in their ignored, +no-overwrite precommit location must pass `verify-frozen-identity-contract`. +That read-only command authenticates H0 and its source manifest, loads the exact +H0 resolver, and consumes the complete record inventory through calibration- +runner v3's identity view. It accepts no H1, model manifest, Hub, cache, or +output argument. Its non-persisted canonical JSON stdout document uses artifact +kind `recurquant_experiment013_frozen_identity_contract_verification`, schema +version one, and binds the H0/source contract, portable Git identity, all four +execution bindings, complete identity/canonical/assignment hashes, public model +and tokenizer contracts, and record count. The bytes that passed are then copied +without modification as the sole H1 tree change; regeneration or hand editing +after that preflight is forbidden. After H1, the read-only command +`verify-model-staging-authorization` must invoke the same authorization path +used by `stage-model` and reauthenticate the H1, index, and +worktree identity bytes; H0 ancestry and unchanged source tree; the complete +frozen identity and execution bindings; and the exact public model-metadata +manifest. The command accepts no cache or output root, imports no Hub downloader, +downloads no file, and creates no directory or artifact. Its canonical JSON +stdout document, which the command does not persist, uses artifact kind +`recurquant_experiment013_model_staging_authorization`, schema version one, and +binds status, runner revision, frozen-identity hash, H1, H0, repository-source- +manifest hash, model-manifest hash, public model ID/revision, Hub-tree-manifest +hash, file count, and total bytes. Only a successful stdout document permits +`stage-model` to be attempted. A semantic or authentication mismatch retires +that H1; it may not be hand-edited or weakened. An argument-parse failure does +not consume H1 because authentication did not begin. A documented infrastructure +interruption before any model-payload access permits only an exact same-command +retry under that H1. + Model payload staging begins only after the frozen identity is tracked with identical H1, index, and worktree bytes. The identity-bound stager downloads only the exact sorted root files in the frozen model manifest at the exact diff --git a/scripts/run_static_q468_calibration.py b/scripts/run_static_q468_calibration.py index e04ac50..9e7b599 100644 --- a/scripts/run_static_q468_calibration.py +++ b/scripts/run_static_q468_calibration.py @@ -56,7 +56,7 @@ CANONICAL_ADAPTER_MODULE: Final = "recurquant.experiment013_qwen35_adapter" CANONICAL_ADAPTER_PATH: Final = "src/recurquant/experiment013_qwen35_adapter.py" -RUNNER_REVISION: Final = "experiment-013-static-q468-calibration-runner-v2" +RUNNER_REVISION: Final = "experiment-013-static-q468-calibration-runner-v3" FROZEN_IDENTITY_SCHEMA_VERSION: Final = 5 FISHER_BOUNDARY_SCHEMA: Final = "recurquant.experiment013.fisher-boundary.v1" FISHER_BOUNDARY_NAMESPACE: Final = b"recurquant.experiment013.fisher-boundary.v1\0" @@ -75,6 +75,12 @@ MODEL_FILE_MANIFEST_SCHEMA: Final = 1 MODEL_FILE_MANIFEST_DERIVATION: Final = "huggingface-hub-pinned-tree-lfs-v1" MODEL_FILE_SELECTION_PROFILE: Final = "qwen35-config-index-safetensors-v1" +FROZEN_IDENTITY_CONTRACT_KIND: Final = ( + "recurquant_experiment013_frozen_identity_contract_verification" +) +FROZEN_IDENTITY_CONTRACT_SCHEMA: Final = 1 +MODEL_STAGING_AUTHORIZATION_KIND: Final = "recurquant_experiment013_model_staging_authorization" +MODEL_STAGING_AUTHORIZATION_SCHEMA: Final = 1 RUNTIME_MANIFEST_KIND: Final = "recurquant_experiment013_calibration_runtime_manifest" RUNTIME_MANIFEST_SCHEMA: Final = 4 RUN_REPORT_KIND: Final = "recurquant_experiment013_calibration_run" @@ -759,6 +765,15 @@ class ModelStagingAuthorization: source_commit: str +@dataclass(frozen=True, slots=True) +class FrozenIdentitySourceAuthorization: + identity: FrozenCalibrationIdentity + bindings: BootstrapIdentityBindings + identity_bytes: bytes + frozen_identity_file_sha256: str + source_commit: str + + @dataclass(frozen=True, slots=True) class RuntimeFileRecord: path: str @@ -1573,19 +1588,16 @@ def _verify_committed_frozen_identity( return actual_head -def _authenticate_model_staging_authorization( +def _authenticate_frozen_identity_source_contract( *, git_executable: AuthenticatedGitExecutable, frozen_identity_path: Path, expected_frozen_identity_sha256: str, - identity_commit: str, repository_root: Path, repository_source_manifest_path: Path, source_commit: str, - model_file_manifest_path: Path, - expected_model_file_manifest_sha256: str, -) -> ModelStagingAuthorization: - """Authenticate promotion, committed provenance, source, and model metadata.""" +) -> FrozenIdentitySourceAuthorization: + """Authenticate one promoted identity against its exact H0 source contract.""" expected_identity_sha256 = _sha256( expected_frozen_identity_sha256, @@ -1628,7 +1640,7 @@ def _authenticate_model_staging_authorization( git_executable=git_executable.path, ) if verified_source != bootstrap_source.manifest: - raise CalibrationRunError("source verifier returned different model-staging evidence") + raise CalibrationRunError("source verifier returned different frozen-identity evidence") if verified_source.get("source_commit") != requested_source_commit: raise CalibrationRunError( "verified source-manifest commit differs from requested frozen source commit" @@ -1658,11 +1670,97 @@ def _authenticate_model_staging_authorization( ): raise CalibrationRunError("full frozen identity differs from its bootstrap bindings") + return FrozenIdentitySourceAuthorization( + identity=identity, + bindings=bindings, + identity_bytes=identity_bytes, + frozen_identity_file_sha256=expected_identity_sha256, + source_commit=requested_source_commit, + ) + + +def verify_frozen_identity_contract( + *, + git_executable_path: Path | None = None, + frozen_identity_path: Path, + expected_frozen_identity_sha256: str, + repository_root: Path, + repository_source_manifest_path: Path, + source_commit: str, +) -> dict[str, object]: + """Verify a promoted identity against H0 without writes or model access.""" + + git_executable = _authenticate_git_executable(git_executable_path) + authorization = _authenticate_frozen_identity_source_contract( + git_executable=git_executable, + frozen_identity_path=frozen_identity_path, + expected_frozen_identity_sha256=expected_frozen_identity_sha256, + repository_root=repository_root, + repository_source_manifest_path=repository_source_manifest_path, + source_commit=source_commit, + ) + identity = authorization.identity + bindings = authorization.bindings + return { + "artifact_kind": FROZEN_IDENTITY_CONTRACT_KIND, + "assignment_sha256": identity.assignment_sha256, + "canonical_evidence_sha256": identity.canonical_evidence_sha256, + "execution_bindings": { + "calibration_runtime_manifest_file_sha256": bindings.runtime_manifest_file_sha256, + "model_file_manifest_file_sha256": bindings.model_file_manifest_file_sha256, + "parquet_materialization_manifest_file_sha256": ( + bindings.parquet_materialization_manifest_file_sha256 + ), + "repository_source_manifest_file_sha256": ( + bindings.repository_source_manifest_file_sha256 + ), + }, + "frozen_identity_file_sha256": authorization.frozen_identity_file_sha256, + "git_executable": { + "sha256": git_executable.sha256, + "size_bytes": git_executable.size_bytes, + }, + "identity_input_manifest_sha256": identity.identity_input_manifest_sha256, + "model_id": identity.model_id, + "model_revision": identity.model_revision, + "record_count": len(identity.records), + "runner_revision": RUNNER_REVISION, + "schema_version": FROZEN_IDENTITY_CONTRACT_SCHEMA, + "source_commit": authorization.source_commit, + "status": "verified_frozen_identity_contract", + "tokenizer_manifest_sha256": identity.tokenizer_manifest_sha256, + "transformers_version": identity.transformers_version, + } + + +def _authenticate_model_staging_authorization( + *, + git_executable: AuthenticatedGitExecutable, + frozen_identity_path: Path, + expected_frozen_identity_sha256: str, + identity_commit: str, + repository_root: Path, + repository_source_manifest_path: Path, + source_commit: str, + model_file_manifest_path: Path, + expected_model_file_manifest_sha256: str, +) -> ModelStagingAuthorization: + """Authenticate promotion, committed provenance, source, and model metadata.""" + + source_authorization = _authenticate_frozen_identity_source_contract( + git_executable=git_executable, + frozen_identity_path=frozen_identity_path, + expected_frozen_identity_sha256=expected_frozen_identity_sha256, + repository_root=repository_root, + repository_source_manifest_path=repository_source_manifest_path, + source_commit=source_commit, + ) + committed_at = _verify_committed_frozen_identity( git_executable, repository_root, frozen_identity_path, - identity_bytes, + source_authorization.identity_bytes, identity_commit=identity_commit, ) model_manifest_bytes = _read_stable_regular_bytes( @@ -1676,20 +1774,66 @@ def _authenticate_model_staging_authorization( actual_model_sha256 = sha256_bytes(model_manifest_bytes) if ( actual_model_sha256 != expected_model_sha256 - or actual_model_sha256 != bindings.model_file_manifest_file_sha256 + or actual_model_sha256 != source_authorization.bindings.model_file_manifest_file_sha256 ): raise CalibrationRunError( "model file manifest differs from the frozen identity/CLI binding" ) model_manifest = parse_model_file_manifest(model_manifest_bytes) - _model_contract_matches(identity, model_manifest) + _model_contract_matches(source_authorization.identity, model_manifest) return ModelStagingAuthorization( - identity=identity, + identity=source_authorization.identity, model_manifest=model_manifest, - frozen_identity_file_sha256=expected_identity_sha256, + frozen_identity_file_sha256=source_authorization.frozen_identity_file_sha256, identity_commit=committed_at, - source_commit=requested_source_commit, + source_commit=source_authorization.source_commit, + ) + + +def verify_identity_bound_model_staging_authorization( + *, + git_executable_path: Path | None = None, + frozen_identity_path: Path, + expected_frozen_identity_sha256: str, + identity_commit: str, + repository_root: Path, + repository_source_manifest_path: Path, + source_commit: str, + model_file_manifest_path: Path, + expected_model_file_manifest_sha256: str, +) -> dict[str, object]: + """Verify model-staging authorization without accessing model payloads.""" + + git_executable = _authenticate_git_executable(git_executable_path) + authorization = _authenticate_model_staging_authorization( + git_executable=git_executable, + frozen_identity_path=frozen_identity_path, + expected_frozen_identity_sha256=expected_frozen_identity_sha256, + identity_commit=identity_commit, + repository_root=repository_root, + repository_source_manifest_path=repository_source_manifest_path, + source_commit=source_commit, + model_file_manifest_path=model_file_manifest_path, + expected_model_file_manifest_sha256=expected_model_file_manifest_sha256, ) + return { + "artifact_kind": MODEL_STAGING_AUTHORIZATION_KIND, + "file_count": len(authorization.model_manifest.files), + "frozen_identity_file_sha256": authorization.frozen_identity_file_sha256, + "hub_tree_manifest_sha256": authorization.model_manifest.hub_tree_manifest_sha256, + "identity_commit": authorization.identity_commit, + "model_id": authorization.model_manifest.model_id, + "model_manifest_file_sha256": authorization.model_manifest.file_sha256, + "revision": authorization.model_manifest.revision, + "repository_source_manifest_file_sha256": ( + authorization.identity.repository_source_manifest_file_sha256 + ), + "runner_revision": RUNNER_REVISION, + "schema_version": MODEL_STAGING_AUTHORIZATION_SCHEMA, + "source_commit": authorization.source_commit, + "status": "verified_identity_bound_model_staging_authorization", + "total_size_bytes": sum(item.size_bytes for item in authorization.model_manifest.files), + } def _path_is_within(path: Path, root: Path) -> bool: @@ -3635,7 +3779,8 @@ def _identity_records_with_fisher_boundary( for name in ("boundary_positions", "input_positions", "target_positions"): values = decoded_boundary[name] if ( - not isinstance(values, list) + isinstance(values, (str, bytes, bytearray)) + or not isinstance(values, Sequence) or not values or any(type(value) is not int or value < 0 for value in values) ): @@ -3643,6 +3788,8 @@ def _identity_records_with_fisher_boundary( f"schema-v5 records[{index}].fisher_boundary {name} is invalid" ) positions[name] = list(values) + normalized_boundary = dict(decoded_boundary) + normalized_boundary.update(positions) if not ( len(positions["boundary_positions"]) == len(positions["input_positions"]) @@ -3681,18 +3828,16 @@ def _identity_records_with_fisher_boundary( f"schema-v5 records[{index}].fisher_boundary hash is invalid" ) from exc boundary_payload = { - name: decoded_boundary[name] + name: normalized_boundary[name] for name in FISHER_BOUNDARY_FIELDS - {"fisher_boundary_sha256"} } - if decoded_boundary["fisher_boundary_sha256"] != sha256_bytes( + if normalized_boundary["fisher_boundary_sha256"] != sha256_bytes( FISHER_BOUNDARY_NAMESPACE + canonical_json_bytes(boundary_payload) ): raise CalibrationRunError( f"schema-v5 records[{index}].fisher_boundary self-hash drifted" ) - normalized_boundary = dict(decoded_boundary) - normalized_boundary.update(positions) normalized_record = dict(decoded_record) normalized_record["fisher_boundary"] = normalized_boundary normalized_records.append(normalized_record) @@ -5434,6 +5579,8 @@ def _capture_manifest_mode(arguments: Sequence[str]) -> int | None: "capture-model-manifest", "prepare-runtime", "stage-model", + "verify-frozen-identity-contract", + "verify-model-staging-authorization", }: return None command = arguments[0] @@ -5447,7 +5594,14 @@ def _capture_manifest_mode(arguments: Sequence[str]) -> int | None: "--package-root-name", default=DEFAULT_PACKAGE_RUNTIME_ROOT_NAME, ) - elif command == "stage-model": + elif command == "verify-frozen-identity-contract": + parser.add_argument("--git-executable", required=True, type=Path) + parser.add_argument("--frozen-identity", required=True, type=Path) + parser.add_argument("--expected-frozen-identity-sha256", required=True) + parser.add_argument("--repository-root", required=True, type=Path) + parser.add_argument("--repository-source-manifest", required=True, type=Path) + parser.add_argument("--source-commit", required=True) + elif command in {"stage-model", "verify-model-staging-authorization"}: parser.add_argument("--git-executable", required=True, type=Path) parser.add_argument("--frozen-identity", required=True, type=Path) parser.add_argument("--expected-frozen-identity-sha256", required=True) @@ -5457,9 +5611,10 @@ def _capture_manifest_mode(arguments: Sequence[str]) -> int | None: parser.add_argument("--source-commit", required=True) parser.add_argument("--model-file-manifest", required=True, type=Path) parser.add_argument("--expected-model-file-manifest-sha256", required=True) - parser.add_argument("--hub-cache-root", required=True, type=Path) - parser.add_argument("--output-root", required=True, type=Path) - parser.add_argument("--local-files-only", action="store_true") + if command == "stage-model": + parser.add_argument("--hub-cache-root", required=True, type=Path) + parser.add_argument("--output-root", required=True, type=Path) + parser.add_argument("--local-files-only", action="store_true") else: parser.add_argument("--output", required=True, type=Path) if command == "capture-source-manifest": @@ -5503,6 +5658,31 @@ def _capture_manifest_mode(arguments: Sequence[str]) -> int | None: ) print(json.dumps(details, sort_keys=True)) return 0 + if command == "verify-frozen-identity-contract": + details = verify_frozen_identity_contract( + git_executable_path=args.git_executable, + frozen_identity_path=args.frozen_identity, + expected_frozen_identity_sha256=args.expected_frozen_identity_sha256, + repository_root=args.repository_root, + repository_source_manifest_path=args.repository_source_manifest, + source_commit=args.source_commit, + ) + print(canonical_json_bytes(details).decode("utf-8"), end="") + return 0 + if command == "verify-model-staging-authorization": + details = verify_identity_bound_model_staging_authorization( + git_executable_path=args.git_executable, + frozen_identity_path=args.frozen_identity, + expected_frozen_identity_sha256=args.expected_frozen_identity_sha256, + identity_commit=args.identity_commit, + repository_root=args.repository_root, + repository_source_manifest_path=args.repository_source_manifest, + source_commit=args.source_commit, + model_file_manifest_path=args.model_file_manifest, + expected_model_file_manifest_sha256=args.expected_model_file_manifest_sha256, + ) + print(canonical_json_bytes(details).decode("utf-8"), end="") + return 0 status = "captured_metadata_only" details: dict[str, object] = {} if command == "capture-source-manifest": diff --git a/tests/test_run_static_q468_calibration.py b/tests/test_run_static_q468_calibration.py index 102d527..a3c3697 100644 --- a/tests/test_run_static_q468_calibration.py +++ b/tests/test_run_static_q468_calibration.py @@ -1,5 +1,6 @@ from __future__ import annotations +import builtins import hashlib import importlib.util import json @@ -229,6 +230,24 @@ def model_staging_authorization( ) +def frozen_identity_source_authorization() -> Any: + frozen = identity(()) + return runner.FrozenIdentitySourceAuthorization( + identity=frozen, + bindings=runner.BootstrapIdentityBindings( + repository_source_manifest_file_sha256=(frozen.repository_source_manifest_file_sha256), + runtime_manifest_file_sha256=frozen.runtime_manifest_file_sha256, + model_file_manifest_file_sha256=frozen.model_file_manifest_file_sha256, + parquet_materialization_manifest_file_sha256=( + frozen.parquet_materialization_manifest_file_sha256 + ), + ), + identity_bytes=b"frozen-identity", + frozen_identity_file_sha256="d" * 64, + source_commit="1" * 40, + ) + + def runtime_manifest_bytes() -> bytes: git_executable = runner._authenticate_git_executable(None) interpreter_sha256, interpreter_size = runner._stream_file_sha256( @@ -947,7 +966,7 @@ def test_identity_view_consumes_schema_v5_bindings_and_preserves_fisher_boundary "repository_source_manifest_file_sha256": "3" * 64, } boundary = fisher_boundary_contract() - decoded_record = { + evidence_record = { "canonical_id": "item-1", "fisher_boundary": boundary, "sequence_length": 3, @@ -955,27 +974,33 @@ def test_identity_view_consumes_schema_v5_bindings_and_preserves_fisher_boundary evidence = { "execution_bindings": bindings, "model_contracts": {"primary": {"id": "example/model", "revision": "4" * 40}}, - "records": [decoded_record], + "records": [evidence_record], "schema_version": runner.FROZEN_IDENTITY_SCHEMA_VERSION, "source_manifest_sha256": "5" * 64, "tokenizer": {"transformers_version": "5.14.1"}, } payload = runner.canonical_json_bytes({"evidence": evidence}) - - class Decoded: - file_sha256 = digest(payload) - canonical_evidence_sha256 = "6" * 64 - records = (decoded_record,) - assignment = () - assignment_sha256 = "7" * 64 - tokenizer_manifest_sha256 = "8" * 64 - execution_bindings = bindings + decoded_artifact = identity_resolver.FrozenCalibrationIdentityArtifact( + file_sha256=digest(payload), + canonical_evidence_sha256="6" * 64, + records=(evidence_record,), + assignment=(), + assignment_sha256="7" * 64, + tokenizer_manifest_sha256="8" * 64, + parquet_materialization_manifest_file_sha256="9" * 64, + execution_bindings=bindings, + ) + frozen_boundary = decoded_artifact.records[0]["fisher_boundary"] + assert isinstance(frozen_boundary, Mapping) + assert not isinstance(frozen_boundary["boundary_positions"], list) class Resolver: @staticmethod - def deserialize_frozen_calibration_identity_artifact(data: bytes) -> Decoded: + def deserialize_frozen_calibration_identity_artifact( + data: bytes, + ) -> identity_resolver.FrozenCalibrationIdentityArtifact: assert data == payload - return Decoded() + return decoded_artifact monkeypatch.setattr(runner, "_load_identity_resolver", lambda _root: Resolver()) decoded = runner._identity_view(payload, tmp_path) @@ -986,6 +1011,33 @@ def deserialize_frozen_calibration_identity_artifact(data: bytes) -> Decoded: assert decoded.parquet_materialization_manifest_file_sha256 == "9" * 64 assert decoded.records[0]["fisher_boundary"] == boundary assert decoded.records[0]["fisher_boundary"] is not boundary + for name in ("boundary_positions", "input_positions", "target_positions"): + assert isinstance(decoded.records[0]["fisher_boundary"][name], list) + + +@pytest.mark.parametrize( + "invalid_positions", + ["0", b"0", [], [True], [-1]], + ids=("text", "bytes", "empty", "boolean", "negative"), +) +def test_identity_view_rejects_invalid_fisher_boundary_position_sequences( + invalid_positions: object, +) -> None: + item = { + "canonical_id": "item-1", + "fisher_boundary": fisher_boundary_contract(), + "sequence_length": 3, + } + item["fisher_boundary"]["boundary_positions"] = invalid_positions + + class Decoded: + records = (item,) + + with pytest.raises( + runner.CalibrationRunError, + match=r"fisher_boundary boundary_positions is invalid", + ): + runner._identity_records_with_fisher_boundary(Decoded(), {"records": [item]}) def test_identity_view_rejects_resolver_record_that_drops_fisher_boundary() -> None: @@ -1783,6 +1835,596 @@ def test_stage_model_authenticates_before_touching_downloader_cache_or_output( assert not output.exists() +def test_verify_frozen_identity_contract_is_deterministic_read_only_and_exactly_forwarded( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + authorization = frozen_identity_source_authorization() + git_executable = runner.AuthenticatedGitExecutable( + path=tmp_path / "git.exe", + absolute_path_sha256="a" * 64, + sha256="b" * 64, + size_bytes=1, + ) + calls: list[dict[str, object]] = [] + monkeypatch.setattr(runner, "_authenticate_git_executable", lambda _path: git_executable) + + def authenticate(**kwargs: object) -> runner.FrozenIdentitySourceAuthorization: + calls.append(dict(kwargs)) + return authorization + + monkeypatch.setattr(runner, "_authenticate_frozen_identity_source_contract", authenticate) + original_import = builtins.__import__ + + def guarded_import(name: str, *args: object, **kwargs: object) -> object: + if name == "huggingface_hub" or name.startswith("huggingface_hub."): + pytest.fail("frozen-identity preflight imported a Hugging Face client") + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", guarded_import) + untouched = tmp_path / "untouched" + arguments = { + "git_executable_path": tmp_path / "requested-git.exe", + "frozen_identity_path": untouched / "identity.json", + "expected_frozen_identity_sha256": "d" * 64, + "repository_root": untouched / "repository", + "repository_source_manifest_path": untouched / "source.json", + "source_commit": "1" * 40, + } + + first = runner.verify_frozen_identity_contract(**arguments) + second = runner.verify_frozen_identity_contract(**arguments) + + expected = { + "artifact_kind": runner.FROZEN_IDENTITY_CONTRACT_KIND, + "assignment_sha256": "f" * 64, + "canonical_evidence_sha256": "e" * 64, + "execution_bindings": { + "calibration_runtime_manifest_file_sha256": "8" * 64, + "model_file_manifest_file_sha256": "9" * 64, + "parquet_materialization_manifest_file_sha256": "a" * 64, + "repository_source_manifest_file_sha256": "7" * 64, + }, + "frozen_identity_file_sha256": "d" * 64, + "git_executable": { + "sha256": "b" * 64, + "size_bytes": 1, + }, + "identity_input_manifest_sha256": "1" * 64, + "model_id": "example/model", + "model_revision": "2" * 40, + "record_count": 0, + "runner_revision": runner.RUNNER_REVISION, + "schema_version": runner.FROZEN_IDENTITY_CONTRACT_SCHEMA, + "source_commit": "1" * 40, + "status": "verified_frozen_identity_contract", + "tokenizer_manifest_sha256": "c" * 64, + "transformers_version": "5.14.1", + } + assert first == expected + assert runner.canonical_json_bytes(first) == runner.canonical_json_bytes(second) + assert str(tmp_path).encode("utf-8") not in runner.canonical_json_bytes(first) + expected_call = { + "expected_frozen_identity_sha256": "d" * 64, + "frozen_identity_path": untouched / "identity.json", + "git_executable": git_executable, + "repository_root": untouched / "repository", + "repository_source_manifest_path": untouched / "source.json", + "source_commit": "1" * 40, + } + assert calls == [expected_call, expected_call] + assert not untouched.exists() + + +def test_verify_frozen_identity_contract_failure_creates_nothing( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + untouched = tmp_path / "untouched" + original_import = builtins.__import__ + + def guarded_import(name: str, *args: object, **kwargs: object) -> object: + if name == "huggingface_hub" or name.startswith("huggingface_hub."): + pytest.fail("failed frozen-identity preflight imported a Hugging Face client") + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", guarded_import) + monkeypatch.setattr( + runner, + "_authenticate_git_executable", + lambda _path: runner.AuthenticatedGitExecutable( + path=tmp_path / "git.exe", + absolute_path_sha256="a" * 64, + sha256="b" * 64, + size_bytes=1, + ), + ) + monkeypatch.setattr( + runner, + "_authenticate_frozen_identity_source_contract", + lambda **_kwargs: (_ for _ in ()).throw( + runner.CalibrationRunError("injected frozen-identity authorization failure") + ), + ) + + with pytest.raises( + runner.CalibrationRunError, + match="injected frozen-identity authorization failure", + ): + runner.verify_frozen_identity_contract( + git_executable_path=tmp_path / "git.exe", + frozen_identity_path=untouched / "identity.json", + expected_frozen_identity_sha256="d" * 64, + repository_root=untouched / "repository", + repository_source_manifest_path=untouched / "source.json", + source_commit="1" * 40, + ) + + assert not untouched.exists() + + +def test_frozen_identity_source_contract_consumes_real_resolver_frozen_sequences( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_manifest_bytes = b"authenticated-source-manifest" + source_manifest_sha256 = digest(source_manifest_bytes) + bindings = { + "calibration_runtime_manifest_file_sha256": "1" * 64, + "model_file_manifest_file_sha256": "2" * 64, + "parquet_materialization_manifest_file_sha256": "9" * 64, + "repository_source_manifest_file_sha256": source_manifest_sha256, + } + first_boundary = fisher_boundary_contract() + second_boundary = fisher_boundary_contract((1, 2, 3, 4)) + evidence_records = [ + { + "canonical_id": "item-1", + "fisher_boundary": first_boundary, + "sequence_length": 3, + }, + { + "canonical_id": "item-2", + "fisher_boundary": second_boundary, + "sequence_length": 4, + }, + ] + evidence = { + "execution_bindings": bindings, + "identity_only": True, + "model_contracts": {"primary": {"id": "example/model", "revision": "4" * 40}}, + "phase": "calibration", + "promotion_required": False, + "records": evidence_records, + "schema_version": runner.FROZEN_IDENTITY_SCHEMA_VERSION, + "source_manifest_sha256": "5" * 64, + "status": "frozen", + "tokenizer": {"transformers_version": "5.14.1"}, + } + identity_bytes = runner.canonical_json_bytes( + { + "canonical_evidence_sha256": digest(runner.canonical_json_bytes(evidence)), + "evidence": evidence, + } + ) + identity_sha256 = digest(identity_bytes) + decoded_artifact = identity_resolver.FrozenCalibrationIdentityArtifact( + file_sha256=identity_sha256, + canonical_evidence_sha256=digest(runner.canonical_json_bytes(evidence)), + records=tuple(evidence_records), + assignment=(), + assignment_sha256="7" * 64, + tokenizer_manifest_sha256="8" * 64, + parquet_materialization_manifest_file_sha256="9" * 64, + execution_bindings=bindings, + ) + for item in decoded_artifact.records: + frozen_boundary = item["fisher_boundary"] + assert isinstance(frozen_boundary, Mapping) + assert not isinstance(frozen_boundary["boundary_positions"], list) + + class Resolver: + @staticmethod + def deserialize_frozen_calibration_identity_artifact( + data: bytes, + *, + expected_file_sha256: str | None = None, + ) -> identity_resolver.FrozenCalibrationIdentityArtifact: + assert data == identity_bytes + assert expected_file_sha256 == identity_sha256 + return decoded_artifact + + source_commit = "a" * 40 + source_manifest = {"source_commit": source_commit} + + class SourceVerifier: + @staticmethod + def verify_experiment013_source_manifest( + manifest: Mapping[str, object], + *, + repo_root: Path, + git_executable: Path, + ) -> Mapping[str, object]: + assert manifest == source_manifest + assert repo_root == tmp_path / "repository" + assert git_executable == tmp_path / "git.exe" + return manifest + + identity_path = tmp_path / "identity.json" + source_path = tmp_path / "source.json" + identity_path.write_bytes(identity_bytes) + source_path.write_bytes(source_manifest_bytes) + bootstrap = runner.BootstrapSource( + manifest=source_manifest, + source_commit=source_commit, + entries={ + runner.SOURCE_VERIFIER_PATH: {"raw_sha256": "a" * 64}, + runner.IDENTITY_RESOLVER_SOURCE_PATH: {"raw_sha256": "b" * 64}, + }, + ) + monkeypatch.setattr( + runner, + "_bootstrap_source_manifest", + lambda data, *, repository_root, require_adapter: ( + bootstrap + if data == source_manifest_bytes + and repository_root == tmp_path / "repository" + and require_adapter is False + else pytest.fail("source bootstrap inputs drifted") + ), + ) + + def load_module( + _module_name: str, + relative_path: str, + *, + repository_root: Path, + entry: Mapping[str, object], + ) -> object: + assert repository_root == tmp_path / "repository" + assert entry == bootstrap.entries[relative_path] + if relative_path == runner.SOURCE_VERIFIER_PATH: + return SourceVerifier() + if relative_path == runner.IDENTITY_RESOLVER_SOURCE_PATH: + return Resolver() + pytest.fail(f"unexpected authenticated source module: {relative_path}") + + monkeypatch.setattr(runner, "_load_exact_source_module", load_module) + authorization = runner._authenticate_frozen_identity_source_contract( + git_executable=runner.AuthenticatedGitExecutable( + path=tmp_path / "git.exe", + absolute_path_sha256="a" * 64, + sha256="b" * 64, + size_bytes=1, + ), + frozen_identity_path=identity_path, + expected_frozen_identity_sha256=identity_sha256, + repository_root=tmp_path / "repository", + repository_source_manifest_path=source_path, + source_commit=source_commit, + ) + + assert len(authorization.identity.records) == 2 + for index, expected_boundary in enumerate((first_boundary, second_boundary)): + assert authorization.identity.records[index]["fisher_boundary"] == expected_boundary + for name in ("boundary_positions", "input_positions", "target_positions"): + assert isinstance( + authorization.identity.records[index]["fisher_boundary"][name], + list, + ) + + +@pytest.mark.parametrize( + "field", + [ + "repository_source_manifest_file_sha256", + "runtime_manifest_file_sha256", + "model_file_manifest_file_sha256", + "parquet_materialization_manifest_file_sha256", + ], +) +def test_frozen_identity_source_contract_rejects_each_full_binding_mismatch( + field: str, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + identity_bytes = b"identity" + source_manifest_bytes = b"source" + bindings = runner.BootstrapIdentityBindings( + repository_source_manifest_file_sha256=digest(source_manifest_bytes), + runtime_manifest_file_sha256="1" * 64, + model_file_manifest_file_sha256="2" * 64, + parquet_materialization_manifest_file_sha256="3" * 64, + ) + decoded_identity = identity( + (), + source_manifest_sha256=bindings.repository_source_manifest_file_sha256, + runtime_manifest_sha256=bindings.runtime_manifest_file_sha256, + model_manifest_sha256=bindings.model_file_manifest_file_sha256, + parquet_manifest_sha256=bindings.parquet_materialization_manifest_file_sha256, + ) + decoded_identity = replace(decoded_identity, **{field: "f" * 64}) + source_commit = "a" * 40 + source_manifest = {"source_commit": source_commit} + bootstrap = runner.BootstrapSource( + manifest=source_manifest, + source_commit=source_commit, + entries={ + runner.SOURCE_VERIFIER_PATH: {}, + runner.IDENTITY_RESOLVER_SOURCE_PATH: {}, + }, + ) + + class SourceVerifier: + @staticmethod + def verify_experiment013_source_manifest( + manifest: Mapping[str, object], + *, + repo_root: Path, + git_executable: Path, + ) -> Mapping[str, object]: + del repo_root, git_executable + return manifest + + monkeypatch.setattr( + runner, + "_read_stable_regular_bytes", + lambda path, *, context: ( + identity_bytes if context == "frozen identity" else source_manifest_bytes + ), + ) + monkeypatch.setattr(runner, "_bootstrap_identity_bindings", lambda _data: bindings) + monkeypatch.setattr(runner, "_bootstrap_source_manifest", lambda *_args, **_kwargs: bootstrap) + monkeypatch.setattr( + runner, + "_load_exact_source_module", + lambda _module_name, relative_path, **_kwargs: ( + SourceVerifier() if relative_path == runner.SOURCE_VERIFIER_PATH else object() + ), + ) + monkeypatch.setattr( + runner, + "_identity_view_from_resolver", + lambda *_args, **_kwargs: decoded_identity, + ) + + with pytest.raises( + runner.CalibrationRunError, + match="full frozen identity differs from its bootstrap bindings", + ): + runner._authenticate_frozen_identity_source_contract( + git_executable=runner.AuthenticatedGitExecutable( + path=tmp_path / "git.exe", + absolute_path_sha256="a" * 64, + sha256="b" * 64, + size_bytes=1, + ), + frozen_identity_path=tmp_path / "identity.json", + expected_frozen_identity_sha256=digest(identity_bytes), + repository_root=tmp_path / "repository", + repository_source_manifest_path=tmp_path / "source.json", + source_commit=source_commit, + ) + + +def test_model_staging_authorization_reuses_frozen_identity_source_contract( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + model_bytes = model_manifest_bytes(staged_model_files()) + source_authorization = frozen_identity_source_authorization() + source_authorization = replace( + source_authorization, + identity=identity((), model_manifest_sha256=digest(model_bytes)), + bindings=replace( + source_authorization.bindings, + model_file_manifest_file_sha256=digest(model_bytes), + ), + ) + git_executable = runner.AuthenticatedGitExecutable( + path=tmp_path / "git.exe", + absolute_path_sha256="a" * 64, + sha256="b" * 64, + size_bytes=1, + ) + common_calls: list[dict[str, object]] = [] + + def authenticate_common(**kwargs: object) -> runner.FrozenIdentitySourceAuthorization: + common_calls.append(dict(kwargs)) + return source_authorization + + monkeypatch.setattr( + runner, + "_authenticate_frozen_identity_source_contract", + authenticate_common, + ) + + def verify_commit( + received_git: runner.AuthenticatedGitExecutable, + repository_root: Path, + frozen_identity_path: Path, + identity_bytes: bytes, + *, + identity_commit: str, + ) -> str: + assert received_git == git_executable + assert repository_root == tmp_path / "repository" + assert frozen_identity_path == tmp_path / "identity.json" + assert identity_bytes == b"frozen-identity" + assert identity_commit == "3" * 40 + return identity_commit + + monkeypatch.setattr(runner, "_verify_committed_frozen_identity", verify_commit) + monkeypatch.setattr( + runner, + "_read_stable_regular_bytes", + lambda path, *, context: ( + model_bytes + if path == tmp_path / "model.json" and context == "model file manifest" + else pytest.fail("unexpected stable read outside common authorization") + ), + ) + result = runner._authenticate_model_staging_authorization( + git_executable=git_executable, + frozen_identity_path=tmp_path / "identity.json", + expected_frozen_identity_sha256="d" * 64, + identity_commit="3" * 40, + repository_root=tmp_path / "repository", + repository_source_manifest_path=tmp_path / "source.json", + source_commit="1" * 40, + model_file_manifest_path=tmp_path / "model.json", + expected_model_file_manifest_sha256=digest(model_bytes), + ) + + assert result.identity == source_authorization.identity + assert result.identity_commit == "3" * 40 + assert common_calls == [ + { + "expected_frozen_identity_sha256": "d" * 64, + "frozen_identity_path": tmp_path / "identity.json", + "git_executable": git_executable, + "repository_root": tmp_path / "repository", + "repository_source_manifest_path": tmp_path / "source.json", + "source_commit": "1" * 40, + } + ] + + +def test_verify_model_staging_authorization_is_deterministic_read_only_and_exactly_forwarded( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + authorization = model_staging_authorization() + git_executable = runner.AuthenticatedGitExecutable( + path=tmp_path / "git.exe", + absolute_path_sha256="a" * 64, + sha256="b" * 64, + size_bytes=1, + ) + calls: list[dict[str, object]] = [] + monkeypatch.setattr(runner, "_authenticate_git_executable", lambda _path: git_executable) + + def authenticate(**kwargs: object) -> runner.ModelStagingAuthorization: + calls.append(dict(kwargs)) + return authorization + + monkeypatch.setattr(runner, "_authenticate_model_staging_authorization", authenticate) + original_import = builtins.__import__ + + def guarded_import(name: str, *args: object, **kwargs: object) -> object: + if name == "huggingface_hub" or name.startswith("huggingface_hub."): + pytest.fail("authorization preflight imported a Hugging Face client") + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", guarded_import) + untouched = tmp_path / "untouched" + arguments = { + "git_executable_path": tmp_path / "requested-git.exe", + "frozen_identity_path": untouched / "identity.json", + "expected_frozen_identity_sha256": "d" * 64, + "identity_commit": "3" * 40, + "repository_root": untouched / "repository", + "repository_source_manifest_path": untouched / "source.json", + "source_commit": "1" * 40, + "model_file_manifest_path": untouched / "model.json", + "expected_model_file_manifest_sha256": authorization.model_manifest.file_sha256, + } + + first = runner.verify_identity_bound_model_staging_authorization(**arguments) + second = runner.verify_identity_bound_model_staging_authorization(**arguments) + + expected = { + "artifact_kind": runner.MODEL_STAGING_AUTHORIZATION_KIND, + "file_count": len(authorization.model_manifest.files), + "frozen_identity_file_sha256": "d" * 64, + "hub_tree_manifest_sha256": authorization.model_manifest.hub_tree_manifest_sha256, + "identity_commit": "3" * 40, + "model_id": "example/model", + "model_manifest_file_sha256": authorization.model_manifest.file_sha256, + "revision": "2" * 40, + "repository_source_manifest_file_sha256": "7" * 64, + "runner_revision": runner.RUNNER_REVISION, + "schema_version": runner.MODEL_STAGING_AUTHORIZATION_SCHEMA, + "source_commit": "1" * 40, + "status": "verified_identity_bound_model_staging_authorization", + "total_size_bytes": sum(item.size_bytes for item in authorization.model_manifest.files), + } + assert first == expected + assert runner.canonical_json_bytes(first) == runner.canonical_json_bytes(second) + assert calls == [ + { + "expected_frozen_identity_sha256": "d" * 64, + "expected_model_file_manifest_sha256": authorization.model_manifest.file_sha256, + "frozen_identity_path": untouched / "identity.json", + "git_executable": git_executable, + "identity_commit": "3" * 40, + "model_file_manifest_path": untouched / "model.json", + "repository_root": untouched / "repository", + "repository_source_manifest_path": untouched / "source.json", + "source_commit": "1" * 40, + }, + { + "expected_frozen_identity_sha256": "d" * 64, + "expected_model_file_manifest_sha256": authorization.model_manifest.file_sha256, + "frozen_identity_path": untouched / "identity.json", + "git_executable": git_executable, + "identity_commit": "3" * 40, + "model_file_manifest_path": untouched / "model.json", + "repository_root": untouched / "repository", + "repository_source_manifest_path": untouched / "source.json", + "source_commit": "1" * 40, + }, + ] + assert not untouched.exists() + + +def test_verify_model_staging_authorization_failure_creates_nothing( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + untouched = tmp_path / "untouched" + original_import = builtins.__import__ + + def guarded_import(name: str, *args: object, **kwargs: object) -> object: + if name == "huggingface_hub" or name.startswith("huggingface_hub."): + pytest.fail("failed authorization preflight imported a Hugging Face client") + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", guarded_import) + monkeypatch.setattr( + runner, + "_authenticate_git_executable", + lambda _path: runner.AuthenticatedGitExecutable( + path=tmp_path / "git.exe", + absolute_path_sha256="a" * 64, + sha256="b" * 64, + size_bytes=1, + ), + ) + monkeypatch.setattr( + runner, + "_authenticate_model_staging_authorization", + lambda **_kwargs: (_ for _ in ()).throw( + runner.CalibrationRunError("injected authorization failure") + ), + ) + + with pytest.raises(runner.CalibrationRunError, match="injected authorization failure"): + runner.verify_identity_bound_model_staging_authorization( + git_executable_path=tmp_path / "git.exe", + frozen_identity_path=untouched / "identity.json", + expected_frozen_identity_sha256="d" * 64, + identity_commit="3" * 40, + repository_root=untouched / "repository", + repository_source_manifest_path=untouched / "source.json", + source_commit="1" * 40, + model_file_manifest_path=untouched / "model.json", + expected_model_file_manifest_sha256="2" * 64, + ) + + assert not untouched.exists() + + def test_committed_frozen_identity_requires_exact_head_index_and_worktree_blob( tmp_path: Path, ) -> None: @@ -2547,6 +3189,172 @@ def stage(**kwargs: object) -> dict[str, object]: ] +def test_verify_frozen_identity_contract_cli_is_canonical_and_exactly_forwarded( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + calls: list[dict[str, object]] = [] + receipt = { + "artifact_kind": runner.FROZEN_IDENTITY_CONTRACT_KIND, + "schema_version": runner.FROZEN_IDENTITY_CONTRACT_SCHEMA, + "status": "verified_frozen_identity_contract", + } + + def verify(**kwargs: object) -> dict[str, object]: + calls.append(dict(kwargs)) + return receipt + + monkeypatch.setattr(runner, "verify_frozen_identity_contract", verify) + arguments = [ + "verify-frozen-identity-contract", + "--git-executable", + str(authenticated_git_path()), + "--frozen-identity", + str(tmp_path / "identity.json"), + "--expected-frozen-identity-sha256", + "1" * 64, + "--repository-root", + str(tmp_path / "repository"), + "--repository-source-manifest", + str(tmp_path / "source.json"), + "--source-commit", + "3" * 40, + ] + + assert runner.main(arguments) == 0 + assert capsys.readouterr().out == runner.canonical_json_bytes(receipt).decode("utf-8") + assert calls == [ + { + "expected_frozen_identity_sha256": "1" * 64, + "frozen_identity_path": tmp_path / "identity.json", + "git_executable_path": authenticated_git_path(), + "repository_root": tmp_path / "repository", + "repository_source_manifest_path": tmp_path / "source.json", + "source_commit": "3" * 40, + } + ] + + +@pytest.mark.parametrize( + "forbidden", + [ + ("--identity-commit", "2" * 40), + ("--model-file-manifest", "model.json"), + ("--expected-model-file-manifest-sha256", "4" * 64), + ("--hub-cache-root", "cache"), + ("--output-root", "output"), + ("--output", "output.json"), + ("--cache-root", "cache"), + ("--local-files-only",), + ], + ids=( + "h1", + "model-manifest", + "model-manifest-hash", + "hub-cache", + "output-root", + "output", + "cache-root", + "local-hub-mode", + ), +) +def test_verify_frozen_identity_contract_cli_rejects_forbidden_surfaces( + forbidden: tuple[str, ...], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[dict[str, object]] = [] + monkeypatch.setattr( + runner, + "verify_frozen_identity_contract", + lambda **kwargs: calls.append(dict(kwargs)), + ) + arguments = [ + "verify-frozen-identity-contract", + "--git-executable", + str(authenticated_git_path()), + "--frozen-identity", + str(tmp_path / "identity.json"), + "--expected-frozen-identity-sha256", + "1" * 64, + "--repository-root", + str(tmp_path / "repository"), + "--repository-source-manifest", + str(tmp_path / "source.json"), + "--source-commit", + "3" * 40, + ] + + with pytest.raises(SystemExit): + runner.main([*arguments, *forbidden]) + assert calls == [] + + +def test_verify_model_staging_authorization_cli_has_no_cache_or_output_surface( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + calls: list[dict[str, object]] = [] + receipt = { + "artifact_kind": runner.MODEL_STAGING_AUTHORIZATION_KIND, + "status": "verified_identity_bound_model_staging_authorization", + } + + def verify(**kwargs: object) -> dict[str, object]: + calls.append(dict(kwargs)) + return receipt + + monkeypatch.setattr(runner, "verify_identity_bound_model_staging_authorization", verify) + arguments = [ + "verify-model-staging-authorization", + "--git-executable", + str(authenticated_git_path()), + "--frozen-identity", + str(tmp_path / "identity.json"), + "--expected-frozen-identity-sha256", + "1" * 64, + "--identity-commit", + "2" * 40, + "--repository-root", + str(tmp_path / "repository"), + "--repository-source-manifest", + str(tmp_path / "source.json"), + "--source-commit", + "3" * 40, + "--model-file-manifest", + str(tmp_path / "model.json"), + "--expected-model-file-manifest-sha256", + "4" * 64, + ] + + assert runner.main(arguments) == 0 + assert capsys.readouterr().out == runner.canonical_json_bytes(receipt).decode("utf-8") + assert calls == [ + { + "expected_frozen_identity_sha256": "1" * 64, + "expected_model_file_manifest_sha256": "4" * 64, + "frozen_identity_path": tmp_path / "identity.json", + "git_executable_path": authenticated_git_path(), + "identity_commit": "2" * 40, + "model_file_manifest_path": tmp_path / "model.json", + "repository_root": tmp_path / "repository", + "repository_source_manifest_path": tmp_path / "source.json", + "source_commit": "3" * 40, + } + ] + + for forbidden in ( + ("--hub-cache-root", str(tmp_path / "forbidden-cache")), + ("--output-root", str(tmp_path / "forbidden-output")), + ("--local-files-only",), + ): + with pytest.raises(SystemExit): + runner.main([*arguments, *forbidden]) + assert len(calls) == 1 + + def test_source_manifest_output_location_allows_only_external_or_ignored_paths( tmp_path: Path, ) -> None: From 84edf4299e8a5b3af970f74f03abc099d3696904 Mon Sep 17 00:00:00 2001 From: HiddenTrojan Date: Sat, 15 Aug 2026 02:33:57 +0800 Subject: [PATCH 19/24] Record Experiment 013 replacement calibration identity --- evidence/experiment013-calibration-identity-v5-h0-85625a5.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 evidence/experiment013-calibration-identity-v5-h0-85625a5.json diff --git a/evidence/experiment013-calibration-identity-v5-h0-85625a5.json b/evidence/experiment013-calibration-identity-v5-h0-85625a5.json new file mode 100644 index 0000000..c548fdd --- /dev/null +++ b/evidence/experiment013-calibration-identity-v5-h0-85625a5.json @@ -0,0 +1 @@ +{"canonical_evidence_sha256":"d469480537141ee6866d41faba3b17cd6397cdb62f1d7204218dcbd1ebe8e324","evidence":{"artifact_kind":"recurquant_static_rht_q468_identity","calibration_binding":null,"calibration_split_half":{"assignment":[{"canonical_id":"872","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":0,"rank_sha256":"01e970ad8f49c615f8994f4fc293d373d517f753cf5cef3ee60e04b0f70e91de","ruler_category":null,"seed":null,"sequence_length":205},{"canonical_id":"907","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":1,"rank_sha256":"0302f20255cec68e850f67a965f94ac9c6e975fdbeb3654b213c03aaf97749f8","ruler_category":null,"seed":null,"sequence_length":195},{"canonical_id":"602","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":2,"rank_sha256":"0533693c5eafac0f2bffd8ecb5f531f37e3220e7347ef75eab142ed8a013bcf5","ruler_category":null,"seed":null,"sequence_length":131},{"canonical_id":"884","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":3,"rank_sha256":"08317423d6ed9f655783d9a55d4773b1ff31c0a8355dfd9186fb319920e3d84b","ruler_category":null,"seed":null,"sequence_length":180},{"canonical_id":"771","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":4,"rank_sha256":"0aa65ef4aee19c821f354944ea63d3a5573594cbf98328585a2bf06c2de91309","ruler_category":null,"seed":null,"sequence_length":210},{"canonical_id":"885","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":5,"rank_sha256":"0b8df4c9f6062430c6c4ccd1cb93ed972689c2181d647b3425c8ac35779c5f9c","ruler_category":null,"seed":null,"sequence_length":193},{"canonical_id":"672","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":6,"rank_sha256":"0d45859c2052e7483b7caae8e52246e30d4af1be980cbcb175164817f5ed511d","ruler_category":null,"seed":null,"sequence_length":164},{"canonical_id":"955","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":7,"rank_sha256":"15743574d0995e07e15392053b3cafd1cef3998b668e87041851ac95861883ca","ruler_category":null,"seed":null,"sequence_length":116},{"canonical_id":"794","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":8,"rank_sha256":"19ee9c8929dc155f23f27a50530b329f876367ad3aea0bf4ad48ce4621328517","ruler_category":null,"seed":null,"sequence_length":154},{"canonical_id":"641","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":9,"rank_sha256":"19fd16656926305e85d6a90a8abfc8688cde21fff9383ac96aa4a507279d79f9","ruler_category":null,"seed":null,"sequence_length":104},{"canonical_id":"744","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":10,"rank_sha256":"1afe31e9cfc41a6df7f6609d1e2119ad55693730c75b4bd46f7c9235522eafa0","ruler_category":null,"seed":null,"sequence_length":137},{"canonical_id":"647","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":11,"rank_sha256":"1b76da1efe10f0a39d6b61952eb3fee319e653390381bd4a8efb060e10161d77","ruler_category":null,"seed":null,"sequence_length":107},{"canonical_id":"763","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":12,"rank_sha256":"1c7722a133b7999ee3e6ff0c4c7d9286371137589ac7b013d5d9a1615c2eca43","ruler_category":null,"seed":null,"sequence_length":181},{"canonical_id":"835","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":13,"rank_sha256":"27e79049077f4f70e44f882154a065bb8fe957aa57c51a5f95d025e7f6b69e7e","ruler_category":null,"seed":null,"sequence_length":113},{"canonical_id":"720","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":14,"rank_sha256":"2c2ba3d77f6374f2c035569f8be3b970f76c25bc587f0f2bd165939b49af1c27","ruler_category":null,"seed":null,"sequence_length":292},{"canonical_id":"677","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":15,"rank_sha256":"3015295069fd5535022e22aa7d447fdfd3b893623d25fbe8b2279d94c7d99ee5","ruler_category":null,"seed":null,"sequence_length":120},{"canonical_id":"819","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":16,"rank_sha256":"3355f161793b19251d4281f96e19b37a970a709a1b417d52e6dfbb2a4812d792","ruler_category":null,"seed":null,"sequence_length":385},{"canonical_id":"950","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":17,"rank_sha256":"339900553b726e1b1256a75b102fbeed3f547bfae8d96f80e8fec0f8dec6c09b","ruler_category":null,"seed":null,"sequence_length":399},{"canonical_id":"622","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":18,"rank_sha256":"363cce01da9a0aa22049e9b3a8e04a2dc931468e02d0086bb7af971621cfc711","ruler_category":null,"seed":null,"sequence_length":373},{"canonical_id":"708","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":19,"rank_sha256":"372acd4106f75fd22c1fdb5d8e13b74c64a9364db2f490f50daa482c0962112d","ruler_category":null,"seed":null,"sequence_length":97},{"canonical_id":"878","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":20,"rank_sha256":"38728fc433a299c9ff5e0f49471dda51ee67c5a980c966c00c715bd76e7f1c88","ruler_category":null,"seed":null,"sequence_length":168},{"canonical_id":"648","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":21,"rank_sha256":"388ca5f79823a878cd3ce1e033b61e5cab8bf668fe57d751138244feef74a2f4","ruler_category":null,"seed":null,"sequence_length":225},{"canonical_id":"937","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":22,"rank_sha256":"3ad5eee8852be5065365af9c25ff78ec2974ee89b2c6fa6759e27b7de255e202","ruler_category":null,"seed":null,"sequence_length":110},{"canonical_id":"894","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":23,"rank_sha256":"3c7238b06352863f75aab36214380eedfa02c7c388385142283f0980d21c1f7a","ruler_category":null,"seed":null,"sequence_length":226},{"canonical_id":"653","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":24,"rank_sha256":"3dd1d2a846dba10b3ffa7e554091ac65fe9284c15f2d83dc44c705155130a763","ruler_category":null,"seed":null,"sequence_length":287},{"canonical_id":"679","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":25,"rank_sha256":"3e59d101b564d5e3344a2e764584a194a5f663f3b20f7e939d7f719f84dacba4","ruler_category":null,"seed":null,"sequence_length":136},{"canonical_id":"657","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":26,"rank_sha256":"3e6b96b2be683d5d193895bbc05beb5a13bfc3b1db01872788e0ae8133c72b36","ruler_category":null,"seed":null,"sequence_length":171},{"canonical_id":"924","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":27,"rank_sha256":"45104189ef24ab437b12b531013325cf0a41819cf135255a4ca993f29f74dfd2","ruler_category":null,"seed":null,"sequence_length":103},{"canonical_id":"686","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":28,"rank_sha256":"47010c76b0b792ee5452a5eb55e8ba40d386f4a17d895b443eb1a66556c90596","ruler_category":null,"seed":null,"sequence_length":285},{"canonical_id":"860","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":29,"rank_sha256":"471b89737df3331c676e99243030a77140d460cde893f819d2c7916e0a16f453","ruler_category":null,"seed":null,"sequence_length":141},{"canonical_id":"698","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":30,"rank_sha256":"4a9f3f47f58cfaabd9afda89457088ca35397d7acc80568d8d218a723451116e","ruler_category":null,"seed":null,"sequence_length":359},{"canonical_id":"827","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":31,"rank_sha256":"4b073dd1bc9f2a79d60985c8d50aae572f2030e62a55477c64702d735d849023","ruler_category":null,"seed":null,"sequence_length":174},{"canonical_id":"651","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":32,"rank_sha256":"4bf2b2787c95b7b8a2f8ff089c36ea2662b4987dbf230b02b81c424382853b7b","ruler_category":null,"seed":null,"sequence_length":155},{"canonical_id":"930","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":33,"rank_sha256":"4ffef951fb9708b54468eaa75c7b42b9f58c198c366cfacd6b4f665c9d77984f","ruler_category":null,"seed":null,"sequence_length":138},{"canonical_id":"962","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":34,"rank_sha256":"51a6c027909f096835086346110e63a131c1462156020bb129058b320f27516a","ruler_category":null,"seed":null,"sequence_length":149},{"canonical_id":"853","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":35,"rank_sha256":"525322d1199e29de4998af2c95211cd981e642e9d602fe16554a8a63fc02d729","ruler_category":null,"seed":null,"sequence_length":219},{"canonical_id":"722","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":36,"rank_sha256":"52acb8744119319aaa1a68b3e192415c937a6822ceb2963069d4ede3e9b75cc4","ruler_category":null,"seed":null,"sequence_length":388},{"canonical_id":"705","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":37,"rank_sha256":"547e7ebc86a8992205db2eda5a7117796f15c3042e90200d556582e58f881c17","ruler_category":null,"seed":null,"sequence_length":258},{"canonical_id":"944","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":38,"rank_sha256":"577f992e266f3ffe7684e9483310b96df9b338bd836e00af79e0116821fddaa5","ruler_category":null,"seed":null,"sequence_length":117},{"canonical_id":"849","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":39,"rank_sha256":"59660871dc9200a74f124b924d6e85f743e541a39ae93af501968a7ed5d0d751","ruler_category":null,"seed":null,"sequence_length":161},{"canonical_id":"919","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":40,"rank_sha256":"5a21a04a8534d8dff43c2d02da1415ffaabed4a09662059b05e916108da7665e","ruler_category":null,"seed":null,"sequence_length":113},{"canonical_id":"629","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":41,"rank_sha256":"5aeb994e6b892f1741b894b66750ea3690b71585304601a66a17d599e3e96bc5","ruler_category":null,"seed":null,"sequence_length":150},{"canonical_id":"820","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":42,"rank_sha256":"6063af147dd771453be5c51054092d7aa85f751eab9d697cc1b77be9259ed7ff","ruler_category":null,"seed":null,"sequence_length":105},{"canonical_id":"627","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":43,"rank_sha256":"608304ac3176b904f89e46ddf06a578afe6f80161ceb1adb2b5049c5b31a8c5a","ruler_category":null,"seed":null,"sequence_length":198},{"canonical_id":"755","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":44,"rank_sha256":"60b435df2cb81d92c2de2d1edc7b06a50a523b8d40047dd521cf241475d6b370","ruler_category":null,"seed":null,"sequence_length":209},{"canonical_id":"911","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":45,"rank_sha256":"614cef397364dc767d15426e06989a10147811417996eb4fa91721038ecac80c","ruler_category":null,"seed":null,"sequence_length":236},{"canonical_id":"874","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":46,"rank_sha256":"62fc38937c372269a01f02df68b088b1fdc004f7ae4590a50cb3673f385737d0","ruler_category":null,"seed":null,"sequence_length":153},{"canonical_id":"918","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":47,"rank_sha256":"6326299b9eaf024dfc7efa6b26bb898a5b869898127d546067800b05ba8c5932","ruler_category":null,"seed":null,"sequence_length":229},{"canonical_id":"803","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":48,"rank_sha256":"65df1000f86d2eacc5ecb7d9b91504d3c6074a9e9c5e6cc8abebfc45c0f60974","ruler_category":null,"seed":null,"sequence_length":142},{"canonical_id":"868","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":49,"rank_sha256":"69fda125d9dfbbac0a44eaf6db26953f3f644a70389add149dd1c47e3ca90bf4","ruler_category":null,"seed":null,"sequence_length":137},{"canonical_id":"666","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":50,"rank_sha256":"6c8674e41d1ec38f03b2d176e08aef538902ff0c35234be50164674f4e52a341","ruler_category":null,"seed":null,"sequence_length":108},{"canonical_id":"768","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":51,"rank_sha256":"71cb4040b63cf195e0e702fd08f6b6e9eeef03a3de4a3c29ca0a715a42e2fbf6","ruler_category":null,"seed":null,"sequence_length":144},{"canonical_id":"636","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":52,"rank_sha256":"7357df8332514f1e676d0104bd545961a71f432580a835738fb62ad7dbc89065","ruler_category":null,"seed":null,"sequence_length":127},{"canonical_id":"671","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":53,"rank_sha256":"7a7a9eda4414dd86e6dbd5ebf5e1ac84df7df6e12e3242cb2574817453c497b8","ruler_category":null,"seed":null,"sequence_length":193},{"canonical_id":"783","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":54,"rank_sha256":"7aabfc7187ec019772ef6f0f0ae5203e0193909146a86ecdc6a1f789f86ff6c3","ruler_category":null,"seed":null,"sequence_length":428},{"canonical_id":"695","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":55,"rank_sha256":"7ce9ea381ac2a57a8373c1253b0dfbedb3b2e838c4dfd92bb0730f83c972434c","ruler_category":null,"seed":null,"sequence_length":168},{"canonical_id":"851","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":56,"rank_sha256":"7df3d4757b59a5be15e6e4b9208bfbdbf189f4590d7b2e93ca0f84047f2a97b4","ruler_category":null,"seed":null,"sequence_length":131},{"canonical_id":"945","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":57,"rank_sha256":"7fdc834d39787ed9f97e73645f7df6692b711d910c52482f0bbab1d41512f720","ruler_category":null,"seed":null,"sequence_length":132},{"canonical_id":"687","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":58,"rank_sha256":"80a7b335ef9b11db152007e896afdd070710cbee60d4766ef829826eacbbf56b","ruler_category":null,"seed":null,"sequence_length":153},{"canonical_id":"908","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":59,"rank_sha256":"8267d3b36347390e9dbe928d4f2af728975e565b8c89232c8fe011b762fb9e75","ruler_category":null,"seed":null,"sequence_length":175},{"canonical_id":"973","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":60,"rank_sha256":"85f2e19defa2f2194e4bd627c43470d94dfce10775f216d0eee0f5b7c89a8113","ruler_category":null,"seed":null,"sequence_length":107},{"canonical_id":"795","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":61,"rank_sha256":"874c549c5d365887eba08a0eac638427d2825d7bedc3da4b4aa6c213de86162a","ruler_category":null,"seed":null,"sequence_length":325},{"canonical_id":"822","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":62,"rank_sha256":"87ee9e94899a058de1bc21331e5ffecb56f11ce924750fc73a6a928a2501d281","ruler_category":null,"seed":null,"sequence_length":203},{"canonical_id":"728","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":63,"rank_sha256":"884e088f56e16b57342c6b5d6b1021b0f7e756e9d8d1720ce65c243eb1c95acf","ruler_category":null,"seed":null,"sequence_length":165},{"canonical_id":"895","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":64,"rank_sha256":"888165087ffe22467a3cefc2c892a649a93905d8f22cd2389a8c7ba0be70f967","ruler_category":null,"seed":null,"sequence_length":293},{"canonical_id":"764","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":65,"rank_sha256":"8981728482ffcdb96388f812ddfe70ae76010f928900d65b3f4a8c8490ed266b","ruler_category":null,"seed":null,"sequence_length":125},{"canonical_id":"804","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":66,"rank_sha256":"8996abd03167db364eb3cf521940939970ca3b49310060c371c0e447dca9f4d4","ruler_category":null,"seed":null,"sequence_length":134},{"canonical_id":"732","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":67,"rank_sha256":"89d9cbbf3458bcc72f432dd40166313978822085bc657d2cc2ff5fe6919c27a8","ruler_category":null,"seed":null,"sequence_length":134},{"canonical_id":"839","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":68,"rank_sha256":"8c48b54163b5a2b5d91d303abe4d2f5990fd2296dd47e1960208c1d0286d9450","ruler_category":null,"seed":null,"sequence_length":365},{"canonical_id":"823","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":69,"rank_sha256":"8df1ac0f53587cc228a0d68d4959d98a79d5c3046a25094f36d1698d013a1a7a","ruler_category":null,"seed":null,"sequence_length":205},{"canonical_id":"713","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":70,"rank_sha256":"8eb162011c59bc92c4728ac7fe2fd64ac0f7f78f7aef1ba048088ed9743e16d2","ruler_category":null,"seed":null,"sequence_length":121},{"canonical_id":"772","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":71,"rank_sha256":"8f08bc003b61873c2e9f27eceb15bf7a3a451a8f4877c759cc73b5f437a170af","ruler_category":null,"seed":null,"sequence_length":166},{"canonical_id":"727","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":72,"rank_sha256":"925293e3739b4a2498fb5d6c9cbf9fbd57520f22f99a7b3468fad6290361cf36","ruler_category":null,"seed":null,"sequence_length":161},{"canonical_id":"858","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":73,"rank_sha256":"935132e4b17a56013a0312175973c9a6b2b092902d5bf1d474b03a34da4953f6","ruler_category":null,"seed":null,"sequence_length":177},{"canonical_id":"800","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":74,"rank_sha256":"9400093971582810aed9b43834286c1bbcdbe15e12fd1f749b9e3a7abbefa8f4","ruler_category":null,"seed":null,"sequence_length":104},{"canonical_id":"751","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":75,"rank_sha256":"96679ba0da36bb95058654f8fea49ff5305709d80cd95a1017d570f0d7997ddf","ruler_category":null,"seed":null,"sequence_length":243},{"canonical_id":"674","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":76,"rank_sha256":"96c806e69e6b65095699a2192e225cc22ad2b21211dbb8eae36893932ef358e1","ruler_category":null,"seed":null,"sequence_length":128},{"canonical_id":"960","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":77,"rank_sha256":"97692bbfd642b241043c3eb82fd110341e33df034bcc65a60b79b26933a15b3e","ruler_category":null,"seed":null,"sequence_length":124},{"canonical_id":"867","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":78,"rank_sha256":"979109f8d629072c0302399d65fa7263972fa9ed22cefff82f4c11028507238f","ruler_category":null,"seed":null,"sequence_length":170},{"canonical_id":"669","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":79,"rank_sha256":"98ddb8c36da9cd7adc3852209dcda61e9da6d3acd0d1fcf1211f25ae04c07cc5","ruler_category":null,"seed":null,"sequence_length":296},{"canonical_id":"656","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":80,"rank_sha256":"9979d6795117639a4d1f8afae4ddfa11dfeeefbf2ca8687a7e67b84643a1744f","ruler_category":null,"seed":null,"sequence_length":169},{"canonical_id":"862","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":81,"rank_sha256":"99e8fb364eec07a688b1d0dbc74ba9372c47719c0193b304c88bc2659ecd75c4","ruler_category":null,"seed":null,"sequence_length":177},{"canonical_id":"635","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":82,"rank_sha256":"9cf09b9e19663d0a700e97d8d627776519ef82d0d8608b62a5645b11309ec58f","ruler_category":null,"seed":null,"sequence_length":266},{"canonical_id":"756","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":83,"rank_sha256":"9d6ace56a87e526ec18af2d10e07c6dd1a3287a2d8271415ac4a0c3a3f642035","ruler_category":null,"seed":null,"sequence_length":139},{"canonical_id":"869","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":84,"rank_sha256":"9d975c8aee95cb7e1e54deeee4073d57a63712f67247e5fb61dcf791dbb16564","ruler_category":null,"seed":null,"sequence_length":352},{"canonical_id":"857","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":85,"rank_sha256":"9e00a569dd1098d4912c32f57e2351709bf3ffce9a75edf00179d5f023c41d41","ruler_category":null,"seed":null,"sequence_length":293},{"canonical_id":"607","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":86,"rank_sha256":"a18910fa12a223a0ecb7d6707500711e3b376af22c72b99625792f523b548d89","ruler_category":null,"seed":null,"sequence_length":207},{"canonical_id":"929","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":87,"rank_sha256":"a39d16127e92e6eb192ad63a45fa7db3125509f3a86fcc647d0a72ffdfc25483","ruler_category":null,"seed":null,"sequence_length":168},{"canonical_id":"606","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":88,"rank_sha256":"a4d325b0ce1088bd97fbaac9299e2d0dbb32b002026f2528825225d44f1e1e60","ruler_category":null,"seed":null,"sequence_length":144},{"canonical_id":"662","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":89,"rank_sha256":"a572f1763ed2dacfc5b6f8bafb04903bbc369519be95c8ecfd7d26a2792dc619","ruler_category":null,"seed":null,"sequence_length":350},{"canonical_id":"877","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":90,"rank_sha256":"a7576eaa33a3fb7087c29ae14949e5bfc2e1ef4449b997384e88a2d5dd631591","ruler_category":null,"seed":null,"sequence_length":93},{"canonical_id":"946","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":91,"rank_sha256":"a884dbdb72cb7d11613027848d08702a496c419bc5c987e9e963e2184c774bd8","ruler_category":null,"seed":null,"sequence_length":209},{"canonical_id":"785","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":92,"rank_sha256":"ab824ea47e78d00ae4aec6a45480e122e3703f1da3037053af0aa7a891b637ee","ruler_category":null,"seed":null,"sequence_length":152},{"canonical_id":"920","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":93,"rank_sha256":"ad357f573b8f19c5b89f84aa9097d23ace8d1488e9cb2c1322577137d0d53c2a","ruler_category":null,"seed":null,"sequence_length":251},{"canonical_id":"616","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":94,"rank_sha256":"b09c89c6a2366a5a51caa203e1f363d01e8340abcd1ca162839843b86eef1161","ruler_category":null,"seed":null,"sequence_length":221},{"canonical_id":"840","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":95,"rank_sha256":"b2be594c39afb592547a351b8235b8061e103c0c75d777ad1549d2e015a87989","ruler_category":null,"seed":null,"sequence_length":131},{"canonical_id":"644","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":96,"rank_sha256":"b6557529f8ec5764b47cd34b0b579eb20371c1c581831a55709bb7d1b27348af","ruler_category":null,"seed":null,"sequence_length":186},{"canonical_id":"704","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":97,"rank_sha256":"ba157c706c3d65835a4022b531b73e3f1b65c1660d0699fd7293821d2ebfe350","ruler_category":null,"seed":null,"sequence_length":153},{"canonical_id":"712","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":98,"rank_sha256":"bb6cd04968145b1904af1e96e70b74699cb7138a72c4c62e0b82ee87a3375fc0","ruler_category":null,"seed":null,"sequence_length":219},{"canonical_id":"882","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":99,"rank_sha256":"bf3e15adb4c326aab0f479085d1e9971db834476e9ecaf1a9c3d7764fec4378b","ruler_category":null,"seed":null,"sequence_length":112},{"canonical_id":"710","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":100,"rank_sha256":"bf5d27c709dca8c30c14453c4e20b7c75f1e93bd5952da6bbc741e43a68f0575","ruler_category":null,"seed":null,"sequence_length":163},{"canonical_id":"821","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":101,"rank_sha256":"c0ab0a4bb636c134ba2233ecc3d1925c81df2a89060c09a2908c9dddbc14b591","ruler_category":null,"seed":null,"sequence_length":282},{"canonical_id":"707","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":102,"rank_sha256":"c43bbdda480f2ebabe97769f2958f17a19febca5a53140948914ad1bdd0a6e3e","ruler_category":null,"seed":null,"sequence_length":200},{"canonical_id":"614","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":103,"rank_sha256":"c5450b1ef19a8eaf7e4710ad9115e76118a4634fb769d3924067bdec342e2e70","ruler_category":null,"seed":null,"sequence_length":165},{"canonical_id":"741","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":104,"rank_sha256":"c7c6fbada9b7a2d3d38750bc16cb99ea97efb71c0413563d3f399cc0722c6bcd","ruler_category":null,"seed":null,"sequence_length":124},{"canonical_id":"702","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":105,"rank_sha256":"c8a56538eebae04a29a21ac3c638e71a9ea53a1fdc12603034b61d9ef4359cc9","ruler_category":null,"seed":null,"sequence_length":332},{"canonical_id":"824","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":106,"rank_sha256":"c9106dec19a3fa31e33d340927464d15b6150cbc251a6aff64336706d0400b98","ruler_category":null,"seed":null,"sequence_length":129},{"canonical_id":"809","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":107,"rank_sha256":"ca59973992bc52f67689974ae41022e87d598c8c31de31955f30bfaf21889c3f","ruler_category":null,"seed":null,"sequence_length":173},{"canonical_id":"886","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":108,"rank_sha256":"cc1cdebfc08022bfe9fae59cf187db791a655ca5b48463b8c2725e74349fa708","ruler_category":null,"seed":null,"sequence_length":152},{"canonical_id":"789","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":109,"rank_sha256":"d71db894fab22f0f681f7e02a1ec80d922272c111bb9462b9d066866ab601c18","ruler_category":null,"seed":null,"sequence_length":98},{"canonical_id":"870","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":110,"rank_sha256":"dacc01a92b59eb20074bd1a8255fdbbc3298f37030e34e3d4e074c0bb66134ba","ruler_category":null,"seed":null,"sequence_length":208},{"canonical_id":"854","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":111,"rank_sha256":"dd5f39a3a28901c30d626c95276606c2a6ad00b0f64384c782406f9a13ef8d1e","ruler_category":null,"seed":null,"sequence_length":235},{"canonical_id":"646","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":112,"rank_sha256":"deb0492a1dc5a716095faeaf6d418cf2bf74040a83940a2690a6ea83442404de","ruler_category":null,"seed":null,"sequence_length":129},{"canonical_id":"605","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":113,"rank_sha256":"e161ff3b2e28c4f02462643895bc6e95b48655efc7ac0ace9eec5c4226576cb0","ruler_category":null,"seed":null,"sequence_length":125},{"canonical_id":"902","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":114,"rank_sha256":"e1ebf3f6386478a8625ad8c53858193b08d3e9a3ec267789cd98e85c340f314a","ruler_category":null,"seed":null,"sequence_length":310},{"canonical_id":"848","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":115,"rank_sha256":"e2bc9c9a9b23e33f1abf22dce38357e1e5c10acff55d2a208423270bc02a7289","ruler_category":null,"seed":null,"sequence_length":129},{"canonical_id":"847","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":116,"rank_sha256":"e351b9058c9e5c2e9e58444e1e4f2346c4886c6705652807fc2baab4f1806bd0","ruler_category":null,"seed":null,"sequence_length":143},{"canonical_id":"630","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":117,"rank_sha256":"e553afed5e6fb0d6843dfe2f22bdb6aa65d8f483b406e1b83ee5a2bf2e87fba2","ruler_category":null,"seed":null,"sequence_length":327},{"canonical_id":"793","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":118,"rank_sha256":"e76c7363df6aa1bca2a659469266d4228a66b4a2e10bf91c06346d0eaf0da89e","ruler_category":null,"seed":null,"sequence_length":209},{"canonical_id":"725","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":119,"rank_sha256":"ebb93bbe303910a142ceeda734226a9701884eaee00b68852b557cfc23fc231b","ruler_category":null,"seed":null,"sequence_length":164},{"canonical_id":"903","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":120,"rank_sha256":"ed33b084d9852e0df6989759582108080423613a5014d3f1150c3cbbc0d60573","ruler_category":null,"seed":null,"sequence_length":161},{"canonical_id":"797","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":121,"rank_sha256":"edb3a9f5d11a94af9db154ef9fee83d36402a77de27eb0ba830a3902dca5536c","ruler_category":null,"seed":null,"sequence_length":147},{"canonical_id":"792","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":122,"rank_sha256":"ee24c9054290bd8ccfeccee4993ac8dc6e0e20c72cdf4fbe5d28593c947a2246","ruler_category":null,"seed":null,"sequence_length":130},{"canonical_id":"615","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":123,"rank_sha256":"eed8e28b1d920cec5525bff2f0f861ca6e43429ed75ae657e5754ce34b81d71a","ruler_category":null,"seed":null,"sequence_length":338},{"canonical_id":"640","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":124,"rank_sha256":"f08156d84d873d2ae9184c8db61828a6c9a15f94b5a43b4ef8fe5c1867c2f16d","ruler_category":null,"seed":null,"sequence_length":118},{"canonical_id":"709","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":125,"rank_sha256":"f94bcd46daf0c1bd69bc60844bac9acd167ddc7981f10db96385b6d755574ec2","ruler_category":null,"seed":null,"sequence_length":390},{"canonical_id":"846","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":126,"rank_sha256":"fb81297eb57060d653be5ab5aa61ab169ac4bb236168fbda838bc333f61d4bc7","ruler_category":null,"seed":null,"sequence_length":305},{"canonical_id":"936","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":127,"rank_sha256":"fefd4316fc5fd98157422b169451622b064b3709b67d87d78e00278992813ea2","ruler_category":null,"seed":null,"sequence_length":297},{"canonical_id":"http://www.gutenberg.org/ebooks/1219","config":"default","configured_length":null,"group":"pg19","half":"a","rank":0,"rank_sha256":"061889c1d506cd1d5fb9d49ee4d63712d7e12e6056ffcc25a15ed400766bb5b8","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/38160","config":"default","configured_length":null,"group":"pg19","half":"b","rank":1,"rank_sha256":"10e82e9edc65238326568f332ff5556646b86707e652e24b051769eb8ad7fb02","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/39447","config":"default","configured_length":null,"group":"pg19","half":"a","rank":2,"rank_sha256":"14037bdad952830661d418776aaf4616905c877bfdb291977c357381de5c229c","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/29413","config":"default","configured_length":null,"group":"pg19","half":"b","rank":3,"rank_sha256":"217a13c587e9ae2e779aae14a77fa4a49dbbcbea34bfd717de53c2ac37ea9675","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/1253","config":"default","configured_length":null,"group":"pg19","half":"a","rank":4,"rank_sha256":"2a24d338d8456351047da3a9c12b7feb6c9e39c040ea62275c9284d4bc3b02f6","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/35786","config":"default","configured_length":null,"group":"pg19","half":"b","rank":5,"rank_sha256":"5b4d0ebce948cb01a07e9166f2db8c4d3163c3ce8780be96639120dc13fb26e2","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/17774","config":"default","configured_length":null,"group":"pg19","half":"a","rank":6,"rank_sha256":"5cfbeaf96d6b8bf5cbe8754a74cd22b63225e5c1775594c88f7ff24fc3be1d90","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/10609","config":"default","configured_length":null,"group":"pg19","half":"b","rank":7,"rank_sha256":"73a499a3165431d2fcf4fe5184ef91447b8184d5bf8f0683adb49a4bf058e0fb","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/20279","config":"default","configured_length":null,"group":"pg19","half":"a","rank":8,"rank_sha256":"7da541ed6b56ea9ddd7749aef82c4f66b36beb7ca9ebd5b3ab2623c60996cac9","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/28065","config":"default","configured_length":null,"group":"pg19","half":"b","rank":9,"rank_sha256":"beee6f7ab98cda2f3595fe22f5221f16fd4ce8717a7fe77344ea7e851c85287f","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/24903","config":"default","configured_length":null,"group":"pg19","half":"a","rank":10,"rank_sha256":"c4b007d21ff91cbeb1959618d1f954e864dc7b874ae3986598c860a30b425cc0","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/33622","config":"default","configured_length":null,"group":"pg19","half":"b","rank":11,"rank_sha256":"dcca53b6159ae6063a9e62f780ecd4130f39bf20f2342fe194d39248032cb43d","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/26008","config":"default","configured_length":null,"group":"pg19","half":"a","rank":12,"rank_sha256":"e5734f2587660eccf1f1cae256502a6d7f22fa8d8898f4dee591a3e3a931de27","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/36808","config":"default","configured_length":null,"group":"pg19","half":"b","rank":13,"rank_sha256":"e9713427a3a331bb20abf5840a4a40e37e72569131f89b7ab0691653bc679901","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/2773","config":"default","configured_length":null,"group":"pg19","half":"a","rank":14,"rank_sha256":"f0e4622ba87c3913d530c0f8f7f20c419b2375ff1ca073e6062914dbabe1a4f5","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/32024","config":"default","configured_length":null,"group":"pg19","half":"b","rank":15,"rank_sha256":"f8ecebbe43befef18b00a5780b7383b2a5e3971aca73ed9caf82dd87b7be9b58","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:fwe:length=2048:seed=12339:sample=0","config":"fwe","configured_length":2048,"group":"ruler:aggregation","half":"a","rank":0,"rank_sha256":"407456144cdf7e7f879eeb8c590fc8039a5de9c5d2a99e094f58c354dc7a2e14","ruler_category":"aggregation","seed":12339,"sequence_length":1951},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:cwe:length=4096:seed=12340:sample=0","config":"cwe","configured_length":4096,"group":"ruler:aggregation","half":"b","rank":1,"rank_sha256":"6eed2f87558224dfd68001e835c96568c05e5f827c533e157752f375f4500e19","ruler_category":"aggregation","seed":12340,"sequence_length":3576},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:fwe:length=4096:seed=12339:sample=0","config":"fwe","configured_length":4096,"group":"ruler:aggregation","half":"a","rank":2,"rank_sha256":"9ba9392d1dc6140c8a464c6ab01eae04fd883d8520b997db8d8e7a81ec69412a","ruler_category":"aggregation","seed":12339,"sequence_length":3987},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:cwe:length=2048:seed=12340:sample=0","config":"cwe","configured_length":2048,"group":"ruler:aggregation","half":"b","rank":3,"rank_sha256":"acfea4ad2d1ba514b26bd0835657efe402f76ae3d519c669004fe8f3398c41ba","ruler_category":"aggregation","seed":12340,"sequence_length":1862},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=2048:seed=12339:sample=0","config":"vt","configured_length":2048,"group":"ruler:multi_hop_tracing","half":"a","rank":0,"rank_sha256":"089158a73f7f141394a585c20b3642a57e30b83534a971bffe8d8cc58d1d88a0","ruler_category":"multi_hop_tracing","seed":12339,"sequence_length":2011},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=4096:seed=12339:sample=0","config":"vt","configured_length":4096,"group":"ruler:multi_hop_tracing","half":"b","rank":1,"rank_sha256":"0c5867a34d4c0109a91512496cb821eeda3eacfff858dc77cce30e9fdd0dec20","ruler_category":"multi_hop_tracing","seed":12339,"sequence_length":4056},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=2048:seed=12340:sample=0","config":"vt","configured_length":2048,"group":"ruler:multi_hop_tracing","half":"a","rank":2,"rank_sha256":"47daa45868dba636bc69249fc76653ed16cc4674127f6229ffb7b5c6e0dc816b","ruler_category":"multi_hop_tracing","seed":12340,"sequence_length":2009},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=4096:seed=12340:sample=0","config":"vt","configured_length":4096,"group":"ruler:multi_hop_tracing","half":"b","rank":3,"rank_sha256":"cea1514b80ab90b346af0afd72d4e596f64be107a08c70781a292c2824193d97","ruler_category":"multi_hop_tracing","seed":12340,"sequence_length":4057},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_2:length=2048:seed=12340:sample=0","config":"qa_2","configured_length":2048,"group":"ruler:question_answering","half":"a","rank":0,"rank_sha256":"362ee5e975e3602b3afddead8d433435ec23dbad4defa23337e192b538091cbc","ruler_category":"question_answering","seed":12340,"sequence_length":1893},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_2:length=4096:seed=12340:sample=0","config":"qa_2","configured_length":4096,"group":"ruler:question_answering","half":"b","rank":1,"rank_sha256":"4e8e5919e5dec4b8c1e7aed66e68abf8414441353d6669352f7bb5eb6f3560ba","ruler_category":"question_answering","seed":12340,"sequence_length":2281},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_1:length=2048:seed=12339:sample=0","config":"qa_1","configured_length":2048,"group":"ruler:question_answering","half":"a","rank":2,"rank_sha256":"743c29a00e4cbe2d6bb62a01300b447fc3a78ded5878e9c03e7d80686d62e6c3","ruler_category":"question_answering","seed":12339,"sequence_length":1368},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_1:length=4096:seed=12339:sample=0","config":"qa_1","configured_length":4096,"group":"ruler:question_answering","half":"b","rank":3,"rank_sha256":"ec18351546c72bc1cad5f950ced4793e70ee2fa618a341924278809a62774186","ruler_category":"question_answering","seed":12339,"sequence_length":4040},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_single_1:length=4096:seed=12339:sample=0","config":"niah_single_1","configured_length":4096,"group":"ruler:retrieval","half":"a","rank":0,"rank_sha256":"4a756a6ea055639254af12c8568b7afc07bb626029548efa51a12d72a0109efb","ruler_category":"retrieval","seed":12339,"sequence_length":3955},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multiquery:length=2048:seed=12339:sample=0","config":"niah_multiquery","configured_length":2048,"group":"ruler:retrieval","half":"b","rank":1,"rank_sha256":"5ac13727e634d2432f98b2670f23761bdef39b61ebfadcecff6aa6e1dc1966a9","ruler_category":"retrieval","seed":12339,"sequence_length":1289},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multikey_2:length=2048:seed=12340:sample=0","config":"niah_multikey_2","configured_length":2048,"group":"ruler:retrieval","half":"a","rank":2,"rank_sha256":"8deff28bb1380b3aa1be3b678ea1793381004620e9b4a67fea888e2a859cb53c","ruler_category":"retrieval","seed":12340,"sequence_length":1828},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multivalue:length=4096:seed=12340:sample=0","config":"niah_multivalue","configured_length":4096,"group":"ruler:retrieval","half":"b","rank":3,"rank_sha256":"b13ed20ccd6d9f0502b4d16468a25828441ad0e8b8e8063b1dd3f5d674a1815f","ruler_category":"retrieval","seed":12340,"sequence_length":3964}],"assignment_sha256":"a42cf4b332cc8cf58b27709d7d261fc03a356b27ec1c9ccd56914d99e60c1797","namespace":"recurquant.experiment013.calibration-split.v1\u0000"},"claim_boundary":"This artifact freezes Experiment 013 data, tokenizer, source, runtime, and metadata-only model-file identity. It is not quality, latency, novelty, state-of-the-art, deployment, or breakthrough evidence.","content_manifest_sha256":"ee72483a8f8b4370c9e667e4287747e5bc358aeb0265a58167140f4e780a7b29","datasets":[{"canonical_id_field":"task_id","canonical_id_manifest_sha256":"f46a3c772f1416c75bdd8668c69f4886a32a243c1737b13bbddfdc23b9605c70","config":"full","dataset_id":"google-research-datasets/mbpp","formatter_id":"recurquant.mbpp-prompt-code.v1","formatter_sha256":"882e20ec9f5cbcb7e6f1310cbf46d19153721beac41ecc7ee308c39be17532ff","key":"mbpp","revision":"4bb6404fdc6cacfda99d4ac4205087b89d32030c","split":"train"},{"canonical_id_field":"url","canonical_id_manifest_sha256":"8d7ed934210f5e78f13559cdf48518b922d3c203df9258d0de3c4d62a18bdaa7","config":"default","dataset_id":"emozilla/pg19","formatter_id":"recurquant.pg19-token-slice.v1","formatter_sha256":"faea2480bf85adcd34339cae88a0e9b631b705eb547020572db74402bf525730","key":"pg19","revision":"c021754c8e01c5b1cc83a1f549c1f97fbbb756b8","split":"train"},{"canonical_id_field":"configuration_id","canonical_id_manifest_sha256":"d7fec5f2a57409fb842dcb967de5ed1efce00253723f2d891b28681b8d57952d","config":"official-generator","dataset_id":"NVIDIA/RULER","formatter_id":"recurquant.ruler-official-generated-record.v1","formatter_sha256":"4f797cbca4dfbe802c47dc5b238111b629f1497e46c84ea116d06319012016b6","key":"ruler","revision":"c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a","split":"generated"},{"canonical_id_field":"task_id","canonical_id_manifest_sha256":"c53bc1db6c7965cb13d4238321ef09c0c7b3dbfeb4f4736bfd6e46e2914449da","config":"default","dataset_id":"evalplus/humanevalplus","formatter_id":"recurquant.humaneval-plus-prompt-solution.v1","formatter_sha256":"12204389715ddc210e5a8b1b291f4fbcaf2b64fde94c22699873de80d682204c","key":"humaneval_plus","revision":"d32357cf319e50e9c8d8dab5ea876c72b0fd321b","split":"test"}],"execution_bindings":{"calibration_runtime_manifest_file_sha256":"80ca233a29af4facbb334fd4fb51a4f6e9a3d6815465cb79b9f3db63ef668d6a","model_file_manifest_file_sha256":"586d9c7e520f3bbd99ecef30663bf07d283eb14622475c58891becd8e033b05c","parquet_materialization_manifest_file_sha256":"ee5628e50e5d3516fd79077542d355fd915455ac0e53128d372f4177ad63d39c","repository_source_manifest_file_sha256":"fafe336a60ed5e42975f08cc042b5e1087403f2239c9fa6fb3f926382d500617"},"identity_only":true,"identity_schema":"recurquant.experiment013.identity-frozen.v5","model_contracts":{"conditional_scale_check":{"cold_start_peak_hbm_limit_bytes":8053063680,"id":"Qwen/Qwen3.5-2B-Base","revision":"b1485b2fa6dfa1287294f269f5fb618e03d52d7c"},"primary":{"id":"Qwen/Qwen3.5-0.8B-Base","revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68"},"weights_loaded":false},"phase":"calibration","promotion":{"candidate_canonical_evidence_sha256":"96742588186fa4e8a0ee00b4de36c64694beae23aab401d9b8bc15a3ebb12453","candidate_file_sha256":"4dcefabb6581b3675b880c43277197a3bf41bda1a655be5146df7b1218873878","explicit":true},"promotion_required":false,"protected_identity":{"ordinary_tests_may_read_protected_content":false,"stage_b_read":false,"stage_c_read":false},"record_count":160,"records":[{"anchor_manifest_sha256":"9727dad06649abe3aad4a15772fd49b9a0ae5c0ef55e61bba6ccfd0445ab8f73","anchor_positions":[7,15,23,32,40,48,56,65,73,81,89,98,106,114,122,131],"anchor_positions_sha256":"24769466c0c30738992bfaea31b322a83686427b8c8c2f42d5977ce98da1e86f","canonical_id":"945","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,31,39,47,55,64,72,80,88,96,104,112,120,129],"fisher_boundary_sha256":"98d4c6b32acad28c83342cab67b79f665cc5144d363b8e6653d190aedaddc45f","horizon":1,"input_positions":[8,16,24,32,40,48,56,65,73,81,89,97,105,113,121,130],"input_token_ids_sha256":"5d42f41e2436cb702ed1f8eb971b20537d123d7e1f41409244fd8dd8129656fc","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,33,41,49,57,66,74,82,90,98,106,114,122,131],"target_token_ids_sha256":"9a4cf735c9ab0560b97058fccd7a2d08e1a7cc3d126701d11007e641f736719b"},"formatted_content_sha256":"aa6feb35c931bc49027c89402eb6fe154ef150b92030a5f3e730756026205a4c","generator_receipt_sha256":null,"identity_record_sha256":"3c49ec4e6136f525892d68d4cf7267b90d1836a9011dc55e73ece42bc882546e","prompt_token_ids_sha256":"5766712b5c83365901b89092414948a16c2a894641c0580978a6720d74c7bb4b","ruler_category":null,"seed":null,"selection_rank":0,"selection_sha256":"006481602de32e6484f069e30640315b4902f198bc2b977318b0a90084e2503d","sequence_length":132,"sequence_token_ids_sha256":"91c8172e893e28a3e7c965916084be5fd76619bd9dc7881f155bfceacf710738","source_content_sha256":"dbd241865f0bf98d18ba09369665294382ef86e1be0440c9b18c083d5cb8529d","target_token_ids_sha256":"16acd83d1b28de14a0fcfcbad7f9a2538e0df11843ba645c1a55bcfeb285cecb","token_span":{"cache_exposed_start":132,"cache_exposed_stop":132,"prefill_start":0,"prefill_stop":112,"scored_start":112,"scored_stop":132},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2efa60ce513767dd377aebb493099ff3ebdc8a0362f30593887e54997aaa3ffc","anchor_positions":[8,18,27,37,47,56,66,76,85,95,104,114,124,133,143,153],"anchor_positions_sha256":"e227da918c51f69b24ec0ca4e8d85817a5330fb6e593d88068dce4e00d5d6d1c","canonical_id":"794","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,18,27,37,46,56,65,75,84,94,103,113,122,132,141,151],"fisher_boundary_sha256":"f690f6f75e34bb40e62a6b7fc6b3bb68573fc9ae30bacc4e60a641b17ec90391","horizon":1,"input_positions":[9,19,28,38,47,57,66,76,85,95,104,114,123,133,142,152],"input_token_ids_sha256":"bf4865a5ba31ed871b7bd8e3db64daa50d06ca08cbecab0fe5005d6b1a8347ad","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,20,29,39,48,58,67,77,86,96,105,115,124,134,143,153],"target_token_ids_sha256":"68567590e4faf2baa82a41a27758544c2001b1f8b881cbca709b0b85b1bb0ad4"},"formatted_content_sha256":"d8535b9c9deb650f87372133fd68f93792a2190a2ef9fe56ca55a611be6956e0","generator_receipt_sha256":null,"identity_record_sha256":"19a48bcd8ab87c826d9cf5e9160f127b2f1f1be8e45a068c8e44863d1648ee1e","prompt_token_ids_sha256":"b21de4f99224333b5f102dbb403f23e8367304e199e27380c91e192867b7cc1b","ruler_category":null,"seed":null,"selection_rank":1,"selection_sha256":"0172341ebe7bf74bb15297781b17837005ed9a9efbcb89d7bac34d2b25000aba","sequence_length":154,"sequence_token_ids_sha256":"b4da2809a0dead622eddd5200a89a8d4ffcc43b2c108c805e70f4449f813d222","source_content_sha256":"27e38eda1acb40e360eceaac200a88d7b2850555fa05a1efc123ec8c4a915f21","target_token_ids_sha256":"348ffb18662c2da5694709279e9caf22da5687654401cf9aa365e6382bba0ba8","token_span":{"cache_exposed_start":154,"cache_exposed_stop":154,"prefill_start":0,"prefill_stop":101,"scored_start":101,"scored_stop":154},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e0ef0af8afe4b6c4a81f3cabbd81a8ee7f213579fe41ab99f6103d4b0219b7e6","anchor_positions":[9,20,31,41,52,63,73,84,95,105,116,127,137,148,159,170],"anchor_positions_sha256":"fa5efe05ea2fd4bb11defcbff4d3edc28a1c8dc11911a2450816825e972052d2","canonical_id":"657","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,30,41,51,62,72,83,94,104,115,125,136,146,157,168],"fisher_boundary_sha256":"68596f206280b567de8e4d47c206f7bf68a9bb74b4435cf78f400f9f1ab7d29d","horizon":1,"input_positions":[10,21,31,42,52,63,73,84,95,105,116,126,137,147,158,169],"input_token_ids_sha256":"e3067d1464515f4ac7220f562d34c1c90289997d9fb5fefc653bd53f5f81f18e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,32,43,53,64,74,85,96,106,117,127,138,148,159,170],"target_token_ids_sha256":"a85ac1c61455daf925b47063d1ff1f66b29773ceec176e8263f5e6f25a1b3ea1"},"formatted_content_sha256":"058b7f8c5600efa023e5b2daf39dfaf3c3ed7a333e538e1af40e7b0b66c74221","generator_receipt_sha256":null,"identity_record_sha256":"53d315ac0c1c29a162fde0f0fcec7febb3700ec10fef28e6ed4749c118c383e8","prompt_token_ids_sha256":"d229c54edd42db7f254ae246f41c7e3e4207ee238c734a8f628e2b15ef6d2ea8","ruler_category":null,"seed":null,"selection_rank":2,"selection_sha256":"01c4ac54c87a838a046990fc691e469914fa8fffc3bd5230e6a5954f963ce09b","sequence_length":171,"sequence_token_ids_sha256":"2cc570399bb13417211fedf4c35131117f7d22b47aa4bd94e603f6303bbecf3d","source_content_sha256":"87238f15d4968d72a1caf68b3a7d6d09014ca1cd79cd8a42053ba1d0df51fe15","target_token_ids_sha256":"470c291e9349adfcd9620fddf232ec6756cdb6300d975612d2a94ad69e7deca4","token_span":{"cache_exposed_start":171,"cache_exposed_stop":171,"prefill_start":0,"prefill_stop":76,"scored_start":76,"scored_stop":171},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1a950a1c62dc0a9cadc655d3670aa54a1e6cb647d9a1892093d06959656b3aa9","anchor_positions":[19,40,61,82,102,123,144,165,185,206,227,248,268,289,310,331],"anchor_positions_sha256":"42314f586870267235ff8a21de13271c7bc12dbde0efe06732f5f12b3dd32240","canonical_id":"702","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[19,40,60,81,102,122,143,164,184,205,225,246,267,287,308,329],"fisher_boundary_sha256":"1c176e9d1164044fc7633185803af174aed6b6dde3205b6a93344da5dde70a67","horizon":1,"input_positions":[20,41,61,82,103,123,144,165,185,206,226,247,268,288,309,330],"input_token_ids_sha256":"6439978ab1e3336355d0e247c88f80b5fe217a10e6f08dbf7d49232dc4526902","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[21,42,62,83,104,124,145,166,186,207,227,248,269,289,310,331],"target_token_ids_sha256":"1e85d0fddb9002f2699f987057a43eb9c49e9fa5ef1f342f1e03060dd826b7df"},"formatted_content_sha256":"154dfb8b73bb69a8b67592d334c059c4ce3fa2a8dfe9716538e93896567d0906","generator_receipt_sha256":null,"identity_record_sha256":"08e19720136cb0b9e54d61f609f904ecacefee69429d33d7a9794d8b46085bf5","prompt_token_ids_sha256":"bddadf4e931724e7fd559f3f6b3d7128e8ca27b120efc2124cf05a63e7b2f6c4","ruler_category":null,"seed":null,"selection_rank":3,"selection_sha256":"01d2ae83b6bab88a4805a43141ed140fa1eb40a0d24f5579edffc9db44efe0e9","sequence_length":332,"sequence_token_ids_sha256":"91c517b2851f293a1c925d44556ecdd49cf6fc1874d095521f4dba829c5d5237","source_content_sha256":"97404d0bd18d3e30402ad3b46bee420986959afd67ef272b725a56c88634eec4","target_token_ids_sha256":"c94cc439fcffac9c155039ed067b4cf6d4dccca08e579f728aaf004b72be2d8e","token_span":{"cache_exposed_start":332,"cache_exposed_stop":332,"prefill_start":0,"prefill_stop":151,"scored_start":151,"scored_stop":332},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3ed130f8f695b8d6eb298a7742bd1582483350b3fd739da368175309b21ef0b1","anchor_positions":[8,18,28,37,47,57,66,76,86,95,105,115,124,134,144,154],"anchor_positions_sha256":"dda22fb55d9a1168e0db21fb1e43033ab66ab6d1c06af8fc27c118e4671afeeb","canonical_id":"651","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,18,27,37,46,56,65,75,85,94,104,113,123,132,142,152],"fisher_boundary_sha256":"9efbc09f1ef4234ec391fd7fee3a2a47a03dcf28666c0aa550bfc04ea6bcde0c","horizon":1,"input_positions":[9,19,28,38,47,57,66,76,86,95,105,114,124,133,143,153],"input_token_ids_sha256":"b5b770374e871c6af58aaed89d7fdf049c3423bc742e960e5f9a6f4e991faf81","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,20,29,39,48,58,67,77,87,96,106,115,125,134,144,154],"target_token_ids_sha256":"8271372b97b8f4205c2218769701468573205610c8d62b3474ad0d357b160840"},"formatted_content_sha256":"1ccb4c308ca379db3003d12f2a08486a59d19d9a104a1208f6f91af5104409dd","generator_receipt_sha256":null,"identity_record_sha256":"aa519005f371474709ebcac572071735625e58adcc2ea5484d987a610bd03f36","prompt_token_ids_sha256":"d3dc05d28fe582fdfab921a7ad3bac6d80d1d9368804e5955561df1f18f29e1d","ruler_category":null,"seed":null,"selection_rank":4,"selection_sha256":"030870e9654f434f272f49b61535aad38a9c0eacc093ccedb4b1328a163d00f7","sequence_length":155,"sequence_token_ids_sha256":"b4fc5ea0199d736cffaba0bfb4e5db058abd8417d844063350138206ca085d62","source_content_sha256":"c093103b7e2b0fc0cff523616e4dd14264349d2add37f8cc2e248dbd630be4f3","target_token_ids_sha256":"39e1f51bdd6ad42a474c0dde935c2ac6a6696b58b1ccc288e77f6bb849d587ca","token_span":{"cache_exposed_start":155,"cache_exposed_stop":155,"prefill_start":0,"prefill_stop":117,"scored_start":117,"scored_stop":155},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"30f57c28b1d21a916283d0e281b04bbb0087858a98279771835d257c5187478a","anchor_positions":[17,35,53,72,90,108,126,145,163,181,199,218,236,254,272,291],"anchor_positions_sha256":"8d9aa0aedd74a7302174328c9e98110efb9351fb25a7e9f001ca338898298fc3","canonical_id":"720","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,35,53,71,89,107,125,144,162,180,198,216,234,252,270,289],"fisher_boundary_sha256":"c47ca1eef3f1024a8ccaaa6a3156efd81373c0981d1b7794d0e74a6a4596fa0e","horizon":1,"input_positions":[18,36,54,72,90,108,126,145,163,181,199,217,235,253,271,290],"input_token_ids_sha256":"699513d81a38caf1a3166032551a6daa9d12868c8369d97465ec8d642a321141","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,37,55,73,91,109,127,146,164,182,200,218,236,254,272,291],"target_token_ids_sha256":"12845035b9a29a7b08b584abc1f28701bd58dc87b1360013b830d1863deed34d"},"formatted_content_sha256":"08569657d1fd1af97815b4671f162ad3cb646b9db6f4ade1b53b237267f3360c","generator_receipt_sha256":null,"identity_record_sha256":"4c68c0c7fd3b4f5ef68188ca20cfd016414d07212968996c98d64d097d36f013","prompt_token_ids_sha256":"41dcd0cbf8f176b6eabaa2189a2cc9dad19e1d0e1b7466a57fcee4fcd8695223","ruler_category":null,"seed":null,"selection_rank":5,"selection_sha256":"0331ac5ad147dee4ef983f58ff124f388c082a848eb8f7a1e2df5c58e3096b83","sequence_length":292,"sequence_token_ids_sha256":"f0361fd70f4b1f9d2a2cbb95c296333e1e9f146ad837ece7cb669c2993a27bab","source_content_sha256":"e916cd640a7665ee439bd983ca0769129de74af589e7e5eba3b559f7ea146c60","target_token_ids_sha256":"bf0037e9004b38848b545334c18c95d9234edfda314a7fc61ac5ba9fd36b25af","token_span":{"cache_exposed_start":292,"cache_exposed_stop":292,"prefill_start":0,"prefill_stop":240,"scored_start":240,"scored_stop":292},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"076f7a7d27bd0edfad63c949645fb3eea8f9d8d32c238b4af19053c5407a8b2b","anchor_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,160],"anchor_positions_sha256":"cea919870992245499fb8d8a858d848e1e76f02a4e2230ada935442533c03198","canonical_id":"903","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,18,28,38,48,58,68,78,88,98,108,118,128,138,148,158],"fisher_boundary_sha256":"dfb2af41b7aacbbc1cb03c5a460d70edf42a188290ce3b01ee741cbb663a7aef","horizon":1,"input_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,159],"input_token_ids_sha256":"8e0cb14d6334e3493c75e96c63848e991fa05b6bc027f04b99b09dd2cbddaf7b","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,160],"target_token_ids_sha256":"6800fb22eb04b0dcd2b08692779155cb065925a066e2f97f52600c51c38ed71b"},"formatted_content_sha256":"f296f132da28a9fa5eeea0a099d9ad97955044e13b9fe30f6d9300ff9e041c78","generator_receipt_sha256":null,"identity_record_sha256":"a64089c9832bde6f4f730dea47c3c93a3ac71dd538c45b2169607180571708d9","prompt_token_ids_sha256":"a454237da9066ead36df97766389f9b28ac1b5df54bd0d9073baa9459aae696d","ruler_category":null,"seed":null,"selection_rank":6,"selection_sha256":"03a537026beb3ae778851b82f172083dc21eb07f0ce7d1683756019ad86e6d98","sequence_length":161,"sequence_token_ids_sha256":"86cdcf0d01cc99b913a8495e4074583c23687d2a77e68c6b0cda5700932a8937","source_content_sha256":"edfb4574ba6b7cb41c0f0d1dc6452e8ac7fea922b120c3ad057a2e77b15fb36e","target_token_ids_sha256":"5fb56e9b4dfe8c41e4ff0043461c0b44248cf165cac01b9f42beb4f0e756f516","token_span":{"cache_exposed_start":161,"cache_exposed_stop":161,"prefill_start":0,"prefill_stop":83,"scored_start":83,"scored_stop":161},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"6ed1f32d878fa277d86681ed3b0a9e7b9b172c3cf90d4213020217eb3296cf68","anchor_positions":[13,27,41,56,70,84,99,113,127,142,156,170,185,199,213,228],"anchor_positions_sha256":"e3fc0847eae8451ba50c7fae586b6479de5eec6dd1bcdf44504db22f0d700d62","canonical_id":"918","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[13,27,41,55,69,84,98,112,126,140,155,169,183,197,211,226],"fisher_boundary_sha256":"53c76b32efa019cb22ef480e27bc5b9dc7c12c0683b0d1b8ad250ac6571380a9","horizon":1,"input_positions":[14,28,42,56,70,85,99,113,127,141,156,170,184,198,212,227],"input_token_ids_sha256":"14e214e41d049733a9ea7e8c02fce42e7aba76edb16178e6d3405bad4c23055c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[15,29,43,57,71,86,100,114,128,142,157,171,185,199,213,228],"target_token_ids_sha256":"24028e01155b1e01eca8ccfc7c2080a6a56b4a3e7c15839043e13b8a00a8ee8b"},"formatted_content_sha256":"f4ed4188dc61ab3e90526a061246e459b1d5fe2fc68d01226a124501eec0be2b","generator_receipt_sha256":null,"identity_record_sha256":"4f00e2535ec2aa52070ed18120472b9d45845790809ac1fb5a7a26a8f6b51d20","prompt_token_ids_sha256":"d8f70964c5c840cc4c74b02a5c432adb08de3a38f706cca7ff9f50f513f616e7","ruler_category":null,"seed":null,"selection_rank":7,"selection_sha256":"03f2b70cb41aa2df2ca06599a3740907c1bd89976be4f7d79613cd3428eb75d7","sequence_length":229,"sequence_token_ids_sha256":"b42ae62c8481ac868fb8f833b03b35ab7f84490d87329022e4548df9b8a49299","source_content_sha256":"1c74c6e9767910ee576993d40b0be7e4f8a3653223ce943b2b48da881f043551","target_token_ids_sha256":"c0130989066437b78efb8eb68c747a3645ecb631cb508775b1f7b001deb4e1e0","token_span":{"cache_exposed_start":229,"cache_exposed_stop":229,"prefill_start":0,"prefill_stop":96,"scored_start":96,"scored_stop":229},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"32d8375f975ffd5157bb123bebf58d926b2fb6e715d79b1c29711866e8ae1d7a","anchor_positions":[11,24,37,50,63,75,88,101,114,127,139,152,165,178,191,204],"anchor_positions_sha256":"48082390de232d2e911509462169a4b3ebe8a03329ea579a66fe38623c52b3bd","canonical_id":"823","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,49,62,75,87,100,113,125,138,151,163,176,189,202],"fisher_boundary_sha256":"ca339ac21962b12f69ea2e43cd5efc5fc9f045e43e311f236cff0f3fc7a9532c","horizon":1,"input_positions":[12,25,38,50,63,76,88,101,114,126,139,152,164,177,190,203],"input_token_ids_sha256":"9060d0dc9e82be8da8cc47571caa8e03245ea38a15689377bf15f3403304608c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,51,64,77,89,102,115,127,140,153,165,178,191,204],"target_token_ids_sha256":"9c64639807a2aba57e44b252f7c9204c7994ff505668345c293c352b795673ab"},"formatted_content_sha256":"72e3934a85cf2d4cb4b035ef066f9a69edbe304668167f633c705ac5a7c3ee74","generator_receipt_sha256":null,"identity_record_sha256":"6775751ca115709c8dd0187f2d2309b90237ea599613ee56420cc077c5a86d67","prompt_token_ids_sha256":"a7322a6a594060b8983da2adc969063889ad65c9524cfc9d9ace5e4856362612","ruler_category":null,"seed":null,"selection_rank":8,"selection_sha256":"04318f2e0a32de485412bc0aa95414984aabdb5973974b77667909ca919923cf","sequence_length":205,"sequence_token_ids_sha256":"c39424dfee203bd58c46a3bbaafc51b57eb8c76c9f8ad175fd4355d227510f19","source_content_sha256":"7be1174fc06994281445e57b76099586c2214066f9358928e22281d793715d3f","target_token_ids_sha256":"c0ca3637b8b1da8af549e87df7bf8e67da2eb444227889016f2678ec5d8b420b","token_span":{"cache_exposed_start":205,"cache_exposed_stop":205,"prefill_start":0,"prefill_stop":120,"scored_start":120,"scored_stop":205},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"bb690c1557693c9a1151a55a056b413abe207a0ede6130a1c4eee76cd7832910","anchor_positions":[8,17,26,36,45,54,64,73,82,92,101,110,120,129,138,148],"anchor_positions_sha256":"34299c685b72af2a076ee41a9cc49b613c0e3c8039202f3412e50762385cd13f","canonical_id":"962","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,26,35,44,54,63,72,81,90,100,109,118,127,136,146],"fisher_boundary_sha256":"f483b9a7b54d8e98c4fb81ac8e8c508a85638477f08243915cc53d3ee584a92e","horizon":1,"input_positions":[9,18,27,36,45,55,64,73,82,91,101,110,119,128,137,147],"input_token_ids_sha256":"562a3beb47bbf20a72e84b88e739d67bf0a6fd83ea3ec4a06ce7d8527bc32c9d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,28,37,46,56,65,74,83,92,102,111,120,129,138,148],"target_token_ids_sha256":"8d91bcd0f3417c6e6ff0e30b8ef2ad4a5200ab497081407b0de689fac826fe32"},"formatted_content_sha256":"35f9c6bf72383681061847ec15dc2a4d69a179546f872d72042b1dbeb7eb3c94","generator_receipt_sha256":null,"identity_record_sha256":"7f528a9f6b6f25735216e2ddd0cf3bc970efee42f4ddc18607d7615c585b5da5","prompt_token_ids_sha256":"ad2690a82eb727187859d0918f78622936f9220218fa708cc305b79f9e3129b2","ruler_category":null,"seed":null,"selection_rank":9,"selection_sha256":"053a8089119cb6c0a1699960c729b5d0bbf904347c2611f2bdbc20d68ac98896","sequence_length":149,"sequence_token_ids_sha256":"39192da46d67ba8a6a1aecc11d2517cb72d4adbd21944f31d854f4aebde4f950","source_content_sha256":"6d510f0a092393c876a9d3827db9cccd4b422742e08dfee9a9094f3786b67fbf","target_token_ids_sha256":"24d4ddf4dc36fb254790754f2826967503025402abfe4dc1c81a57b608e358c1","token_span":{"cache_exposed_start":149,"cache_exposed_stop":149,"prefill_start":0,"prefill_stop":87,"scored_start":87,"scored_stop":149},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"510bbf0acc641074d0ed15a7484a726b97a35b663fe88ff076ca0808ea574c8f","anchor_positions":[5,12,18,25,31,38,44,51,57,64,70,77,83,90,96,103],"anchor_positions_sha256":"307c6a1944f2d94c59465c6af96e251aaf0ea665f8f1a3e0e55e3eb0568d8551","canonical_id":"641","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,11,18,24,30,37,43,50,56,62,69,75,81,88,94,101],"fisher_boundary_sha256":"74d1a793166b47319560030b1ea4243c84cca2ef770c4bd539a7069d57a6516c","horizon":1,"input_positions":[6,12,19,25,31,38,44,51,57,63,70,76,82,89,95,102],"input_token_ids_sha256":"693ad260a9464069c62f03bdc9f97044efaadaf73fba426e3993f726d373516e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,13,20,26,32,39,45,52,58,64,71,77,83,90,96,103],"target_token_ids_sha256":"24fc6cf6c761d9d2d39f6a0e2d1988fd8c0568a28c602b83a298a2efe14f5b75"},"formatted_content_sha256":"0d5d8389772326b9a23b37c5d64023b87ff76167c738f0f04a73f350d75146a1","generator_receipt_sha256":null,"identity_record_sha256":"d257e6e664de7884c47c6c8efd30d9049f908be0c1f0f15604c5b1e680278db7","prompt_token_ids_sha256":"8b73365de43af24e2b250e80fbb6c87c821ce915961cb60bb5f92dd492aa4e25","ruler_category":null,"seed":null,"selection_rank":10,"selection_sha256":"0561b3053aedb5bf75595d41c4c1d56dffc9d7d22d4f594c77acd661612fbbda","sequence_length":104,"sequence_token_ids_sha256":"edaafb16efe7861b6641b4284c1b03cabf4553b33c20400911590227e2717873","source_content_sha256":"b4af7687b1fdeeba11f23fbb43c11d07ffdb4918405f0574dba17f6932087d51","target_token_ids_sha256":"10c4c8e1ab1664e434fe26b8093fe1929571437707a4a4c5082c71bb6879f108","token_span":{"cache_exposed_start":104,"cache_exposed_stop":104,"prefill_start":0,"prefill_stop":80,"scored_start":80,"scored_stop":104},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"77e62d656a470f85ca90c98dfc91b52de62fec0d714ab54d9d78b6111588449a","anchor_positions":[16,34,51,69,87,104,122,140,157,175,192,210,228,245,263,281],"anchor_positions_sha256":"d3c1ef41c3fc59649a4da8be56fc51bf8a16621374c8218a7236e1c502a9ed29","canonical_id":"821","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[16,34,51,69,86,104,121,139,156,174,191,209,226,244,261,279],"fisher_boundary_sha256":"6ce814085aa258c14254daa52b6c87e5c35bb95bb9ce43819236089ed670a938","horizon":1,"input_positions":[17,35,52,70,87,105,122,140,157,175,192,210,227,245,262,280],"input_token_ids_sha256":"52e2a2814003d5a0c9558d2807c349140c1daee27300a41a386f98243a887244","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[18,36,53,71,88,106,123,141,158,176,193,211,228,246,263,281],"target_token_ids_sha256":"a841d33aff758b3642820a6e5c0b714add3840501814ab31d90f6dc5b41336dd"},"formatted_content_sha256":"9b4c39a7f47e7c5485594bfc1f0bc255be70f74e8e73be2bfab32e77bd11cb37","generator_receipt_sha256":null,"identity_record_sha256":"bbe20a7f23dfdd53afdd97893a491105ab3d0d770f28da4cafd4d0f760a10e07","prompt_token_ids_sha256":"8438715e3ee27d1c3e5e158afeb22e85f9543635b4c24c7151828b62262082b1","ruler_category":null,"seed":null,"selection_rank":11,"selection_sha256":"05ca9ce71ec3ccbff2dc9051e6519ff621aeaf051f13006ef2a903db53cfa322","sequence_length":282,"sequence_token_ids_sha256":"d4a708e94f01dc74ebdbaa4a0850da22f19f8c7616df125de0fa93a03cd941d1","source_content_sha256":"6cdab710688743670db0377d5ff8806298470ee86aa94a2f4fd430f2542e26b4","target_token_ids_sha256":"52fee9ad2633a9725f62006b3a2a598f494e32fd6e12dcf43893a911e7032a91","token_span":{"cache_exposed_start":282,"cache_exposed_stop":282,"prefill_start":0,"prefill_stop":246,"scored_start":246,"scored_stop":282},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"88e7f4c26333024cf786b4c1d640f43c25aadabb8055c4006984b0b773283a63","anchor_positions":[6,13,20,27,34,41,48,55,62,69,76,83,90,97,104,111],"anchor_positions_sha256":"70a9c6d084ac7d3e411adc69adf277b706895b48989c28a96164561595a44b13","canonical_id":"882","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,19,26,33,40,47,54,60,67,74,81,88,95,102,109],"fisher_boundary_sha256":"af33587dbbed535e25e0f9ff381dc426652e98f76482f907c3a1b91a43afe65b","horizon":1,"input_positions":[6,13,20,27,34,41,48,55,61,68,75,82,89,96,103,110],"input_token_ids_sha256":"2e82b13f62352679ed5dae9b03694e1c2eeb56e00f762a0373672ed48440f408","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,21,28,35,42,49,56,62,69,76,83,90,97,104,111],"target_token_ids_sha256":"d9c7534eb6dbff16502f2748b6ab6576ab5d76645faba1e8bc35a67323be8c39"},"formatted_content_sha256":"8164f8fb3728db418776886f3233778e01a1c5b7e54b4f350ff9c98d6f4c3018","generator_receipt_sha256":null,"identity_record_sha256":"79d822ab45c87b51b8aa2c0ecb29bfbbd85453f82f6a3e764bec8805d2f9708b","prompt_token_ids_sha256":"4c61f175c320b5f738697b92438e1bb158f00bd6a061e49a529ad3158d4f02d5","ruler_category":null,"seed":null,"selection_rank":12,"selection_sha256":"06182bf7b5db58b26f203c4f99a7fb97d3252f5d92dd5f4333d47e49043b0f9d","sequence_length":112,"sequence_token_ids_sha256":"a69b8ee713bb9e4fdf0cdec61d49a10f148243a2cf8fe92a4420823b82bb3839","source_content_sha256":"f9edf8d912223ba1c68389438b5a930e3ff8116a79ef7e9e4ab5b7f48cad2e1e","target_token_ids_sha256":"b9330c3608f52265a4d54352f0d4d3821513a3adbea5b89bf9f5146143a012fd","token_span":{"cache_exposed_start":112,"cache_exposed_stop":112,"prefill_start":0,"prefill_stop":90,"scored_start":90,"scored_stop":112},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"026a11f123c6c02b349d154973681d1484316586c77571635cb75396928e8801","anchor_positions":[8,18,27,37,46,56,65,75,85,94,104,113,123,132,142,152],"anchor_positions_sha256":"75c1fd0cbacbbfa75bfedc1647ca1a15436ad446c2021b146a8d7af4567321f4","canonical_id":"874","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,27,36,46,55,65,74,83,93,102,112,121,131,140,150],"fisher_boundary_sha256":"69de8bf124376d38cfd76f8801bc74e0e352d8cd33f8dc6b34cb10e4c6ff73d3","horizon":1,"input_positions":[9,18,28,37,47,56,66,75,84,94,103,113,122,132,141,151],"input_token_ids_sha256":"75967a14501363b155570c3d4d66959cde5cce50c53c8fa1fadf71f39e4218de","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,29,38,48,57,67,76,85,95,104,114,123,133,142,152],"target_token_ids_sha256":"38400510781c2bec0042957302068be57d0438f8b57f3a4a775204a7ae26b03b"},"formatted_content_sha256":"8ee29791043cd7726bdd815083a307f0d693443f77d0fc9af6c9249422f82a60","generator_receipt_sha256":null,"identity_record_sha256":"5f505f1a07e986bb3a69f547553e68edda687e89817230981e3d65a20f3b5d61","prompt_token_ids_sha256":"af780c9a24c687de5d7bb86c8de2e3a05d07e1e769e2bed9aef9bc82e0c2b3e0","ruler_category":null,"seed":null,"selection_rank":13,"selection_sha256":"06e6c099b79a31351c8aa115e8c35681452320d265ba3e7c9deb647189b5aedf","sequence_length":153,"sequence_token_ids_sha256":"059517b1e6703c7276b532d66d5dc53b137178142a0fbcebbf7a85ed806bb2df","source_content_sha256":"7be81d34f54ea71fa53a94438dc656b1e14cf83c1f0cb8b85a03aeec675a95cb","target_token_ids_sha256":"a00664d6a993b427df9b77218f9f6ab241e98e255c2754127a6aeb30818bf82b","token_span":{"cache_exposed_start":153,"cache_exposed_stop":153,"prefill_start":0,"prefill_stop":82,"scored_start":82,"scored_stop":153},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7e8e992a57a2c58864e789d43316ec4ca5557685022402a123eb7bec86e0a31d","anchor_positions":[8,17,26,35,44,53,62,71,80,89,98,107,116,125,134,143],"anchor_positions_sha256":"486e7f47d72e2b3d4a862156a2b41e9dbb05a70d78349fdcac0b01e09667750a","canonical_id":"768","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,25,34,43,52,61,70,78,87,96,105,114,123,132,141],"fisher_boundary_sha256":"2f57f4c3f66cd0d31d7220f66c2563c2f42ed9d860ccd032cab629a4a62b2d7d","horizon":1,"input_positions":[8,17,26,35,44,53,62,71,79,88,97,106,115,124,133,142],"input_token_ids_sha256":"951070a8ec9f2842c988081eae148ff3e650f6e041bad423eb8cf80577aa09ca","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,27,36,45,54,63,72,80,89,98,107,116,125,134,143],"target_token_ids_sha256":"be7a411acd1f8c53e98dbbe661a8f0e08925a5287a3a082d567b4348f5daea4a"},"formatted_content_sha256":"19fb2a15b1650f22847079c098721ed0b4ae36db1faae49cb10e7f8b7f8a951c","generator_receipt_sha256":null,"identity_record_sha256":"4f37dd32d3554ba3f0e3842355c0812d1532bf5eeb4d09edf28e778ec744907b","prompt_token_ids_sha256":"5abd62e58720f9b71da73a2f319d99d1dd3f0fe89c148c1e335674d33e0db6bc","ruler_category":null,"seed":null,"selection_rank":14,"selection_sha256":"08d405d25f1c1656f2532ca7d15875b63c46516c4e46a564326b3696f5943347","sequence_length":144,"sequence_token_ids_sha256":"d924aa1344bedd7c5de7beb28914b5f02d01c9242e93916c3ebf5e13a1b777e2","source_content_sha256":"96a4ff323c23dc1eba72d4b800a65cd77f46bad4c7b017e7716f69b066cf251e","target_token_ids_sha256":"2bb90e5895fb397d1b3239e2f50aab932ff08813656890b44cefc5327ce9f0fc","token_span":{"cache_exposed_start":144,"cache_exposed_stop":144,"prefill_start":0,"prefill_stop":79,"scored_start":79,"scored_stop":144},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2a32d45497a46d8765c2bef47acddb45c96e41bdb29282abb99a946dd86db58b","anchor_positions":[15,32,48,65,82,98,115,132,148,165,181,198,215,231,248,265],"anchor_positions_sha256":"0348f2e9e936ebff365da2919e86c0d7b23f030deacd2557e2975ff2c1bb6238","canonical_id":"635","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[15,32,48,65,81,98,114,131,147,164,180,197,213,230,246,263],"fisher_boundary_sha256":"bbb2193a791665a372d6f0871c8b972270ed33e5afe4d627b3f13c69caa5172e","horizon":1,"input_positions":[16,33,49,66,82,99,115,132,148,165,181,198,214,231,247,264],"input_token_ids_sha256":"54ae9eda3529af89c8491dfa14c002e2bbbddacc5f7fc9bd8f98dfd841106b17","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[17,34,50,67,83,100,116,133,149,166,182,199,215,232,248,265],"target_token_ids_sha256":"8e9dff8e5415e27d61633f2fbae7b6f5a404922b80ae0dc2d994004e0c00ff77"},"formatted_content_sha256":"252812434def51f2af7fe0f324a5922cddb2523bd81d2dfc65720c67ba9d9871","generator_receipt_sha256":null,"identity_record_sha256":"e2908b95f86b0b763a5d77e597f5d8cc0e9ff3b96e4129645e8ee178160dc9e8","prompt_token_ids_sha256":"99adc1c15eb492246a3ad45eb9cd6dc0a5f949c95b99aa332c2c8fe742dac882","ruler_category":null,"seed":null,"selection_rank":15,"selection_sha256":"09bdeb42fd2df571b9c13e3aecc461eec43355320ee267a53066bcbb23769161","sequence_length":266,"sequence_token_ids_sha256":"2ca3fac260d1b58347c86dbb21f794efef0df7c42a0de4d942d549481c583b2d","source_content_sha256":"b3bbd3ae7ff06e057b52c17af8b75ac83d20e8b436ca1c82f6f7ccf94d0a9214","target_token_ids_sha256":"b44b18d625486bf3a519bc0a87d4dbfbfb4851798e7e50bacb07a66cab69939c","token_span":{"cache_exposed_start":266,"cache_exposed_stop":266,"prefill_start":0,"prefill_stop":214,"scored_start":214,"scored_stop":266},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e6416dbb61ce7ce16cf3fff7ce25a1bacfecbb3dbf2fde6f06d8fc6699998cd6","anchor_positions":[5,12,19,26,32,39,46,53,59,66,73,80,86,93,100,107],"anchor_positions_sha256":"b9647c4baef8fee686d8a882e4d530418b6f7cc8044a0a83370fa394c3bc5ef2","canonical_id":"666","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,18,25,32,38,45,52,58,65,71,78,85,91,98,105],"fisher_boundary_sha256":"1a644132ebeecd04edf2c98217977f7fcbdedbec84e64b6b50a88a6151232772","horizon":1,"input_positions":[6,13,19,26,33,39,46,53,59,66,72,79,86,92,99,106],"input_token_ids_sha256":"32f5ac39d606f1ddf6a4e4346735931cd6faad7feec35e950f17b8edbcb045a2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,20,27,34,40,47,54,60,67,73,80,87,93,100,107],"target_token_ids_sha256":"eed792902f0c9e240b65aaacd0467e2ff2f4a7eba83a333214fd67d942feac55"},"formatted_content_sha256":"e3a3c1e1ed1b292d4877ff78da3ca92b8112f6b28b41a2eb63f18c8d72057088","generator_receipt_sha256":null,"identity_record_sha256":"ec5fb87c296ab7d9fc8671afd53e5bc11fd05c7ee0bf20f88d758453cb95da2d","prompt_token_ids_sha256":"92348f208743eebc810c3eeffd27e797bfbebe693b75b29192878c2766c17cb5","ruler_category":null,"seed":null,"selection_rank":16,"selection_sha256":"0ad2bb04e64cf7bafe43963d9c975c7597c00f97a065f172225e36b8de1be401","sequence_length":108,"sequence_token_ids_sha256":"05aa7077f2114bfc39c337134a54171570e1bf7f5e40e875dfd4ccc76cd3c5bd","source_content_sha256":"c5657a0fc079e76b566db9dad6a041aa51a885e91a3ce60f4b56e268d4f2283e","target_token_ids_sha256":"34eea4ea001c4d78cc33cc3368d1d71873c37b7e910507eceb9597f35255e3ef","token_span":{"cache_exposed_start":108,"cache_exposed_stop":108,"prefill_start":0,"prefill_stop":69,"scored_start":69,"scored_stop":108},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b641a12066174c147da77baa5576307cddc4326d490da447977a9880c63d2be5","anchor_positions":[19,39,59,80,100,120,141,161,181,202,222,242,263,283,303,324],"anchor_positions_sha256":"229481c1cae947d75547887104b565fe100ef01406aaa1ae33b6680fa4942d03","canonical_id":"795","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[19,39,59,79,99,120,140,160,180,200,221,241,261,281,301,322],"fisher_boundary_sha256":"075b4606ad628cb7bea3693f59ccdb0efa6c6599cce58a6552534c47baf39ab1","horizon":1,"input_positions":[20,40,60,80,100,121,141,161,181,201,222,242,262,282,302,323],"input_token_ids_sha256":"23b5ebefaddbeeafb86642c66656119a5122ac4590adfc291caa30b36d9aa0f0","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[21,41,61,81,101,122,142,162,182,202,223,243,263,283,303,324],"target_token_ids_sha256":"67fe7ce1a7b687bd4b008457e44ac3e68602c818d9241e2c029c21d035fb039c"},"formatted_content_sha256":"ba441afe1323472e606fbc265ed2df392c0910aa7b69c72034eb237f6f1eac12","generator_receipt_sha256":null,"identity_record_sha256":"462faa7c85aef3e98a366b53fff20c26ce10eaccece6d0cf5e990a957d47bab0","prompt_token_ids_sha256":"276d4bcbbf513e902b3e32cea17826db3f7c89340cf3d4a5944f77e4fe398378","ruler_category":null,"seed":null,"selection_rank":17,"selection_sha256":"0b64b061a95be11408788bd7d48178a22fe67b04451aacb2824273de604613c6","sequence_length":325,"sequence_token_ids_sha256":"3bed81ccd0aff452ebe2792b7538c8dab5d74a0ddf5fc9de5e67201d559dd9f4","source_content_sha256":"0c82e760835310eb8ee865c643c6bbd2fe9a92e58f43e818f9fe857fa8bb2b34","target_token_ids_sha256":"07898ad18dc9917a62d1d7d2c673264e506ffa3cd0296f61f5355d73ff74e2a1","token_span":{"cache_exposed_start":325,"cache_exposed_stop":325,"prefill_start":0,"prefill_stop":290,"scored_start":290,"scored_stop":325},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"73863421843464e12bf37b9bdf19118b2a0edae3ea6b28b7bb83ae34025fdb99","anchor_positions":[6,13,20,28,35,42,50,57,64,72,79,86,94,101,108,116],"anchor_positions_sha256":"65c4371e0db2357f72608fb79361ad62ba950063eae1f1df95567c3178cfceb4","canonical_id":"944","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,13,20,27,34,42,49,56,63,70,78,85,92,99,106,114],"fisher_boundary_sha256":"8f181fb4592d87100a01b9b5022fe28dec60a6c1a630f0253a95ef609f751477","horizon":1,"input_positions":[7,14,21,28,35,43,50,57,64,71,79,86,93,100,107,115],"input_token_ids_sha256":"bb992854b746a2d0322b0c73747a5b6e481b4ac55013af85dda8fc8c0bc4e324","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,15,22,29,36,44,51,58,65,72,80,87,94,101,108,116],"target_token_ids_sha256":"efc4327e8bc745b94eb74df47cb9d34c605ce69f88ab01258f192e2678c5c5c8"},"formatted_content_sha256":"588cb6ded28fd05719867c9a3c2ef078440822880f6fbe41587f8ab604fc41cf","generator_receipt_sha256":null,"identity_record_sha256":"a019ad696f9c4fe577d6efe7e08f14e983f53e6a023062d550dc4ffd16fef99d","prompt_token_ids_sha256":"6430dca518b59b80172b9552c64bfa32b6116fc5855d53c075e496d174d33f72","ruler_category":null,"seed":null,"selection_rank":18,"selection_sha256":"0b9a44c3022b9f3f3616c768c1535452329656aeea799dcb45a719751b3126e4","sequence_length":117,"sequence_token_ids_sha256":"a3b8d96a49a5559eb5401f9e9f9b40c0785ee84f73cf189a884618ba0cc77865","source_content_sha256":"2099f30aac30b18f4a8fa27a7204c64734c587cf50057d835eae933211f12c7c","target_token_ids_sha256":"73123ac6d971eeb25450ba23964d4d95870b3421781c2b721b8e3f5217fe1ffd","token_span":{"cache_exposed_start":117,"cache_exposed_stop":117,"prefill_start":0,"prefill_stop":91,"scored_start":91,"scored_stop":117},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"a3fb6da730653e3034c80d7d7da56cd1a27244994198f932d4b653175e07528d","anchor_positions":[16,34,52,70,88,106,124,142,160,178,196,214,232,250,268,286],"anchor_positions_sha256":"88ca5e0c73dcbbad356df2a5c320d1e65852f5459c97030e41b86ba39ef13c38","canonical_id":"653","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[16,34,52,70,88,105,123,141,159,177,194,212,230,248,266,284],"fisher_boundary_sha256":"f3b0c45ebf98940a7a54cb33ffec71487e06a0482e824289ffd032de00327ea1","horizon":1,"input_positions":[17,35,53,71,89,106,124,142,160,178,195,213,231,249,267,285],"input_token_ids_sha256":"d7dccbe6c436987756ad6166a8d563fa686494edff0e090f19a94996077cdfc1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[18,36,54,72,90,107,125,143,161,179,196,214,232,250,268,286],"target_token_ids_sha256":"8493cc8e2ef026b2ca43108aa6e7db63ec761ce211452609a1bb2760a0f66f2b"},"formatted_content_sha256":"157b49e7f32965e28ef9bb626baf1d0356b6d5f38d74f4c673a15e0198ce6751","generator_receipt_sha256":null,"identity_record_sha256":"31193a44e28b9b97aab66d0eff836463790dd908e352239fd1a3dee6e1ad0a3a","prompt_token_ids_sha256":"a3e2cc82d1e964fda655ec40dc4f4058a3f6f4adaa8b6def330531e94950c8e6","ruler_category":null,"seed":null,"selection_rank":19,"selection_sha256":"0c32b016c1fcca024f19127827b8ece87776b1df963f55614f9a15b354218018","sequence_length":287,"sequence_token_ids_sha256":"1d7f3ec55af99be19b85291e5b80c9d9555e9360cd4af99ded037154beaaad61","source_content_sha256":"e8146f7d1066326e2120f2f49e89fe13318dde45bc338d1b6a350b475840face","target_token_ids_sha256":"f4ea61620cf4a9d1cd26e8136680d5f09ad7d79a197bd08433ab22b67e683560","token_span":{"cache_exposed_start":287,"cache_exposed_stop":287,"prefill_start":0,"prefill_stop":249,"scored_start":249,"scored_stop":287},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"adeda0b376d342263631c70ee440fab7010e921ef7e9bc2997f75798d31a644d","anchor_positions":[17,35,53,72,90,108,127,145,163,182,200,218,237,255,273,292],"anchor_positions_sha256":"c72256a78f816fa7bf8e540cc3aaf3fc1795369a3859efe48a9c4a8cce0d2489","canonical_id":"857","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,35,53,71,89,108,126,144,162,180,199,217,235,253,271,290],"fisher_boundary_sha256":"f5b74e46bd51ccc196ee08a6967a1597fb09f1c1b2ba66309ed0bcb627f70190","horizon":1,"input_positions":[18,36,54,72,90,109,127,145,163,181,200,218,236,254,272,291],"input_token_ids_sha256":"555cbfa7f8498b7200297ecda4d0d95381d1e0e366a25c75a729a308b819e751","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,37,55,73,91,110,128,146,164,182,201,219,237,255,273,292],"target_token_ids_sha256":"ded8ec9b50bd3a7f2b508c6b34745e277cf32245748a23e733afeb28ab5be811"},"formatted_content_sha256":"4156119aeb73684201921a09e3d2fef7e8426ebd10babb52788c0f7e8e9813be","generator_receipt_sha256":null,"identity_record_sha256":"670b8008445ce0955e14e6aa12268d4f91c343a57d01f2618609843306761743","prompt_token_ids_sha256":"5e5ccf81697344dfc507d09f593651afc2e0d360fb85f31936aa419b069701de","ruler_category":null,"seed":null,"selection_rank":20,"selection_sha256":"0f3520a5dd05d707070cfc5723f57c992db4df4d8d98965890fa13a516a29c32","sequence_length":293,"sequence_token_ids_sha256":"ddd8c7e579f53e5c52e946378d772b93a5ed813a063da195e831d38abb0cc958","source_content_sha256":"22096f4ec39e2e444074f7f2cc3873ad56b3e27bb407587437fb05693324c674","target_token_ids_sha256":"66d83ee4ec91a16b9bd4d5c77a1b45e1850fd3594f689ead1e55c781506cfb6f","token_span":{"cache_exposed_start":293,"cache_exposed_stop":293,"prefill_start":0,"prefill_stop":271,"scored_start":271,"scored_stop":293},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"fa5b57e6457aed64b16e39b74ac6aa26eccd3fc3019570266249d9c77b045ed2","anchor_positions":[10,21,32,44,55,66,77,89,100,111,122,134,145,156,167,179],"anchor_positions_sha256":"fe99fb9f0105e4d1413d20b0bbf51aa70554cf263494d9944696f0b9ca84c89e","canonical_id":"884","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[10,21,32,43,54,65,76,88,99,110,121,132,143,154,165,177],"fisher_boundary_sha256":"7a8012c30b1d696314102eab8bd4402dcf733de71d67674552a2c672d81c2cc8","horizon":1,"input_positions":[11,22,33,44,55,66,77,89,100,111,122,133,144,155,166,178],"input_token_ids_sha256":"391312b446f027fbfa057f16da7c25100505a8356e160bc745be4b022c43ebba","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[12,23,34,45,56,67,78,90,101,112,123,134,145,156,167,179],"target_token_ids_sha256":"4f09e35ae3b70a36f49caa3032145ff047fcc34b2e0c9615bdfa4ec5effcacf4"},"formatted_content_sha256":"e2917eaf992f7bb16e2280d5b40f59553c293a9c2ff814a21cdbf4193246972d","generator_receipt_sha256":null,"identity_record_sha256":"170955cbe700df9f79cb32d7fd38d8902e1a304a3a7a3844b606bfcee8b8366b","prompt_token_ids_sha256":"676841a61aa3135e7a3ec64d03888de021cc34698ee1b25aebfff640fc502cf1","ruler_category":null,"seed":null,"selection_rank":21,"selection_sha256":"0f7355686d92107703dcf05c4240667b8c62bf7fc1f0c45fe1704ffc1686d685","sequence_length":180,"sequence_token_ids_sha256":"fc621b262cdee5eeb2686915a8d202aba1e489894daef49f381679515598da1f","source_content_sha256":"9cea206d7dc3308fcc17678f64598526990d06b799c3a6060837e47d5e0ac2ea","target_token_ids_sha256":"18f70329692e7510f6bc5c35dd7288681cb2d7fc3a45398cb46fe8c4064fc5b4","token_span":{"cache_exposed_start":180,"cache_exposed_stop":180,"prefill_start":0,"prefill_stop":112,"scored_start":112,"scored_stop":180},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"c56fd94d27ac497a874a2872a9378539bce23e6f40c0c88da8548530343fcee1","anchor_positions":[9,20,30,41,51,62,72,83,93,104,114,125,135,146,156,167],"anchor_positions_sha256":"eaf3a0e814f88177633695c30e9a05b6b492aa64c90bf98da98cb9be3917c812","canonical_id":"878","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,30,40,50,61,71,82,92,102,113,123,133,144,154,165],"fisher_boundary_sha256":"45cdd8b5f6368af3ea5312cd27a1db7a47955c6f64da0b88e27faf49b3c816e1","horizon":1,"input_positions":[10,20,31,41,51,62,72,83,93,103,114,124,134,145,155,166],"input_token_ids_sha256":"800e65a128ae4172d15f3bf9d35062d514957f43f113ef8390515f7bad4eb648","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,32,42,52,63,73,84,94,104,115,125,135,146,156,167],"target_token_ids_sha256":"cf7c2293e05be3baaf208adda2a202c2dac0599b886a00e133b951f71671b17f"},"formatted_content_sha256":"9fa1e6f58327152b9fd6caf18d856e0920f7b962af65f17c0b516a39a15e889c","generator_receipt_sha256":null,"identity_record_sha256":"9c45e96b568ec62ab8d97f8c4b13adc325eb92bf34bc8cb045cbf49deb8d226a","prompt_token_ids_sha256":"280373fa3a2da39e89a082829b753432d7f22d6ef6366e83598df36823d26b55","ruler_category":null,"seed":null,"selection_rank":22,"selection_sha256":"10b149086df38d9067b558f5886d640b68f4204287a8886f65495246268313d4","sequence_length":168,"sequence_token_ids_sha256":"0bd5f59a58f0f806de221fe3c637774e5c9e47f72c11f0b19b9f2cb06d977dfe","source_content_sha256":"44ecb08d42ef8de88bcb0e909c2e9befe4a195daaebf6c8c10e1baaf0438944c","target_token_ids_sha256":"5b3eced18f97c0b17b291bb1e4d2e732f1ef7b1b304ac11beb67409c8f2d3bc6","token_span":{"cache_exposed_start":168,"cache_exposed_stop":168,"prefill_start":0,"prefill_stop":139,"scored_start":139,"scored_stop":168},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2d6e6a2df8f80c87c48f19e05e8d274114304b958532988537c2cfd6c7ba447a","anchor_positions":[11,24,37,49,62,75,87,100,113,125,138,151,163,176,189,202],"anchor_positions_sha256":"0e1a219b40ec9118f3f3cab251874074a9782d1da2d399b19337c5a52ff6788a","canonical_id":"822","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,36,49,61,74,86,99,112,124,137,149,162,174,187,200],"fisher_boundary_sha256":"6667326d144985716907297f9fcad0d67d899e5951cfd594d7c70e6f0b3e7ec4","horizon":1,"input_positions":[12,25,37,50,62,75,87,100,113,125,138,150,163,175,188,201],"input_token_ids_sha256":"70f7035d5ef6a263a7316e7d5fe9a1d64361df51aca8f8b97408bc66942765a2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,38,51,63,76,88,101,114,126,139,151,164,176,189,202],"target_token_ids_sha256":"b72dca6074d7b5b12b92fb70cfe49900c4d0b2869f86c323e978902c8635fa80"},"formatted_content_sha256":"e207600ca200fa35202626097b7ce35aabbc9a909f74a7f4a9634823421b3f85","generator_receipt_sha256":null,"identity_record_sha256":"6f1cb4fcebbe64bdece660cf8e72d4b0d933e1d5e76363e0fc65bc84df091ccf","prompt_token_ids_sha256":"a45819695adc36f87315b6277308921c06ea8a973dd4acd584c641b42a489685","ruler_category":null,"seed":null,"selection_rank":23,"selection_sha256":"10e329cfe9c44c390c0748888b664bc73f711559a095943ffb716a5c49a43bfe","sequence_length":203,"sequence_token_ids_sha256":"7ca67bbfaf0cf4d58cd3fbf7999f0bdea010a29908aa1277c433c3640ccc93fb","source_content_sha256":"c34011770653c990605bfd0878f3caa642227458bbd94408174d49073634373c","target_token_ids_sha256":"d355095fa08c0462073247a017dce85a1e1739895f047802a3f3fba9380d16ac","token_span":{"cache_exposed_start":203,"cache_exposed_stop":203,"prefill_start":0,"prefill_stop":71,"scored_start":71,"scored_stop":203},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"89cd198269ee29a7bf3a758e494f86cdd7b4c8eb634e87418ec24fa6977ffb90","anchor_positions":[8,18,27,37,46,56,65,75,85,94,104,113,123,132,142,152],"anchor_positions_sha256":"75c1fd0cbacbbfa75bfedc1647ca1a15436ad446c2021b146a8d7af4567321f4","canonical_id":"687","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,27,36,46,55,65,74,83,93,102,112,121,131,140,150],"fisher_boundary_sha256":"ce22804019d01ac4a4b1ecc051fc015d0fc2edfdc083fe9b46238115a22d2681","horizon":1,"input_positions":[9,18,28,37,47,56,66,75,84,94,103,113,122,132,141,151],"input_token_ids_sha256":"fc68589c14fc3c7f27c207419457fdea2a3ca0ea5556ab837c7e77422872e50d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,29,38,48,57,67,76,85,95,104,114,123,133,142,152],"target_token_ids_sha256":"2aafef74e6bdf1473faf98f2e2f51258d9876ed6aa14fd3c8d901e57a6a67670"},"formatted_content_sha256":"34caea8f7845c77c8c7c5cb04e37b6b1594df9805f377e4ad243d7b6bd787df4","generator_receipt_sha256":null,"identity_record_sha256":"5ead1d0afb62e76c6d447e766cb0e770b7b3122bc85e14984b945f91559084eb","prompt_token_ids_sha256":"7231469d89e6746cffb3134204ab6225bd06db47e1a21a1b29978270a92818f3","ruler_category":null,"seed":null,"selection_rank":24,"selection_sha256":"1241cc732d851bc4d6e31425c413658747a68ac5bf1df371701b6bac3339de38","sequence_length":153,"sequence_token_ids_sha256":"0ff8fac034540d5fae83061b024846460ae750e94227132be74a1f8b13d0c5f7","source_content_sha256":"570e823f5d18678380cee81c7f8171a7dccbbe1956f576e5bef831ef0417fbfc","target_token_ids_sha256":"9dd750e2b89888cecb138a1c442f682c640b1ce4caf2a92c920fc630aff9b969","token_span":{"cache_exposed_start":153,"cache_exposed_stop":153,"prefill_start":0,"prefill_stop":89,"scored_start":89,"scored_stop":153},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4c2c007c3a069505a8984ea48151216aceab5ff41919fef5caa62a9c95724294","anchor_positions":[5,12,18,25,31,38,44,51,58,64,71,77,84,90,97,104],"anchor_positions_sha256":"3d14c7e60b70361fe3f48971fff7c78a0e3a89500325f79e8590e96f876c6d84","canonical_id":"820","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,11,18,24,31,37,44,50,56,63,69,76,82,89,95,102],"fisher_boundary_sha256":"a5d52b2d880ccd9e8c57da79f29d3e8871447c9521ac6f1afce731259780a7b4","horizon":1,"input_positions":[6,12,19,25,32,38,45,51,57,64,70,77,83,90,96,103],"input_token_ids_sha256":"f6d753d6a779f0b43d8eb24d7213ecc4ab0ee16118bcbc17c3cb053d5730670b","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,13,20,26,33,39,46,52,58,65,71,78,84,91,97,104],"target_token_ids_sha256":"75597ca42c1c682f0e8f425a875b8483f8c45067ec1d096e0fd49b964402322c"},"formatted_content_sha256":"c5473e872de3d1d10c5f923d4246d5e6e41987ce6e60afff50ad39c665b75d2b","generator_receipt_sha256":null,"identity_record_sha256":"bfa5a8c5d9a1e111c3b629cdc0f09fc8a0dada73721cc93beb5ce531328d1f35","prompt_token_ids_sha256":"9edc0e91d3b033f1f21804597df8021ebb7cede1211331d0898ea29ce2062107","ruler_category":null,"seed":null,"selection_rank":25,"selection_sha256":"1269c5ac64a35cc30e3aab0138aa3e2e5cf40fa1a583bd687f3718d0b78632b5","sequence_length":105,"sequence_token_ids_sha256":"d9f20cb4b5b15d6acba099a4e472ad67b130db359488d35bb838bd17655f28b3","source_content_sha256":"1cda14e8c3edcbccfd8ccbee369285b27f5c9203af7f1d0171339a6fc431ddbd","target_token_ids_sha256":"845201cf3c1564028c1f00ea5c73aa8c447fdf960863b76d2f859ed548bcd8a1","token_span":{"cache_exposed_start":105,"cache_exposed_stop":105,"prefill_start":0,"prefill_stop":74,"scored_start":74,"scored_stop":105},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"f689b6853781e9f787183224c1fd3cee86b9e7cc33bd38b263db4768f634cf37","anchor_positions":[14,30,46,61,77,93,108,124,140,155,171,187,202,218,234,250],"anchor_positions_sha256":"ee1858c2e927a82713536b65cb17e66e4c3e2dba47ef6753229f8cbe7def8d65","canonical_id":"920","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[14,30,45,61,76,92,107,123,139,154,170,185,201,216,232,248],"fisher_boundary_sha256":"0d18b72689a6cc158337087ae0bf96353aba3ba691e2c3ad3383f821d707d7a0","horizon":1,"input_positions":[15,31,46,62,77,93,108,124,140,155,171,186,202,217,233,249],"input_token_ids_sha256":"af45fdc8e583f1a48a51520447e6ae51e48ff97bf7106924892afcd5b445f380","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[16,32,47,63,78,94,109,125,141,156,172,187,203,218,234,250],"target_token_ids_sha256":"9fb5b7afdc77178b1ad18dc9c00bf26c57c1c7f7283fef7c2915df9d37953a04"},"formatted_content_sha256":"ca28cff2764b8d05657413d50f09faa64113177e180abfde5843c1dd2e08ff63","generator_receipt_sha256":null,"identity_record_sha256":"af618c65e0f65410c7799e66f057af21d4ef73c2941c226d581f4f95c21ec761","prompt_token_ids_sha256":"5b15d5261297dc4ac22a17ffede7ef05db15e3f11f0c33e02fb3a3b0e0716a02","ruler_category":null,"seed":null,"selection_rank":26,"selection_sha256":"131778dc6e39fd27c980789805b135c6f9d71c73da87e2cf5d8f360898a2dfd2","sequence_length":251,"sequence_token_ids_sha256":"c20ed19761b4f9b066a1329b36e80f89c62a064c75eaf0dc86e28e961252022c","source_content_sha256":"a36a2af1cadc864f5b43b1a1232f93e477ab83df4d2faa13fb766e215ba061ce","target_token_ids_sha256":"3f83098fcd8caa7e8285f24d6317b53da289468363aa33dd7c3ad415fa2f3ee7","token_span":{"cache_exposed_start":251,"cache_exposed_stop":251,"prefill_start":0,"prefill_stop":215,"scored_start":215,"scored_stop":251},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1df227dd0db9ee9876c5933450c5e9d7dad0c0b569bfe3c3c9999cb603466a66","anchor_positions":[12,25,38,51,64,77,90,104,117,130,143,156,169,182,195,209],"anchor_positions_sha256":"33a74989d1780ad7a110110bf9c662c1607a2b0974227d3d4808405cc10473b3","canonical_id":"771","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,207],"fisher_boundary_sha256":"ca40d8ca4200a96f42fdd96325a52572dfd4c03985d05bc5122da59ebb5411f5","horizon":1,"input_positions":[13,26,39,52,65,78,91,104,117,130,143,156,169,182,195,208],"input_token_ids_sha256":"4a16859fc5dbcd936377e8ff10cf7ee24881b00e341b233f5d9eb6bcea0473b2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[14,27,40,53,66,79,92,105,118,131,144,157,170,183,196,209],"target_token_ids_sha256":"5ca4ff0a723a1b4d28096b092591f8877fbcca9586d49c5ca5cec329031ee210"},"formatted_content_sha256":"12a4cc60de825c6e0b4030d980e6efa58ef1883c0dc4e19c9247b74a5cf96589","generator_receipt_sha256":null,"identity_record_sha256":"b0f0620e81f02da6395b8984ee9a7ea9030dc636bfe539b7bdc6cada58f330c8","prompt_token_ids_sha256":"f7dad551ce5d1c884e8df0a56f12ba6a9086a14bf88ab24c0585392f67253369","ruler_category":null,"seed":null,"selection_rank":27,"selection_sha256":"13342bc1fc998ac6effb2746ad6dd189bd6e08647608c26dfca8ba6e4aa9bc7d","sequence_length":210,"sequence_token_ids_sha256":"880adca8dd7b7e2a5ad8761db452ab4f71a3a2ddb89a2fa3d3f5a58203aefb0b","source_content_sha256":"f9bc388a30026b9c50d24c57c12c983ad57c610e3b1805c0f31da6113e7f0655","target_token_ids_sha256":"946d1f65de1ab807e8d9cdfe1ab226186db882b34df45f0fb02c5dc1c917ec65","token_span":{"cache_exposed_start":210,"cache_exposed_stop":210,"prefill_start":0,"prefill_stop":75,"scored_start":75,"scored_stop":210},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"ba5d2d0908e27069abed969317f8767e700f0653a11ffa619097bf013257832e","anchor_positions":[21,43,65,87,109,131,153,175,197,219,241,263,285,307,329,351],"anchor_positions_sha256":"cac63358bd3a7b31df1079c37c0d5ebcf2a9d198f8fbdbfabcbd20682cead52a","canonical_id":"869","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[20,42,64,86,108,130,152,174,195,217,239,261,283,305,327,349],"fisher_boundary_sha256":"2f61fd72d21168bcf628f22bfa5c23cc9ae7adad9c324ddc9436931a5fc1078a","horizon":1,"input_positions":[21,43,65,87,109,131,153,175,196,218,240,262,284,306,328,350],"input_token_ids_sha256":"4681d4fb7732b42586111bbc62eecd3d429bea617e31837702eba8e398a2cdec","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[22,44,66,88,110,132,154,176,197,219,241,263,285,307,329,351],"target_token_ids_sha256":"4703a2598c12a6054f3c45ce108e141b4a06b5c95d21417d8999f4cbef5fb5b9"},"formatted_content_sha256":"a0eb7c677531c1708849fe663f5664ca25a2bb7202ed10517fbd94dbde91daa8","generator_receipt_sha256":null,"identity_record_sha256":"f65800d83e860fc9520ba9f8b4f750dbd13615d735ec53de0c7592e72d90220a","prompt_token_ids_sha256":"b39968cf98321965c22a9c256c9561e9cc6e27d9a79af006cc89227ec7120efd","ruler_category":null,"seed":null,"selection_rank":28,"selection_sha256":"146f84886fd8ea499b01ff6f18e9470af408d245c5e61abbaa90c362cb26c801","sequence_length":352,"sequence_token_ids_sha256":"1b54a34c4d7ffc13d96aefb4e29514f90f431dde94601897a2dbb235edc52720","source_content_sha256":"aaffc31c45370a56d2068d7903cea3700521215fda95a64f3df629df4b8eeb38","target_token_ids_sha256":"604ca3456eb59e105b48ea010ab6e7f9f2dcfb395b4632901098473fb57640a5","token_span":{"cache_exposed_start":352,"cache_exposed_stop":352,"prefill_start":0,"prefill_stop":305,"scored_start":305,"scored_stop":352},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2bb8e9815739f4c21c430d24915060fe6bd9dba2fc037408c960a316957378f8","anchor_positions":[7,15,23,31,39,48,56,64,72,80,89,97,105,113,121,130],"anchor_positions_sha256":"4b59ffbfdc6171d05db7e4ce13fd81c13fb1c2704ef54cbfaede26eafe2b1382","canonical_id":"851","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"fisher_boundary_sha256":"0b7b6d1d65846d9e17cf0ca2a441e8df727713097fccdd92a3cd8bdb20be9263","horizon":1,"input_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,129],"input_token_ids_sha256":"08877abaf9a7a17c7f7f67ebc2737c37d6453391c6d7b8ec1fbcd14c32c4e1fa","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,33,41,49,57,65,73,81,89,97,105,113,121,130],"target_token_ids_sha256":"f8b2c9f022f43056408bb17bc5c2c95096dde7d6dfeeade0d28c5bf51c33888e"},"formatted_content_sha256":"571149c507816476043bbda2c1fdd724f3ecb9950bd16943ff2ad7eb4ea3f436","generator_receipt_sha256":null,"identity_record_sha256":"7f27dea0fefa17b65ffacb46cfdb9b6d59002d7ca0b9361a0895371d7110d026","prompt_token_ids_sha256":"286d768c4aa541e076ba087112772071c0ed8c5cf796ee399892f394bcbf99e9","ruler_category":null,"seed":null,"selection_rank":29,"selection_sha256":"148757e7e40834cbdd797fc8415b8ff9af1ad4582e23345b11d6f4df20524e54","sequence_length":131,"sequence_token_ids_sha256":"8d6714078aaa6f0fbd4d5dffc75dbd163c2b9867713c0dffd9d5fd8f7017a68b","source_content_sha256":"a2daebc00878723580b55d11ecc7c733a3b30dc6ac7d2162e0a5ddb14516f888","target_token_ids_sha256":"2007499fc1bb16a5e15e13b3ebe6e9c56b039a7ed23a50d2c653934a2788085c","token_span":{"cache_exposed_start":131,"cache_exposed_stop":131,"prefill_start":0,"prefill_stop":95,"scored_start":95,"scored_stop":131},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"348a971192939b69d748687327be2a9dfe6f51af36995ecaa6f6c112cad31b71","anchor_positions":[9,19,29,40,50,60,71,81,91,102,112,122,133,143,153,164],"anchor_positions_sha256":"e70afdac88bd21ff5ffd057462f0868a1aaa408a52296b58ecd4900f7f13759c","canonical_id":"728","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,39,49,60,70,80,90,100,111,121,131,141,151,162],"fisher_boundary_sha256":"de5b5407dcf2dbe46dcd273dbcb9112fd1e97d1b3786ce35fa8124c57a372da9","horizon":1,"input_positions":[10,20,30,40,50,61,71,81,91,101,112,122,132,142,152,163],"input_token_ids_sha256":"164cace43db8303a6bad35155786734ad14df0e29d82329399b4eb77c2ab2a51","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,41,51,62,72,82,92,102,113,123,133,143,153,164],"target_token_ids_sha256":"cf736f31488418c09e969cb79604c6e24ee27824d7b99f2cf3615788cd99df11"},"formatted_content_sha256":"876263584044b675e092dfa08f39f07f3de5267b9c2cc8ebc0f0dcd3f7e69bed","generator_receipt_sha256":null,"identity_record_sha256":"07b155ad0595cc23e7b6f3ba638e79b47f8790d682a99a574040b9b316eb1770","prompt_token_ids_sha256":"7673bf381655d73528d4ffb834dd0461fec79f874ac5706819d772a4fd92f274","ruler_category":null,"seed":null,"selection_rank":30,"selection_sha256":"1488999c83a1784fe27b3d4c152a98b782b1ce9c84a2375a4dba8bc8a3845802","sequence_length":165,"sequence_token_ids_sha256":"cbf68229cee850133e8ccd5fc251f1c822ecfb6aa1c9466194360609b2644aa8","source_content_sha256":"1e63d65d4f23efcd1ebde304ca66d531e9fdfa44f73cf04c72637826af3e7c94","target_token_ids_sha256":"519538cb0a590f8242c78a833368585a5650cdbbaffd4159acb21311b25be8bb","token_span":{"cache_exposed_start":165,"cache_exposed_stop":165,"prefill_start":0,"prefill_stop":128,"scored_start":128,"scored_stop":165},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"fd0a190e30ef1d0dac28f65cd40bb5a6bcf3b1813b383cf8c3595fc399c3cf68","anchor_positions":[8,18,27,37,46,56,65,75,85,94,104,113,123,132,142,152],"anchor_positions_sha256":"75c1fd0cbacbbfa75bfedc1647ca1a15436ad446c2021b146a8d7af4567321f4","canonical_id":"704","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,27,36,46,55,65,74,83,93,102,112,121,131,140,150],"fisher_boundary_sha256":"3f642dc0ec35708247c2582e0525688edfba25c8e20f052b50fddeb5a552f23a","horizon":1,"input_positions":[9,18,28,37,47,56,66,75,84,94,103,113,122,132,141,151],"input_token_ids_sha256":"c7f609f9659eee383626ea3792b82efa4d88eed93f90d975eb30c098b4b461a1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,29,38,48,57,67,76,85,95,104,114,123,133,142,152],"target_token_ids_sha256":"04c201a63e6b0f0929e297c5714da1234f09af3ca5cb45bc1b660e160389b369"},"formatted_content_sha256":"cae02a47f70b35282c8bb71d3b57044d50c57479d6d860ab99959bd7d6a92fc2","generator_receipt_sha256":null,"identity_record_sha256":"5ee66dd333ec6248d85a5f68ed91d44953b5193938257c58e4a4d12487288565","prompt_token_ids_sha256":"049fdd2c0e15e77a6301829c690e8a3c665ce6861ab91933d27aa964bb51de01","ruler_category":null,"seed":null,"selection_rank":31,"selection_sha256":"149396b0bf9de8c384e257ecabf9d186fe87150d089c3ff0d04e2adce108c249","sequence_length":153,"sequence_token_ids_sha256":"d06f1e44e281b3cfff0ab509061ec63103257ce65b9382636808622420a3847b","source_content_sha256":"9acfae6a9b6bf5138640d082e7cd35c1728c6aef929c1163bdc349fd4539d647","target_token_ids_sha256":"413994788ec19a2c46be1007e6767b52c27d228534ace24859c56b31248a539d","token_span":{"cache_exposed_start":153,"cache_exposed_stop":153,"prefill_start":0,"prefill_stop":114,"scored_start":114,"scored_stop":153},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"30b1518b021112b74a55061d787eb229a1e0d9413d6dacc4728a4dab232a67b7","anchor_positions":[9,19,29,40,50,60,70,81,91,101,111,122,132,142,152,163],"anchor_positions_sha256":"868d4ebf721b69ebc32ed80fb23287a93509d3cda6e8df7dfacdd2b56b0be622","canonical_id":"725","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,39,49,59,69,80,90,100,110,120,130,140,150,161],"fisher_boundary_sha256":"9143ea3e85262814307335f186e336430d15da7ab0b7b3d21349bf2995db95b8","horizon":1,"input_positions":[10,20,30,40,50,60,70,81,91,101,111,121,131,141,151,162],"input_token_ids_sha256":"596886915afefd5186de7498848bbc83ca47716f8e4d7009a44f03810ca08ca5","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,41,51,61,71,82,92,102,112,122,132,142,152,163],"target_token_ids_sha256":"3039436dc9e159ba1ad775ff006ed4075d724a7118b474bb9fb8a1f2e8adf4fa"},"formatted_content_sha256":"ba9dc9a284681815182d925a5298e2f7da2b4d824b73ef346adbda8f64016575","generator_receipt_sha256":null,"identity_record_sha256":"e596da18d1c37026ef17f3e29e6e8ff1d0a69c227e1cbbad9c536f121a0a3fa1","prompt_token_ids_sha256":"9d320289cf51463daa5b252b98ca89089d036d231b6c861dc2405bc54707a135","ruler_category":null,"seed":null,"selection_rank":32,"selection_sha256":"14ed1df91ab46cf4aa753625f81c4925778f6febc4e611062de8ad6126487c18","sequence_length":164,"sequence_token_ids_sha256":"b6e36e161450f26908c1d62afda5bc88c353a091bbe87f9de4dd698d1df5afe4","source_content_sha256":"d222c883a2e980af7460497236d6865d722ce738a2d6303dcb39cefe0ea11658","target_token_ids_sha256":"da7b35aeba5cd5e4fc9de97bd08c7c8e930056923fea9d072dfd25b57ef54703","token_span":{"cache_exposed_start":164,"cache_exposed_stop":164,"prefill_start":0,"prefill_stop":138,"scored_start":138,"scored_stop":164},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"33a12c7dff5a4e3f2b0e78cb96a2e15ffca6d15f728512f48ad3d2a521547859","anchor_positions":[12,26,40,54,68,81,95,109,123,137,150,164,178,192,206,220],"anchor_positions_sha256":"9cd7cad8fc6a3e6b59742658c86bba1f77992157cd1a23eb295add08927945d5","canonical_id":"616","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[12,26,40,53,67,81,94,108,122,135,149,163,176,190,204,218],"fisher_boundary_sha256":"7fb746d0e7bad7bf9d3e8daeb30cb4fb5e6d8058e6e2f60f208e45d9d3074ef8","horizon":1,"input_positions":[13,27,41,54,68,82,95,109,123,136,150,164,177,191,205,219],"input_token_ids_sha256":"2984e5c39ef7c7a9017c275ad13366ddae1e4a944113a0d7634a8743fe7d838e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[14,28,42,55,69,83,96,110,124,137,151,165,178,192,206,220],"target_token_ids_sha256":"f8ac64591035ce55623fdb95289f20ce8ce2512016905cd8c55cdbd1eaf0892e"},"formatted_content_sha256":"4c31755567c8f3a5c363166b6b45afe0e9756596cab355453c6d1bede973f1cb","generator_receipt_sha256":null,"identity_record_sha256":"2d8025f16c114b643173084fd6a55004c4cc8d89c19ba719eaf77c614f3de5bb","prompt_token_ids_sha256":"e3e1a17d69f2fe4102ab51a9bb887707430d5bd517f752ff79cb5ba3bc19372e","ruler_category":null,"seed":null,"selection_rank":33,"selection_sha256":"15619d70a9d1e84b42b7fe6f67fb25251ed4e033a22a1f6b44afdb5190e0d9e4","sequence_length":221,"sequence_token_ids_sha256":"c14c8b8afb98ee4e0db1a7d15b4a5ad8f4ceed0533d70f5c7f3e3069aa019030","source_content_sha256":"1754fb38b7c558a69da8e9273fbe27d53c7428292d713cdcc926533238ea0dd2","target_token_ids_sha256":"67f9de9099caf2a8542295769dccf90b9868362889179ae8905fbdfffbfa259d","token_span":{"cache_exposed_start":221,"cache_exposed_stop":221,"prefill_start":0,"prefill_stop":172,"scored_start":172,"scored_stop":221},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"df20597eac46fe01ffe3b097acab98b2815becb962637bf63848ef96325ccb51","anchor_positions":[16,34,52,70,88,105,123,141,159,177,194,212,230,248,266,284],"anchor_positions_sha256":"875344e57ef3cc2aeb5a93339807a07f348a76ff8e531ac8b017af57bdfad6d8","canonical_id":"686","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[16,34,52,69,87,105,122,140,158,175,193,211,228,246,264,282],"fisher_boundary_sha256":"a3d4c1cea37694ff9e48138187338acaf151bbb01faf6beacef99dbb88b7a3f4","horizon":1,"input_positions":[17,35,53,70,88,106,123,141,159,176,194,212,229,247,265,283],"input_token_ids_sha256":"e11b1d7a44256148ceeca485fc09b3a0bbcb82bb1fc8f781be0549ae79fe1f2a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[18,36,54,71,89,107,124,142,160,177,195,213,230,248,266,284],"target_token_ids_sha256":"89eebf9531df2b055714e83202f980d1b550ad965a5daf4c7bdb06671f9842cf"},"formatted_content_sha256":"7351330811ac6c42a440a1ec8bc6d19bdf453494f7fd6204e144c59693722149","generator_receipt_sha256":null,"identity_record_sha256":"66b94f7added65074e9aa08de692376e4e61c77477886d2f1f02bd7a5f752106","prompt_token_ids_sha256":"c9ee635d4f345c508119836f02286e8a883aaafc0a5504f5149f7ce65b3e8a58","ruler_category":null,"seed":null,"selection_rank":34,"selection_sha256":"15a952922cfb59e1c69a11112c4ca386841fc6ef24bc7d4b5506686777d97a8b","sequence_length":285,"sequence_token_ids_sha256":"fcc6b59259d07edcb03ac21e620699ab241c7ce75c923d03ee63d3146e5b591b","source_content_sha256":"84f9793e00ac40962b5d2488ed7ea603278d041e9ce4d1fdb4c80a3cb038287b","target_token_ids_sha256":"f0132a38a41b59f822f698b45717035a5147aeda26ea2ca039b6d45f96067ef7","token_span":{"cache_exposed_start":285,"cache_exposed_stop":285,"prefill_start":0,"prefill_stop":239,"scored_start":239,"scored_stop":285},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"969aa08fc8cf0fcef08b212a868ffd0ef6cba7360b8cdfd4c7e057bdeb42b613","anchor_positions":[23,47,71,96,120,144,168,193,217,241,265,290,314,338,362,387],"anchor_positions_sha256":"5de510290d52f91a9e5aaaf002df55793d821f7c4325536087b3fe975a00c5db","canonical_id":"722","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[23,47,71,95,119,143,167,192,216,240,264,288,312,336,360,385],"fisher_boundary_sha256":"5bcdece273089c73eeb060133d26006686f4ba55e2f679475c817201f0f06663","horizon":1,"input_positions":[24,48,72,96,120,144,168,193,217,241,265,289,313,337,361,386],"input_token_ids_sha256":"387e8436656479c468480e0d7ab6fb8a04966a25430d2cf7ba6b946e933a5988","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[25,49,73,97,121,145,169,194,218,242,266,290,314,338,362,387],"target_token_ids_sha256":"d4c1910bee8959bc2af95badfdaa811dfbfff9ce3708801ed5e9602140453d3f"},"formatted_content_sha256":"66886a05110a3b2882fef0d7f3d69fe9e4813e5a230a25f32ee61406ab9d5da2","generator_receipt_sha256":null,"identity_record_sha256":"dae3872dc358589c8a222ed2b8f364150c472c88b23243e89cbb751bec3bf394","prompt_token_ids_sha256":"e0dd20f7abb2fd8622ac76878d7d8dba366a9267d17eb23cde37996d51815bad","ruler_category":null,"seed":null,"selection_rank":35,"selection_sha256":"165983c8c5992c88338b5a83c3e955a3b379e2938838aa804f752b6af24364d5","sequence_length":388,"sequence_token_ids_sha256":"be18de19017b46d5b7be59183a8ecda4c60a214ae50f068dd8ce1b4ea6848323","source_content_sha256":"20253504a8b3a9c5fa26e7d29c98f783efb48d1960c35ff104acfdc24f41bd57","target_token_ids_sha256":"5e021e0c05f5c5bc55c907080f16d71f9b17ea980d1c4a1c35db2a4193541b91","token_span":{"cache_exposed_start":388,"cache_exposed_stop":388,"prefill_start":0,"prefill_stop":344,"scored_start":344,"scored_stop":388},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"da5073651bb69cf285a41628d42d9f0b3d2b37c1bf90aa7aab1b73c4d9096d21","anchor_positions":[6,13,20,27,34,41,48,55,62,69,76,83,90,97,104,112],"anchor_positions_sha256":"311e7702507c904e70997e0b842193b8c519ddd45b6861c3e52fdd12fcbe662f","canonical_id":"919","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,19,26,33,40,47,54,61,68,75,82,89,96,103,110],"fisher_boundary_sha256":"46adb9e95c41a48fc3a2299f313d710d273451aec915fcc44050066d7e62fbe7","horizon":1,"input_positions":[6,13,20,27,34,41,48,55,62,69,76,83,90,97,104,111],"input_token_ids_sha256":"27444c193cd674bda4df496bd61a9e9daaf7ab3670462bf76e183b0cb9f16771","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112],"target_token_ids_sha256":"0c19883626f27df1a14cd3ef14afef3bc348c0cd1bda790d1e08185bca73a595"},"formatted_content_sha256":"032c4c4ef8d61477f89f627c7da791c2a394cc3d48b63ae5e73ee2fc12217e59","generator_receipt_sha256":null,"identity_record_sha256":"3ff17e45b728b8142b6678e3a33eac324c6262961cd2c9f7c19d1310c440aeb7","prompt_token_ids_sha256":"23b1b41e6716b12d13650e7aa105d3a76602e537cb02e4accc5f592ffd9c138a","ruler_category":null,"seed":null,"selection_rank":36,"selection_sha256":"165a7fd7caa5808dac419a6b8728f561f43cf78710437e2638e5c6a82bb7617e","sequence_length":113,"sequence_token_ids_sha256":"4582364354dd4ea26e506fc7c1845a9a6a878d03218ac93a0725170aef66c313","source_content_sha256":"30238651c7d4eb4dd16bbcf4c54ef3c91884f1d06eeb02ff38b97cb5e03a6225","target_token_ids_sha256":"9f6a8727f6485bb7bce051a35976a9ccfdc83c3af9309cb39f3a93789a51f928","token_span":{"cache_exposed_start":113,"cache_exposed_stop":113,"prefill_start":0,"prefill_stop":86,"scored_start":86,"scored_stop":113},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0a798f70d770e513f2e911c70a33835f666289b6e23ff31551a773ef75b38aec","anchor_positions":[6,14,22,30,38,46,54,62,70,78,86,94,102,110,118,126],"anchor_positions_sha256":"5dd8de14849d09f9037c328d85f9f1208f24e656ecfb3354483ee5aa62c57bab","canonical_id":"636","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,30,38,45,53,61,69,77,84,92,100,108,116,124],"fisher_boundary_sha256":"5f2e24eb47dfbebce18f2c2f64fd45917172c9eab2bd9cec54bb4fd111eb2142","horizon":1,"input_positions":[7,15,23,31,39,46,54,62,70,78,85,93,101,109,117,125],"input_token_ids_sha256":"4dc0a187eaaa7554b02100bc49019061f57a4f625ac418d8d6b7708a4c85f659","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,32,40,47,55,63,71,79,86,94,102,110,118,126],"target_token_ids_sha256":"3904fbb873aaac279d16ea071f1eecd284ddfaf982e8b4b118e527a7c5cd1663"},"formatted_content_sha256":"38120716e8579745533e38a1b2e5c3ab60964ce9826f4692be5d38a23625c7d5","generator_receipt_sha256":null,"identity_record_sha256":"6b00dd2ed569a9e5852db2dae0aa92821b9dbc00778205f14f3b67a886091087","prompt_token_ids_sha256":"c53edd29ef5c93c57c169235bf2b32cb118e20df561bc94d9b39faebc2a587c0","ruler_category":null,"seed":null,"selection_rank":37,"selection_sha256":"198ca758f04e266276eba3d3b469b52d783bc89308214f93d72af2c6d4e11dc0","sequence_length":127,"sequence_token_ids_sha256":"60e0f64c1dcb89512f1401f29f0d940ea152343ebc4764e52809ea4c5588f847","source_content_sha256":"054f097d046f21661577c0ed913168b649add5d35032de9efef4938f8a1a24ee","target_token_ids_sha256":"d1bf3a7c3a1a467f3f8e1eac6c4887bdfbdc387af6b7c29fcb19d45df9277f3b","token_span":{"cache_exposed_start":127,"cache_exposed_stop":127,"prefill_start":0,"prefill_stop":94,"scored_start":94,"scored_stop":127},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"d11ddad979bbc50d6db14cf77f968c46cb786a0ce2dc77ba5f5b28d500fc3596","anchor_positions":[11,24,37,50,63,76,89,102,115,128,141,154,167,180,193,206],"anchor_positions_sha256":"c25b8f380f538bd903f3e5b66a99e3d50700aeecbba5fa891f2350659ebff705","canonical_id":"607","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,50,63,75,88,101,114,127,139,152,165,178,191,204],"fisher_boundary_sha256":"a15bb1e5b2d2a3301ccdb2e0cb8998f04674305617ecc9881b7d20c208dd21c1","horizon":1,"input_positions":[12,25,38,51,64,76,89,102,115,128,140,153,166,179,192,205],"input_token_ids_sha256":"7ffaff1442181b24e662e144cb14110cb590e37477796426bec7dca4fc4d39ae","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,52,65,77,90,103,116,129,141,154,167,180,193,206],"target_token_ids_sha256":"05b028901595cb91b630adba80dd4555b614c059c122db592c5aa980143feafb"},"formatted_content_sha256":"ef35e3704df515714d76b547483c483743548e7cc069f31e95b66dec72a510e5","generator_receipt_sha256":null,"identity_record_sha256":"80bb8f3d83b1925af377e081ac063f63989368883ef4e04dc49b8fd8f00c500b","prompt_token_ids_sha256":"5bd1a90f46714aa81aed8226efa90f47442e2892377804149115218b9fd3b0c7","ruler_category":null,"seed":null,"selection_rank":38,"selection_sha256":"19faecf7ffafd480a6fa766dc937ba0709e42a5d5263c93130eb1037763f8871","sequence_length":207,"sequence_token_ids_sha256":"020df0554b8bcb9a4b7f491a4513578dc9b60d47ffc0d439886b3997c36eb986","source_content_sha256":"c541b88b979a7ec60640a49c7a9db7db502b97f5fed4d1970da91470204697c0","target_token_ids_sha256":"28dd01dbcdd5f0d19a9bef9f9fbc6491786783fdb97ad1fb07f945a93ffc7cc9","token_span":{"cache_exposed_start":207,"cache_exposed_stop":207,"prefill_start":0,"prefill_stop":141,"scored_start":141,"scored_stop":207},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"46371c5d2e1a407eb39f03172e7b9a21c7b932f5d655758a3c0ce64ba8c669a2","anchor_positions":[23,48,73,98,123,148,173,198,223,248,273,298,323,348,373,398],"anchor_positions_sha256":"8d9eba8a281ab33b27dd14ed9d345bfd773fffcabf08f24d986bb5ecd2f5015a","canonical_id":"950","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[23,48,73,98,123,147,172,197,222,247,271,296,321,346,371,396],"fisher_boundary_sha256":"843c418f1af24b45838fcae84492105e4320ec3725733998cacab7b6853efde8","horizon":1,"input_positions":[24,49,74,99,124,148,173,198,223,248,272,297,322,347,372,397],"input_token_ids_sha256":"d9a45e7a4afdd75464b73c4c6185b6888c636708adbfe2ece93d9586f2e48950","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[25,50,75,100,125,149,174,199,224,249,273,298,323,348,373,398],"target_token_ids_sha256":"b25ab7ecce47ea372a3f59f87969ac12e02c7358ce537fa47921278d5bbace65"},"formatted_content_sha256":"99720600b7a4e17df2bcc2762c83cdf70cc53626e5ec631c9bb32d7398d93e45","generator_receipt_sha256":null,"identity_record_sha256":"6d22eef549de3a81fc0a232affc18a615e9f0fce2de79d390b05e11ebdbf763d","prompt_token_ids_sha256":"8801aebf9d0be65bf1d4cac5f7ba35ceab845d7b0e20d5f6bfe717cf70fcac04","ruler_category":null,"seed":null,"selection_rank":39,"selection_sha256":"1b30f64a04ff98347d4a23bd49f8964fc6d3976e5d73b89babb36d891b544775","sequence_length":399,"sequence_token_ids_sha256":"0c20245c48d7a840676ca88457d31004701b407e4992f56ed7e4aca800bd3cee","source_content_sha256":"67c27b9d1db42865751e90ac9eb8dce913fa954d522bc50f1a124ed869dcc3b6","target_token_ids_sha256":"80ad07987d1074046d11f85abd02309f81753af95f7b2510f15b49ce074774dc","token_span":{"cache_exposed_start":399,"cache_exposed_stop":399,"prefill_start":0,"prefill_stop":85,"scored_start":85,"scored_stop":399},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"88045b346b8da1ff4c8b2efa7461ce5138643eb8b76bb78bed0cb8b0de1efbf4","anchor_positions":[11,23,35,47,59,71,83,95,107,119,131,143,155,167,179,192],"anchor_positions_sha256":"97c036645b9b6a78e53e848879a7e3ee08a70ff938ed6e98aa7098b21ab4319c","canonical_id":"885","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[10,22,34,46,58,70,82,94,106,118,130,142,154,166,178,190],"fisher_boundary_sha256":"810182ed20f2eb26e385debb7eceaceabe95cdd144936b72eb2f3945a5610256","horizon":1,"input_positions":[11,23,35,47,59,71,83,95,107,119,131,143,155,167,179,191],"input_token_ids_sha256":"76887b11d917341d902fd087a797d1b8148960fe5a5c3bd03223991175a7d00d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[12,24,36,48,60,72,84,96,108,120,132,144,156,168,180,192],"target_token_ids_sha256":"6d849b678d0b30547bb1a54e0ea0ac4ce468ce61603c7c10ffdab4c4e48459ab"},"formatted_content_sha256":"a16af1d88bf778a540f042c4c4f117a8a55e110b844fac976861bf833b58928c","generator_receipt_sha256":null,"identity_record_sha256":"210e74e96d878fcdcc906fc9925e4f204d9363fe486a768730fb41f707b3047a","prompt_token_ids_sha256":"731bddd1572f8ac6e10c56476d8d41ce0f5b35ad3da79217ad289e53697fc68f","ruler_category":null,"seed":null,"selection_rank":40,"selection_sha256":"1bf4005ab722efdcf542cb1b134158d2a2ec80c4bb99a978bf37320204ea651d","sequence_length":193,"sequence_token_ids_sha256":"221d7396fd6ba0730b59b2d163b37447727eeada7ea48a5bc944748d4c03df26","source_content_sha256":"c09a41add9e81f949a02fef6dde8a12287741692b045d26b3459d9cbf51a329b","target_token_ids_sha256":"a28fddf10e4bd937c83e1787639bbae521e60b6a3d5d421086e7e8d04ced9afc","token_span":{"cache_exposed_start":193,"cache_exposed_stop":193,"prefill_start":0,"prefill_stop":82,"scored_start":82,"scored_stop":193},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"5631703787387bea51be9c3d8508c71583e012079113a9f10bbec4245f396cf1","anchor_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"anchor_positions_sha256":"debc2f860d51bf30c9ccfe7850d3428800e80a005f976ab2943a8642c33fb808","canonical_id":"646","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,30,38,46,54,62,70,78,86,94,102,110,118,126],"fisher_boundary_sha256":"ac073c216400b3bc77300dbc3f3bd0735b111cf717a637bf602d1117e5d70dd5","horizon":1,"input_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,127],"input_token_ids_sha256":"06dec0ee7d5561ebf4a6924837605f275d606eb9ead89de4bbe6b7ec86a0c69d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128],"target_token_ids_sha256":"819be8fbf775e438d651a64a9a89d31a86d639d85622f2e6a979e72bf5469879"},"formatted_content_sha256":"42d2696d713ac8654462eed80e51a0f4a279ae3bd0b2fac3a803e706e31df249","generator_receipt_sha256":null,"identity_record_sha256":"a62e891eaa4a9d3d4ea47d185ffa6c03a9b1d28dcc7dec3429aece7413ef0936","prompt_token_ids_sha256":"e5220dd3a6c5ee6f26bffcedb6d05bd93a890b9b7d74832a9669975115e37150","ruler_category":null,"seed":null,"selection_rank":41,"selection_sha256":"1de0cdbbd1b50e7684dc3deead2b93036bacf525241fea00a2c68aeb5d1e9cf0","sequence_length":129,"sequence_token_ids_sha256":"eba6b177f82a24193e70a824138922ac8eee52933ebb80cb81a777d6e0f85943","source_content_sha256":"14c86e5b0221ac25a5d693c8996c478c0b60503bd396981a5ba2629d0c6b6f99","target_token_ids_sha256":"6c75f6536a4d7c5a2942fd540f9268e2e1961c4c3dd4523408da058253f543b3","token_span":{"cache_exposed_start":129,"cache_exposed_stop":129,"prefill_start":0,"prefill_stop":88,"scored_start":88,"scored_stop":129},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7105aa2d8387cb83f89b844f1f0a3176a54c1be40233dec8811a37cb7bee09d4","anchor_positions":[5,11,17,23,29,35,41,47,53,59,65,71,77,83,89,96],"anchor_positions_sha256":"bbeb8538883fad44ee21dd0032311ae447587513a5cfd0b0e896721a67ec397d","canonical_id":"708","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[4,10,16,22,28,34,40,46,52,58,64,70,76,82,88,94],"fisher_boundary_sha256":"977350ea6b433e2f68043b28b121e4816ce8fb2f7358ecf47291b22c9adc1129","horizon":1,"input_positions":[5,11,17,23,29,35,41,47,53,59,65,71,77,83,89,95],"input_token_ids_sha256":"a2ea5635b427055cb1e2e99736b9dd1cada7580fac25e57e539b72697c2a8ea9","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96],"target_token_ids_sha256":"81ee8d37b3f38a8c027e847f395edfdc8428e67a0cfd5bb3a5d12bfdac304ab3"},"formatted_content_sha256":"c810e44140744e432b3b9ff12b48167b78e7a00f4f078b3d9b5a3f539a0438dd","generator_receipt_sha256":null,"identity_record_sha256":"ab8e48a41997cface254c352c24a699bef6c43ae1858c93baddd4ba95bbef8e1","prompt_token_ids_sha256":"5a9bc94048f86f543dd1e7c3e8040a400a75b1f99b6ef637101d4590969f22bd","ruler_category":null,"seed":null,"selection_rank":42,"selection_sha256":"1e4c5d9ae6b7070238249263248d1c8593c90a70aeeb70af89325a5b3be72629","sequence_length":97,"sequence_token_ids_sha256":"23e232e07e81e613d2b01d8540dfd4e20039938471bf3dbebd3ba8c57449e8b0","source_content_sha256":"a670341f8d1f903c87ab584beb9dd6355bf23c707ac087606d85d5e1094169ba","target_token_ids_sha256":"bc70ef552d90fd9d0e6a01a2dad09bad247647bd98b2af5d9afba83f63d97b8a","token_span":{"cache_exposed_start":97,"cache_exposed_stop":97,"prefill_start":0,"prefill_stop":79,"scored_start":79,"scored_stop":97},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"21fdfa6c62576c444b2ede715e8c0f394692bdae740be5657a09fb9ac5465380","anchor_positions":[7,15,23,31,39,48,56,64,72,80,89,97,105,113,121,130],"anchor_positions_sha256":"4b59ffbfdc6171d05db7e4ce13fd81c13fb1c2704ef54cbfaede26eafe2b1382","canonical_id":"840","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"fisher_boundary_sha256":"f1a95ea699018f29238a0e61c6067bc03c5622df1f9f38ee7310c244a4f3bc65","horizon":1,"input_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,129],"input_token_ids_sha256":"db9ca78375767d3dc1fc43cd6274a248a1200a4e579725f08ae34f2d1594030c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,33,41,49,57,65,73,81,89,97,105,113,121,130],"target_token_ids_sha256":"d4ee1e1651e25e76151be8dfefda3d59a9153f9a6418389ce485081fde471280"},"formatted_content_sha256":"9aced07cb336136d232b88c651523d949c3ed1b58608f3e91ab8eb1e0f92a7d3","generator_receipt_sha256":null,"identity_record_sha256":"baa3355a5d2917d2ff4b713073abc3c98d3168719b1d9a669dfc8d2745e62db8","prompt_token_ids_sha256":"9b97f8bc9144022005c6d732aa6daf7482cce1d0462c9f41e4e54f455d770a9b","ruler_category":null,"seed":null,"selection_rank":43,"selection_sha256":"1e5afc59d9964484ca6906bae912fa0a9c9871e246e9a5aa6f4e2d2cd74a61e4","sequence_length":131,"sequence_token_ids_sha256":"ac96beecd6d569ac72fd89a9d15d46adb2c7e2ff3ab53a38a47e2ebac4b5c19a","source_content_sha256":"144d4c36a2f15b6079c87286abc57c5c33d11947809eec3b55d2c56b0abdca35","target_token_ids_sha256":"60cc8ab098d798eaf3a7d1177d7c65092515a6756c2fb167019eb1588a7a5f57","token_span":{"cache_exposed_start":131,"cache_exposed_stop":131,"prefill_start":0,"prefill_stop":98,"scored_start":98,"scored_stop":131},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"92d974a85b650173fa2852040052d155e429f141353a391c32eb9c8b22bf9582","anchor_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,208],"anchor_positions_sha256":"e3c9aaa391385675ab3475417e405da92d531052d937e046ddc609185dc2c7d4","canonical_id":"793","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,50,63,76,89,102,115,128,141,154,167,180,193,206],"fisher_boundary_sha256":"bcee19793289e8c6540ea881a8f492bb16b99a21c7f08133aae2e4a62163e585","horizon":1,"input_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,207],"input_token_ids_sha256":"ea265790ccf2b7ad1bc5812d8c7ad1653088a1b938adf116bbe32e00c4b3161c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,52,65,78,91,104,117,130,143,156,169,182,195,208],"target_token_ids_sha256":"03166cef7fd0abd95fc728344274b68579c18cdff9751955cb55d37472750e8e"},"formatted_content_sha256":"5990b44a405865af219b0ebfe72ca5f4807c7fa9ea8d17a998d181a60e7e6e2a","generator_receipt_sha256":null,"identity_record_sha256":"7694eeb6f1cf845cfd47b3f8c037b4453af3e6f1af57cbc1b271567d78747feb","prompt_token_ids_sha256":"a215965eef2fc3b7d4240f33fb78e20e16353f44c552a86876020a731cfb23ca","ruler_category":null,"seed":null,"selection_rank":44,"selection_sha256":"1efdf31f8359ff37ddbd6972a5b1f70a89aad0090992f1439e95a68b17f5805e","sequence_length":209,"sequence_token_ids_sha256":"b0ada4ec9a33d0cbd5317f8965026c54cce944770d6db49dedf1045e3e120210","source_content_sha256":"f51880de3d4f3f8df181ab757d4737250300a2719eb925110ed1a7025baefea4","target_token_ids_sha256":"1bb894993d9298db11a3f618f8c67b31b484d0fa664e5d9a74950bae08de8292","token_span":{"cache_exposed_start":209,"cache_exposed_stop":209,"prefill_start":0,"prefill_stop":108,"scored_start":108,"scored_stop":209},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"467b274cf4fd1ebd3fc1f65e47b4ebcef7ec6daac053d9c806d526d02b1aa9d2","anchor_positions":[7,16,25,34,43,51,60,69,78,87,95,104,113,122,131,140],"anchor_positions_sha256":"adf29a4dd3126019eaa451ccdb6bbdf039b38d8dc8e8cbe224d799517db47f03","canonical_id":"860","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,25,33,42,51,59,68,77,85,94,103,111,120,129,138],"fisher_boundary_sha256":"4298d52c37b3378977ffe8fcdbbb8de8a2001e201f91fa007b5bc8caf344f153","horizon":1,"input_positions":[8,17,26,34,43,52,60,69,78,86,95,104,112,121,130,139],"input_token_ids_sha256":"ff88adf412ffcfaf2eb5e6577b0406f784344c338d07181aab928b4e45c4aedb","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,27,35,44,53,61,70,79,87,96,105,113,122,131,140],"target_token_ids_sha256":"cb42fbfadbf84d40f31a090f455333c32b8d82097fa30bbe81c411778c79d549"},"formatted_content_sha256":"9fa46af05168957f4bcef1fe5cc161d5fd1f8aec927a8f95b00c13ce334a2346","generator_receipt_sha256":null,"identity_record_sha256":"f03d6c0de51871dd6a4c550ca9bb84df67fdd8703da60597ef107f6b4b5998cc","prompt_token_ids_sha256":"a9df32740687672e9bcea5fc074876046b128fd5263c7a057d9f57e081343d25","ruler_category":null,"seed":null,"selection_rank":45,"selection_sha256":"1f2c607266f18f654c50ab67f529127bf81f03531b6d79c46ac97f5081f09a65","sequence_length":141,"sequence_token_ids_sha256":"13d911b52ef7a2c203854b8ab41f9083733e9bda1ca8ed67607907aacf4bb0f8","source_content_sha256":"6948e55f2c0c12a3d6c826e33977a3b358f503e8cfccd743ee937a4f53cbcd15","target_token_ids_sha256":"37c5ab0e675f04a104215395c9a2a8503e4871823234c8abbb133ac6babb0449","token_span":{"cache_exposed_start":141,"cache_exposed_stop":141,"prefill_start":0,"prefill_stop":95,"scored_start":95,"scored_stop":141},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"37d7574d8d2b723c82778a768a2a566b7f49e8ebb69db0fb5d1e6794a930a2a8","anchor_positions":[5,12,18,25,31,38,44,51,57,64,70,77,83,90,96,103],"anchor_positions_sha256":"307c6a1944f2d94c59465c6af96e251aaf0ea665f8f1a3e0e55e3eb0568d8551","canonical_id":"800","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,11,18,24,30,37,43,50,56,62,69,75,81,88,94,101],"fisher_boundary_sha256":"54b8850d26dd8a6a43bc711ad584ae0be7143cd933d1768dbf78ab9d2e4721bf","horizon":1,"input_positions":[6,12,19,25,31,38,44,51,57,63,70,76,82,89,95,102],"input_token_ids_sha256":"5182574bad58e65b680691603b61908c731b487efc2d522fac61ffb07e8601e2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,13,20,26,32,39,45,52,58,64,71,77,83,90,96,103],"target_token_ids_sha256":"8836df71d734324c70c31d4de1e0ff0cdde8edd0e57b420ca0ed65319cddd528"},"formatted_content_sha256":"b47d463d8300a4f8eb63bc1a7f94408c9b894a1853d6b4b63d05eed61f97e078","generator_receipt_sha256":null,"identity_record_sha256":"83d65e18df6abcd124b640d68f38f75296045dc30136a8095de98e9a56ff6059","prompt_token_ids_sha256":"f32d5f100044bbea939023be6610b696d665c2e49ac55c3b2478fd990e92c75d","ruler_category":null,"seed":null,"selection_rank":46,"selection_sha256":"1fda23ecb8320de65f93f8ffeca9bc66d1d6ba82fa80bd9e2d70cb080ea19be8","sequence_length":104,"sequence_token_ids_sha256":"2095ea60e58d7a748f80d99738c7213d620ebe1064bc23e828b8298509455223","source_content_sha256":"251053bc7b023ed45edfd2a618f866b0c06fd77cb43c5c6c4b6ad81c515873fc","target_token_ids_sha256":"8ddb65e178ceb2fa2d463b1981c1b2a0d308bd079411c8156ce1cba86f28173f","token_span":{"cache_exposed_start":104,"cache_exposed_stop":104,"prefill_start":0,"prefill_stop":83,"scored_start":83,"scored_stop":104},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e3228f24ef2326d02eac4d51adfc4dac5f98a046b3f1fa2c9d460bc590e7ad00","anchor_positions":[9,20,30,41,52,62,73,84,94,105,115,126,137,147,158,169],"anchor_positions_sha256":"1de8f9c2ad33c2ca63d619d4e737066294b86b501bd566dd6f84cd9fa731efd0","canonical_id":"867","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,30,41,51,62,72,83,93,104,114,125,135,146,156,167],"fisher_boundary_sha256":"6cdf085217e9a92a862d08a98b72af716319adcb02639b733b87d792f63b1261","horizon":1,"input_positions":[10,21,31,42,52,63,73,84,94,105,115,126,136,147,157,168],"input_token_ids_sha256":"b3f41b7d18076730b37f241371d12f38791b7734d302d899fcfb2ae98bfa7e20","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,32,43,53,64,74,85,95,106,116,127,137,148,158,169],"target_token_ids_sha256":"f68aa42c7401aa2aec59c231aba070689444ab604d38e31d8644873caf22a672"},"formatted_content_sha256":"c5d4a00fec65b64c0d75bc557b515c301fadade8597055f2ebb3d094051ad750","generator_receipt_sha256":null,"identity_record_sha256":"ac31836b26155996736a3cd0eebcf92e2d25df1fbba42f351c07926a34d557cc","prompt_token_ids_sha256":"3965eb8ca467ffc97d00622d372a5e88bf51d885d8f90daaad663ac6a3125362","ruler_category":null,"seed":null,"selection_rank":47,"selection_sha256":"207c6e9619ca7dfd647996b0ddc78892694e1df2e515a63deabfce4784b7300c","sequence_length":170,"sequence_token_ids_sha256":"a45517724ce7682620b190d1954ae73f1c66fd716f768ae8411b3ebd337b8523","source_content_sha256":"2da1fe1d0cc596300fd25d9a0f0e96da35c84b424c329057298f8e3540157f26","target_token_ids_sha256":"c7498f64ba251e766cd9dfe1464d55b2e2348ec90765cbe0ec26f908cbac2a9c","token_span":{"cache_exposed_start":170,"cache_exposed_stop":170,"prefill_start":0,"prefill_stop":114,"scored_start":114,"scored_stop":170},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2f3dfc43992ed341ed7978295644b024d0c191c9ec9c0ab7120ce608b8e69aa6","anchor_positions":[22,45,68,92,115,138,162,185,208,232,255,278,302,325,348,372],"anchor_positions_sha256":"3ce5659702a26cd1b59b1d16461526035d0580f1b9adb943021fa501179be80c","canonical_id":"622","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[22,45,68,91,114,138,161,184,207,230,254,277,300,323,346,370],"fisher_boundary_sha256":"3dd21721af5c559d0bf97f5a0d8335eac58669690242a9a65414cb07123ceab2","horizon":1,"input_positions":[23,46,69,92,115,139,162,185,208,231,255,278,301,324,347,371],"input_token_ids_sha256":"b7ce16f1b70f8a32ac421274ec67536496f84420741a5dccf20534b7d1c9f94d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[24,47,70,93,116,140,163,186,209,232,256,279,302,325,348,372],"target_token_ids_sha256":"d9f650368349431e7cbd102c1fa8783d9dc57c0d60cd0a5c4c784738af1f4c19"},"formatted_content_sha256":"216e5e12034d35e11932dcbdc1b82573653565c8944e50d962328af76cf74782","generator_receipt_sha256":null,"identity_record_sha256":"6ac868c67f1d02fb4ef0c5a4472eb0b28625461b9b81108f54d3aa3aed1b7d70","prompt_token_ids_sha256":"521456ee3e26ec55cb42c2589084173f76e5275561b51201d380d4eed502f265","ruler_category":null,"seed":null,"selection_rank":48,"selection_sha256":"210000c62df8127b95ae009f6518f2d6201b9506f6c8506835dca4eb8bbf293f","sequence_length":373,"sequence_token_ids_sha256":"1407f58699c66ad8f21f74fcf9606d047865d22dea2021a94e18700ba9dec3a8","source_content_sha256":"2000263b26b6240378d05d98725621f6c3e2bdd39265a3d356bf34ba4967d9ee","target_token_ids_sha256":"4676880dc99f1babf2689d76705c312671bc2ff81019432ec05b98c6156e2587","token_span":{"cache_exposed_start":373,"cache_exposed_stop":373,"prefill_start":0,"prefill_stop":189,"scored_start":189,"scored_stop":373},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"38589b6a6ebd3baba9bd23ed78bdceef6846e6529843ad4232525292bdec2cbe","anchor_positions":[6,14,21,29,36,44,51,59,66,74,81,89,96,104,111,119],"anchor_positions_sha256":"e83bd2e331c896fb6e9946121cbf13d77b4819161e7591880bc3055c1dbc069a","canonical_id":"677","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,13,21,28,35,43,50,58,65,72,80,87,94,102,109,117],"fisher_boundary_sha256":"683f76b65c4ebceff74d9ba1f3da44c4048119e0043947c4b3fe743eb29acf47","horizon":1,"input_positions":[7,14,22,29,36,44,51,59,66,73,81,88,95,103,110,118],"input_token_ids_sha256":"5605175529bcf8df342d6991176c6dcd594b6850a1f276f1e516f05daaf07e33","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,15,23,30,37,45,52,60,67,74,82,89,96,104,111,119],"target_token_ids_sha256":"25df50b9075a9d7a5a4d1e2bac00e89268616661a6d17f51d339ca2bb9a7ad99"},"formatted_content_sha256":"441853ff923cc424ab0aa7fec31c4aaa73bd11a24cff64d1daad0da0b63dda69","generator_receipt_sha256":null,"identity_record_sha256":"0d876619f10e2042398c541c7464cf9b347fb64e1ac8107841be3973e213cc36","prompt_token_ids_sha256":"0d2e23bc0f9b51a958475c21ec2c44735aa0f223221380fb0cbc53e877eea9e9","ruler_category":null,"seed":null,"selection_rank":49,"selection_sha256":"213beddb6f6d7179f5bc0d037d084c135da38be5e6cebc224e9ee64574990d1b","sequence_length":120,"sequence_token_ids_sha256":"ef85c6f458a4c28417a4ebf89c6986376b20129c419177fae9b89453d5ee6bad","source_content_sha256":"33110b4565d527f05bb143f7b956a5df85324d2da6b82595171ebb52b1c28695","target_token_ids_sha256":"cb45a27d57424658ef3029fa23e359436aec54c0636af1e913e0b7b0b1a63c83","token_span":{"cache_exposed_start":120,"cache_exposed_stop":120,"prefill_start":0,"prefill_stop":85,"scored_start":85,"scored_stop":120},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0af2d86cf1695b8e2f191abffee4365a9731009779912cf6c2993138f08a66fc","anchor_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,160],"anchor_positions_sha256":"cea919870992245499fb8d8a858d848e1e76f02a4e2230ada935442533c03198","canonical_id":"849","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,18,28,38,48,58,68,78,88,98,108,118,128,138,148,158],"fisher_boundary_sha256":"dacbce55717fb2ad9e2c1c565b4cdc586f652e71481489ba9f2fe0fbdb01d223","horizon":1,"input_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,159],"input_token_ids_sha256":"abbb2fdf32082b7edb9cffcb691959ba71fa426d6242d133388daf5b12207e54","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,160],"target_token_ids_sha256":"b6dd6b170c917647555dae8b2934c8757a2075a9fd62bdcc5c741d21a155b8c1"},"formatted_content_sha256":"b97e246c5fb6006e691f8dcc86a87679113f7a78d64aeac8c4bdcded6e3527a5","generator_receipt_sha256":null,"identity_record_sha256":"4e41b786817605fe72e6c9fdd1fee25d51c2700b517525bce5f2ed34d6095542","prompt_token_ids_sha256":"57f92b2bb27768c978d8e3c41a9efc9e0ebbc4d569c63d7d84dd70b17515d47c","ruler_category":null,"seed":null,"selection_rank":50,"selection_sha256":"21f0779cb4b05ae80d3df3e594cf70e52fde9b8adc7fd7457a46be4c7a8aaf45","sequence_length":161,"sequence_token_ids_sha256":"aec4c452b53bb10d8fe6bcd4d1f7770db728b6d1f1928e376b6757e469976806","source_content_sha256":"fedc2450a7af5f522a92d0e1052fa8033d9fb19a48f2d667a7f31093a26f405c","target_token_ids_sha256":"45ed26c261c710a2cc312afbf071387265818704baa2885344b040c5f1d5b70f","token_span":{"cache_exposed_start":161,"cache_exposed_stop":161,"prefill_start":0,"prefill_stop":75,"scored_start":75,"scored_stop":161},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"f8a20553c70e4c3efaf0458db6c7c1d06d0e2dfc9ab4f2783f010af84e82721d","anchor_positions":[7,16,25,33,42,51,59,68,77,85,94,103,111,120,129,138],"anchor_positions_sha256":"08b85b98703c7ea55928e16b78a87011e7254eb0d0516881c1cef449b5ad76ae","canonical_id":"756","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,24,33,41,50,58,67,76,84,93,101,110,118,127,136],"fisher_boundary_sha256":"859315451573e13275eaa90fef292f4abcb53e2d7979744ca1a559cd552531ab","horizon":1,"input_positions":[8,17,25,34,42,51,59,68,77,85,94,102,111,119,128,137],"input_token_ids_sha256":"c70dd0ee0947d887043e4ef090390d9b55dcb4940348ebb271a5299668f94c6a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,26,35,43,52,60,69,78,86,95,103,112,120,129,138],"target_token_ids_sha256":"844ca5f05e19572138e424833fe71cb51a25987fd79b470130b42fa975c1a39b"},"formatted_content_sha256":"79bb829fa41f1b56a259ec1d6f01ea3454814c377128ca4380d61741eac44e55","generator_receipt_sha256":null,"identity_record_sha256":"1eb2616b62b6f8e94a42917d3cba2e04e70983835585b47745bb90e8e2aba08c","prompt_token_ids_sha256":"f39e99e8861e0fba4e540dc710f8fcf78133eb7be50e166ae431c650dfe3c4c9","ruler_category":null,"seed":null,"selection_rank":51,"selection_sha256":"21f1cc6a98b0063838189643fcd01e4b5a84fbc7b4cc8effe18cc4d33580ac9c","sequence_length":139,"sequence_token_ids_sha256":"c8504b3fc7419f64b9e0f081c9f129180471f53a90800918c11449b486ba7987","source_content_sha256":"cb27d1dd0a93a0f5b25d8f2486a5017c8d78e16aaed3c07e71a04cc440d1aa5a","target_token_ids_sha256":"66b6c358b326ed60b26f60ece3aa3a72bae96813ca059d87e44cc7dfcdb72c3b","token_span":{"cache_exposed_start":139,"cache_exposed_stop":139,"prefill_start":0,"prefill_stop":91,"scored_start":91,"scored_stop":139},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e32814316554b0f79cc5669981964cddf032347b4da8365b25ddadb67a9a91c9","anchor_positions":[6,14,22,30,37,45,53,61,68,76,84,92,99,107,115,123],"anchor_positions_sha256":"4161f0e0da9e01e9fca23a34e70979683578e6b7ae825c0791088249cd49ed8e","canonical_id":"960","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,21,29,37,44,52,60,67,75,82,90,98,105,113,121],"fisher_boundary_sha256":"1e87180a8c7bf1f65c301b64b88ff3c34dd59beba4dc53810695b834c0c5e9de","horizon":1,"input_positions":[7,15,22,30,38,45,53,61,68,76,83,91,99,106,114,122],"input_token_ids_sha256":"3f2fcef30c7d427d14cbcac1db88f5e74dec170dbc068333bce564584133cb71","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,23,31,39,46,54,62,69,77,84,92,100,107,115,123],"target_token_ids_sha256":"cbc1bf9d8cb5df620060a192384df8638cb094155f8978e9bcebfa5f6b4e3276"},"formatted_content_sha256":"3e4da97d0bddb12409d97ddcd69beb695eb66abac525bc59990220b07aa13b42","generator_receipt_sha256":null,"identity_record_sha256":"f15b0404946c938be5d272c212d67ae55ec18c9f4abddd2bf888443f303e871c","prompt_token_ids_sha256":"d4bcbfe95a1ebf023872fe53289683ee41d253e100dab52d598c34e9239b284c","ruler_category":null,"seed":null,"selection_rank":52,"selection_sha256":"2296dea570e1139150ccd317f7f4064da5f91c662c46dc92bacd2cafbd8693bc","sequence_length":124,"sequence_token_ids_sha256":"9c259585007e9ef7b2dde2fe8a3ecce5850274464b4c662d95e215932a3c753e","source_content_sha256":"b9eab9a91cd92a96f9a16006fb04105309ec0c2a0fe6dcbb8b8d6fd5c5ad7e81","target_token_ids_sha256":"ff3e6cdf6ef65136f4d5fc6ee8370dd4b879f47033e6e5ea58283168cdb8eae9","token_span":{"cache_exposed_start":124,"cache_exposed_stop":124,"prefill_start":0,"prefill_stop":65,"scored_start":65,"scored_stop":124},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0dad65eaaf28f71b8151a7dd5bfd62b3f371591729ce9a325cd6fdb1ee5d99d2","anchor_positions":[6,14,22,30,37,45,53,61,68,76,84,92,99,107,115,123],"anchor_positions_sha256":"4161f0e0da9e01e9fca23a34e70979683578e6b7ae825c0791088249cd49ed8e","canonical_id":"741","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,21,29,37,44,52,60,67,75,82,90,98,105,113,121],"fisher_boundary_sha256":"b7544cc2c06d73eb766987d95229a15c4109c2c9f31ee7a8109e1411b1f01f8f","horizon":1,"input_positions":[7,15,22,30,38,45,53,61,68,76,83,91,99,106,114,122],"input_token_ids_sha256":"523dd1079553b7be63d71e5a3bbe2ebd41dd50b1105590b40352ba6d4ac94529","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,23,31,39,46,54,62,69,77,84,92,100,107,115,123],"target_token_ids_sha256":"2ee4f36ffbf325d3c0f22cbdeb5f755c29cd1766c4b85bc98367f2e4e3a615fd"},"formatted_content_sha256":"d9e493674ac0bc6520c04b25bf6f2a33ab4010024516b62c9111a1e85fd4b7ed","generator_receipt_sha256":null,"identity_record_sha256":"09fd441b743d4db344555f2991adfeebeff8cbe76ceaaaa25dd4bfc5c8a9c2cf","prompt_token_ids_sha256":"a05bc2e81b72ead55f4ab23081aa3740bd5e94b12c6910ff66b4d5cd8eadc104","ruler_category":null,"seed":null,"selection_rank":53,"selection_sha256":"23920e4407f636372457bfab6052016160b52662fc2489114a897651a77f02a1","sequence_length":124,"sequence_token_ids_sha256":"a1e0d450709d8ec3da71149a480fb07c80668b98425c657471e339ba4b919bc2","source_content_sha256":"1b782afb3ea77d326167c0cdc75cf3bafd8c54f38e6656373399c6745f9a6cda","target_token_ids_sha256":"97d18e68c69d67cc25a2770126b11ec9a4009b8103ec5fbc12cf9840d69f3f7f","token_span":{"cache_exposed_start":124,"cache_exposed_stop":124,"prefill_start":0,"prefill_stop":77,"scored_start":77,"scored_stop":124},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"5220a6e57a98f18ce0fc3020e6cb3e579946006f270473a46fff4ee28d307f6e","anchor_positions":[10,21,32,44,55,66,78,89,100,112,123,134,146,157,168,180],"anchor_positions_sha256":"ecfeff7f0aa8026e3a3e67c7fccf117f33e2cac59220b1af92163b27f13f2667","canonical_id":"763","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[10,21,32,43,54,66,77,88,99,110,122,133,144,155,166,178],"fisher_boundary_sha256":"10832530119cc9dc7e8f231ec358a5a567e0064379fc6c4d752e727059021017","horizon":1,"input_positions":[11,22,33,44,55,67,78,89,100,111,123,134,145,156,167,179],"input_token_ids_sha256":"d8e6511cf865ccef1847855dd93e81b5f9a377da5712f1e2b5e6b7a1d615ed7b","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[12,23,34,45,56,68,79,90,101,112,124,135,146,157,168,180],"target_token_ids_sha256":"98ea0caf56819312967e777bc6cb39044d229924079654c58f5313c625d125a9"},"formatted_content_sha256":"1b9193d68d5fbee2ffb750c200030e38b128c512a5dd53901cd0fbc6e9110bf6","generator_receipt_sha256":null,"identity_record_sha256":"a6f87c07fe76e3512de8f93c16a19c9ddf32b2cb8f73284e08fa27a983b4e54a","prompt_token_ids_sha256":"e163147f03192bd26bc2e593039155967245812898896a5b3ccb488a19a3c2d5","ruler_category":null,"seed":null,"selection_rank":54,"selection_sha256":"253e3c374826f96651ce10f0c8bf234e0019e15677351f0a5fa4ac234821073d","sequence_length":181,"sequence_token_ids_sha256":"b058735b952391450e5ec12b9e1a916c6996ae263f8a33dfff1ecab2519a6f62","source_content_sha256":"c7874b2b046cc08bdc1d3485818b5e8aeab82684fbbfe9b46528e3f93a8c34c0","target_token_ids_sha256":"6c2dfbee49b175c1cadd33dbec419143d5d228a7875f0d9be04cef929c3fa52f","token_span":{"cache_exposed_start":181,"cache_exposed_stop":181,"prefill_start":0,"prefill_stop":113,"scored_start":113,"scored_stop":181},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1f12a81f4bcfa4113d746de8d83b09b12d5bf22d5365a3e007cbd5533bc387bf","anchor_positions":[5,11,18,24,31,37,44,50,56,63,69,76,82,89,95,102],"anchor_positions_sha256":"09c822b94eb395a26f0742b1f48efa1c07ea626dba571c077685b9f6cf7f997a","canonical_id":"924","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,11,17,24,30,36,43,49,55,62,68,74,81,87,93,100],"fisher_boundary_sha256":"9076cfcc1469ae3fc7502f41192086d69ec6e14f6981282dccd0613cf5f2b721","horizon":1,"input_positions":[6,12,18,25,31,37,44,50,56,63,69,75,82,88,94,101],"input_token_ids_sha256":"9c80dd3459e67298ce46335c51e9430667ca9c812c445dd5bda165182570e1d0","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,13,19,26,32,38,45,51,57,64,70,76,83,89,95,102],"target_token_ids_sha256":"5e9a4a1aad513778895ed6ffc2980b76512073ab29d4f8af14ae4e26e16fedfe"},"formatted_content_sha256":"e2c6271b02f2f53ca02cc8d25e64ff5633054d7c8d080fc11d7ea4f2182c8230","generator_receipt_sha256":null,"identity_record_sha256":"3c4356ae7b50150676b6594053c05745a4c6c37d4741a1a66db1c3426dfd2121","prompt_token_ids_sha256":"ec37b31ed3931f32732446c5e1b53853e57d83b71f6a60ee0973fc8544644cd2","ruler_category":null,"seed":null,"selection_rank":55,"selection_sha256":"257fcddb413e4d8b091f6e8d2f933e3a3702db72c89eba0d10b57f6f9f5e5204","sequence_length":103,"sequence_token_ids_sha256":"cfe2397b7ef9e35d26711486c836646e4598be218d361850d477f09eb5e24830","source_content_sha256":"fd2c6ff7f8bea809d74c434b1e914cc7cad37c2c16e71760033867b551f8b17a","target_token_ids_sha256":"e984af6102dc89335e836cc40d95bac564e888d124d337cc3b4d2293c69a593c","token_span":{"cache_exposed_start":103,"cache_exposed_stop":103,"prefill_start":0,"prefill_stop":79,"scored_start":79,"scored_stop":103},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b4b039a0bf0a677663a27b0e6225ea5659bf81bf4816ccd4aa2be56fbfe16c56","anchor_positions":[21,44,67,90,113,135,158,181,204,227,249,272,295,318,341,364],"anchor_positions_sha256":"2f350e6327acbc5ad68acdc17f3114f18631a5418462d1efa07225d17dee32b6","canonical_id":"839","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[21,44,67,89,112,135,157,180,203,225,248,271,293,316,339,362],"fisher_boundary_sha256":"b3aaea1f73f541b61f04c11357ca1b117be9c6ed377f11f60a993f9550d29493","horizon":1,"input_positions":[22,45,68,90,113,136,158,181,204,226,249,272,294,317,340,363],"input_token_ids_sha256":"14db27ee204411b76eac1110cc2eae63a9f7b2b51b8dbe5b39e057e911aad683","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[23,46,69,91,114,137,159,182,205,227,250,273,295,318,341,364],"target_token_ids_sha256":"db3248528512ace9a5926f28510801745ad1c3f595f00a85a70220785378a26f"},"formatted_content_sha256":"1fa99f6cf27a6488f0fc65f628a1282d325113dc82a07170cace8f0ceb591a14","generator_receipt_sha256":null,"identity_record_sha256":"6b6998a44c9be26bdd92e8fd5c97777ab53fadd0893edc4f08c5b3238dd07613","prompt_token_ids_sha256":"4be0192a68dd883cc6c905e04481fe02c58edb320e51cffa3148acdb235c27c9","ruler_category":null,"seed":null,"selection_rank":56,"selection_sha256":"25acfdc7bd97d8b8918d952ccd13767a2fe3d9db392ff96de0914fe990c96767","sequence_length":365,"sequence_token_ids_sha256":"64f01ee3dd9e450ba6406f15a3eb8206442ffeb1f8bad35135cdc10036187849","source_content_sha256":"e16fff97f3fb1d8593c7f03f52f248825604aeb9e11bb511b613867c12fe0d69","target_token_ids_sha256":"beb8edb12ebba49a31efbb316488aaea2754c69c9a528a3fc8ce2e25cfd56f2b","token_span":{"cache_exposed_start":365,"cache_exposed_stop":365,"prefill_start":0,"prefill_stop":292,"scored_start":292,"scored_stop":365},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4e7d65195091102780a3b899d1d9c169f8999fd52b4fce6cd4807da888ce72a6","anchor_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,207],"anchor_positions_sha256":"db093e77a831efd90fa9c853cbfd07cb4a1ce073a3d26ebd584589b17a67dcac","canonical_id":"870","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,50,63,76,89,102,114,127,140,153,166,179,192,205],"fisher_boundary_sha256":"e55af1a7964dd59bb74a21c91ffa723d2be196dc03399d61f54ed5e27acc11f1","horizon":1,"input_positions":[12,25,38,51,64,77,90,103,115,128,141,154,167,180,193,206],"input_token_ids_sha256":"7e51406e9532ccd83f5838a79143de8b54bddcb3cd4af48ff57ce8fd2068004b","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,52,65,78,91,104,116,129,142,155,168,181,194,207],"target_token_ids_sha256":"8dfb99d49da149bda798f4f1bea7693afcc685391f66174bd7dbd7f9f212d23a"},"formatted_content_sha256":"0671c01c177d3d976dc26f74b1532f125da8dbbcba201bd969a8df89be404212","generator_receipt_sha256":null,"identity_record_sha256":"80065f067cbcd3b6b24a83d3b0920140345cdb44209861d9942cfc40e6285afd","prompt_token_ids_sha256":"11ac1cd8c2c5a0bfd9a43b00c6dbafc129c255c7fcab76b0fccec71b0a1ca47a","ruler_category":null,"seed":null,"selection_rank":57,"selection_sha256":"25d5e31e60deea1f490886549d37023191fddfed4a7488c2ecb258dc7b690a34","sequence_length":208,"sequence_token_ids_sha256":"b95e7638baa81487e766778b92d51418007619c24c88596cb9d255d17707ad82","source_content_sha256":"25b998a219b3e624a18b48a424e9201d591646f6ccffd2c34ccd994d27c33467","target_token_ids_sha256":"2c9edc0f2e71ceb46be5cf74a768dd85d0f0568d35da9346930b4437902992c2","token_span":{"cache_exposed_start":208,"cache_exposed_stop":208,"prefill_start":0,"prefill_stop":171,"scored_start":171,"scored_stop":208},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7179a32a80e84307ba53490fd3d26ee4976520fb95463fd4d99c5eeea5f95e3f","anchor_positions":[10,21,32,43,54,65,76,87,98,109,120,131,142,153,164,176],"anchor_positions_sha256":"1a7be41f5dd5dc6797d88e73a587747cdd29dd38e9373f3f584d8051e8e93686","canonical_id":"862","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,31,42,53,64,75,86,97,108,119,130,141,152,163,174],"fisher_boundary_sha256":"1eb3dcd5d0f40487914047d0829182878435e58383687684c8e2311b203aae37","horizon":1,"input_positions":[10,21,32,43,54,65,76,87,98,109,120,131,142,153,164,175],"input_token_ids_sha256":"ab632089e4e539c8bd8472cc5c658a6c7457d8f007cff89f0e00204509423230","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,33,44,55,66,77,88,99,110,121,132,143,154,165,176],"target_token_ids_sha256":"a1faf4cf5f2ef1412bad68d667096b1c2dd483ffe1a6e641ce38ec7132214cfd"},"formatted_content_sha256":"16b2614d3f18e7e4f72755cb6542e4c740bbd15964d402f5b8deeaaf2091d4c5","generator_receipt_sha256":null,"identity_record_sha256":"40dd54e4e34693f385579ecbdbbfe9d627292003fa2bb0e3fa8bb8b2106964dc","prompt_token_ids_sha256":"a20a042120ba3b2dda09c804a843b20532bf4809c88d3fbe22620ef7ac56bdcf","ruler_category":null,"seed":null,"selection_rank":58,"selection_sha256":"26d4c81b5d924024df658aeaf2f322dcb51e74e59299d6c5bbd15f755f66be5e","sequence_length":177,"sequence_token_ids_sha256":"f51964e8ff8c85d8f090c0e12f2c0d8d6234b49c9b6256141d05ab65cbec2f56","source_content_sha256":"4ca1375a378716ffef02c6f4407d8935736ee95bd694754667893b4d008fe701","target_token_ids_sha256":"643aa7765da4c82ea6a56654b78d8a8b5489c8fe86d2c44a8b4ac1960295ddbf","token_span":{"cache_exposed_start":177,"cache_exposed_stop":177,"prefill_start":0,"prefill_stop":130,"scored_start":130,"scored_stop":177},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"c008fb00c4b75ed5a7d4956b63ea532e7def48f60c4527826df4af3da44bf419","anchor_positions":[4,10,16,22,28,33,39,45,51,57,62,68,74,80,86,92],"anchor_positions_sha256":"adf3767c4c3aef64d448368c378ec2e8623df9971d13f03dfbf0a245e0d32c61","canonical_id":"877","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[4,10,16,21,27,33,38,44,50,55,61,67,72,78,84,90],"fisher_boundary_sha256":"2b4cbf4ae8461adf7ece8f2d75ed8ba8fa4760a94e8ccbc37beabe48a2e53822","horizon":1,"input_positions":[5,11,17,22,28,34,39,45,51,56,62,68,73,79,85,91],"input_token_ids_sha256":"11d9eb276cb6e38836f4280a877421b19798f0a03c8c6a955cf798cffbc5e066","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[6,12,18,23,29,35,40,46,52,57,63,69,74,80,86,92],"target_token_ids_sha256":"9a807734bc0b75cf522407e78a2432d110e1c4f600034c299f2c1f1025bf651b"},"formatted_content_sha256":"f0d53963f87750fbc186ec354a9b0e5f12a9937a4a1e0aa5b14a393ae9209437","generator_receipt_sha256":null,"identity_record_sha256":"b69efa65a89b1b762f27440c2fa6d016a863c179526e602947c476c8cce0c500","prompt_token_ids_sha256":"decd083790a01b0d5ff5f3cedced5d9fc60db4ac239215f87d18bd13829340ee","ruler_category":null,"seed":null,"selection_rank":59,"selection_sha256":"27e1898502ba84f86677c9a8c281f209e5e5ca3ef3fa0ac74ccb2e5a0f96925e","sequence_length":93,"sequence_token_ids_sha256":"0c186015d98c4b3b835615571d680037c4e9ff03bc0bf360d7edcc64ea7e02ce","source_content_sha256":"a4d868515f48168874e3345764d92c8f9bed1be2211fa137d653955ce91e9231","target_token_ids_sha256":"05325f40cfa6f59d7dae81f8cf700730aebfff74fb1d0bfbf6df3b22888a7ec9","token_span":{"cache_exposed_start":93,"cache_exposed_stop":93,"prefill_start":0,"prefill_stop":71,"scored_start":71,"scored_stop":93},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"69bc3d226f2cb8a6dd05d7b71e6b82bd59279baf47672ff9264b7f90657b2ed6","anchor_positions":[9,19,29,40,50,60,70,81,91,101,111,122,132,142,152,163],"anchor_positions_sha256":"868d4ebf721b69ebc32ed80fb23287a93509d3cda6e8df7dfacdd2b56b0be622","canonical_id":"672","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,39,49,59,69,80,90,100,110,120,130,140,150,161],"fisher_boundary_sha256":"824632fca0be6e4fc45bd8a157d6c3d7331848be3ae62303610c9ced5cdc4fe5","horizon":1,"input_positions":[10,20,30,40,50,60,70,81,91,101,111,121,131,141,151,162],"input_token_ids_sha256":"4a43f0478e855c786a9ba41f412abef099a0aa6d336add8b63253a02536aff3a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,41,51,61,71,82,92,102,112,122,132,142,152,163],"target_token_ids_sha256":"5a469a6d9ca0e01f116bfc1713d0adff5f5c3384fa311978e20183e1b4e4cf16"},"formatted_content_sha256":"bd36f5a6ad55b181f08602b3596e02127fc18650dff36d4e78601272f208d8d5","generator_receipt_sha256":null,"identity_record_sha256":"b5b377fb132254b8031c1002255dcb18bec16d00c60ca2ad998a8649aee007f6","prompt_token_ids_sha256":"6b00d6059701e0be4b83e9dac571ad728ef3c38c00c26f87d6b67a85ef08f81a","ruler_category":null,"seed":null,"selection_rank":60,"selection_sha256":"28dac04649a15d3243c93bc4206efd4f0acd9190fdbbdfa2ab5345af799d2bf1","sequence_length":164,"sequence_token_ids_sha256":"db9853f0252606fe408f3f653940737ffa69d88b087eaf4b282f42551da66ed9","source_content_sha256":"2e61b0813b9f940f7d87c5121e73ed7a0b8b2d507012929e153f7b120c8c94ea","target_token_ids_sha256":"4282f90a8268a9ed7bc92f471b79152055c095686127816dd9ad5e82abe8893d","token_span":{"cache_exposed_start":164,"cache_exposed_stop":164,"prefill_start":0,"prefill_stop":87,"scored_start":87,"scored_stop":164},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1de6107371ef2883413c366d370b6b037d94c76ec7232ca2b34bb6d3efeed728","anchor_positions":[13,27,41,55,69,83,97,112,126,140,154,168,182,196,210,225],"anchor_positions_sha256":"87cf1b468f68c1b9ae4a445f5ca6ddac019379b906bcb9569e18bc77f8dab19a","canonical_id":"894","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[13,27,41,55,69,83,97,111,125,139,153,167,181,195,209,223],"fisher_boundary_sha256":"50541ef032b9ebd546fb5146735c5dc646a875ccea81f95e3135cfb092f777e8","horizon":1,"input_positions":[14,28,42,56,70,84,98,112,126,140,154,168,182,196,210,224],"input_token_ids_sha256":"29b3d6ebd87eeec326946bda12535cdd931e5c75481c27025e5044b0cf8d5ae0","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[15,29,43,57,71,85,99,113,127,141,155,169,183,197,211,225],"target_token_ids_sha256":"468a710354f96e062d0f534f2df0e373bceff5193c12fd9ae67b05776f4cb12d"},"formatted_content_sha256":"85fe4f431f987f9bcccff54e9fff464b3411229d14fbb421bec0c42bf02159f6","generator_receipt_sha256":null,"identity_record_sha256":"78c4954da5b1f2cd2dea1777612b390c51dd7f053d98f84f8dd93e4c16182ef1","prompt_token_ids_sha256":"366bcf6fff0a082cca6c159ef4ee352857b3adb5dc747a33fbe9d5a187cf1a70","ruler_category":null,"seed":null,"selection_rank":61,"selection_sha256":"2a07715411e7dcd5d13fd6bdab1eedf96aa272627f4dcec5410c9273f87bda59","sequence_length":226,"sequence_token_ids_sha256":"060aa83ed0ef449b43aebe0dba81de96624ea81a91b9f2bbf5346ec7ebba5d2d","source_content_sha256":"25f8c228a65fe0501ff126cf8215ef8c40f73ba446adccaac8696c0592d9cb99","target_token_ids_sha256":"5d62950c056f097d29c50fc4fa74da37364692566b85b25d8801ce7fd0cfae6e","token_span":{"cache_exposed_start":226,"cache_exposed_stop":226,"prefill_start":0,"prefill_stop":198,"scored_start":198,"scored_stop":226},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e9fc3e673c65e510fa4e60bee32295e16ef953843d59dc679325a149cb63954e","anchor_positions":[13,28,43,58,72,87,102,117,131,146,161,176,190,205,220,235],"anchor_positions_sha256":"a5ffcefdcb22ed36cff0992ac04165c50a214c58858b64309c2292bae4ad2df4","canonical_id":"911","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[13,28,42,57,72,86,101,116,130,145,159,174,189,203,218,233],"fisher_boundary_sha256":"a343b73549c45e626b75cab9d67da7ec20d01d1df92c5ea43c27940cd5ac7b0d","horizon":1,"input_positions":[14,29,43,58,73,87,102,117,131,146,160,175,190,204,219,234],"input_token_ids_sha256":"fd590f614ed1c54400b44a8923440159f0d548ea06103f72a56c7794eef8d355","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[15,30,44,59,74,88,103,118,132,147,161,176,191,205,220,235],"target_token_ids_sha256":"84f73363046430aa4e0c6c4dcd11b1cf627b0582b1c34fc1e0b65e35a6e9f9db"},"formatted_content_sha256":"3af453e7ae9642b7b3a6fcb1a5eeaa953e64e0f8b2a19e654ef7c76e72f1ff54","generator_receipt_sha256":null,"identity_record_sha256":"f4ea17bad40bae1caf3eba8695e7ad325112bd00019e2511ff3614fb4f3a37f4","prompt_token_ids_sha256":"0b618b4457d4e22f6ca61e219aee86cfa88f0ae704ed487315c854f83ad3c11b","ruler_category":null,"seed":null,"selection_rank":62,"selection_sha256":"2a3d19972c5985a3c0f494d32fc1cf627ec69e3ceffcf0239862d2c14a69ee16","sequence_length":236,"sequence_token_ids_sha256":"4b476aa638606f6f73086d180b8d59e8f59f03a7f0df8b934e603b7a121b0987","source_content_sha256":"fdcefafb28d63d4587a84cb4bad47b369563ee74a482a1216341f7f6dba5f2d6","target_token_ids_sha256":"de9d6a1e7f34b8648d9cc024467db18a0100561d5d78da9dd043028f0c4b1058","token_span":{"cache_exposed_start":236,"cache_exposed_stop":236,"prefill_start":0,"prefill_stop":171,"scored_start":171,"scored_stop":236},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7b01d8ec76dac2b7f8b13a0dee07ad2225b142de86be4c08fe417f349fab06b2","anchor_positions":[11,23,35,47,59,72,84,96,108,120,133,145,157,169,181,194],"anchor_positions_sha256":"366062d114b4b8abb7d67fff75b399eddce3c91d9f37bfd07f29cdd16e658363","canonical_id":"907","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,23,35,47,59,71,83,95,107,119,131,143,155,167,179,192],"fisher_boundary_sha256":"3e6fc1c5625bb4f9b6651ced2a909d037dc5aba4278dcfc13b85144e9142af0f","horizon":1,"input_positions":[12,24,36,48,60,72,84,96,108,120,132,144,156,168,180,193],"input_token_ids_sha256":"ede325c0962debe0dc616d99ffbf44bf49b11833fab47e5f51463bdf2ffd6011","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,25,37,49,61,73,85,97,109,121,133,145,157,169,181,194],"target_token_ids_sha256":"1d7cf9fe79dc3a0f99a9c1c0dc95972f78b47a89b20d976ae513c41b67a2b4ea"},"formatted_content_sha256":"99bd7655cb5e40f7f33cb82dee142cbc78854fc3a5fe6777c50f8ee8f6340cbd","generator_receipt_sha256":null,"identity_record_sha256":"8d7818a7f7ed0b3732c51529861e8d1f37b442f565e0755ae9412f4e544cb0d8","prompt_token_ids_sha256":"1d66bbe818bc1c82dece92e787302ed8be7bcca6dd90590e6de294c4d367d8be","ruler_category":null,"seed":null,"selection_rank":63,"selection_sha256":"2a42596fa08d9ebf6f8f895ed62f34cfc00f343a1b8fb9eaba4fce50e57632ff","sequence_length":195,"sequence_token_ids_sha256":"0239420cf0bc52b4ff133acfa7d4ba2d445aa4efa7ca99f95e10b1e7fb7bff32","source_content_sha256":"523274b507fb693b2bb38a9148d6f328b15704929202f43a1eadb3f695c038b9","target_token_ids_sha256":"8225f2a151273aa3c714667358b8a7f304f775c7bed4fa3e4aececc369b6aff8","token_span":{"cache_exposed_start":195,"cache_exposed_stop":195,"prefill_start":0,"prefill_stop":139,"scored_start":139,"scored_stop":195},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"dca37f2fe51bc394708fd6c2564b0cbf119163fd665608a4d93cd495b3998ad1","anchor_positions":[7,16,24,33,41,50,58,67,76,84,93,101,110,118,127,136],"anchor_positions_sha256":"b394d1b64998869298fd2b292d4ab84e6faf0b5a40eeb811c668f2563bb4df64","canonical_id":"744","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,24,32,41,49,58,66,74,83,91,100,108,117,125,134],"fisher_boundary_sha256":"31ee43d6395a61f1005ea89235db2d9169957c4fa0ecd817d09d92731e4bfff7","horizon":1,"input_positions":[8,16,25,33,42,50,59,67,75,84,92,101,109,118,126,135],"input_token_ids_sha256":"305b211372837f28d3c09a9195ff9da8870eb18255d1a5ade3aa23447ae066d7","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,26,34,43,51,60,68,76,85,93,102,110,119,127,136],"target_token_ids_sha256":"55671dc7b5a880e5aa6b49754d8ad5916d76392b74ba47f6eb14a67a99a65dd0"},"formatted_content_sha256":"24bfd2c8acd42c24dd0175dade66586d4ef9c8fba6740ab43be8a40d2344a8f0","generator_receipt_sha256":null,"identity_record_sha256":"0d3378f11bdab707d5a5e866bf31ece77bae782c5e98264606c8669811279e26","prompt_token_ids_sha256":"45cbb79cde4efb50926321115cb572632eaa8fc91f1b03e56edb1ec881aac0d1","ruler_category":null,"seed":null,"selection_rank":64,"selection_sha256":"2b946010dfbfbf789ca965f025d6a4ef990e3f3685bc221716f664a8af147cdc","sequence_length":137,"sequence_token_ids_sha256":"27302f59061cb4d77641b855190e68d1082efe4fa19ef17f6d1886ea443d271c","source_content_sha256":"388472fd029ca253a1e8ebe0b058b306b19e8d0a99d80ac0ff0cc80f6ef6401e","target_token_ids_sha256":"f529366ede3a080401140af1ed24d8b5dc889784a050980a1baf21988730f4ef","token_span":{"cache_exposed_start":137,"cache_exposed_stop":137,"prefill_start":0,"prefill_stop":106,"scored_start":106,"scored_stop":137},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"72f1915bbcebb3cd41116e7d18a62d81645a57d6ab6aefc5dd4d833040f03e86","anchor_positions":[10,21,32,43,54,65,76,87,98,109,120,131,142,153,164,176],"anchor_positions_sha256":"1a7be41f5dd5dc6797d88e73a587747cdd29dd38e9373f3f584d8051e8e93686","canonical_id":"858","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,31,42,53,64,75,86,97,108,119,130,141,152,163,174],"fisher_boundary_sha256":"dce1d589b941aa5d007836b839a80510d6ed27d69a6bc4826bc4f9d1be58fa8a","horizon":1,"input_positions":[10,21,32,43,54,65,76,87,98,109,120,131,142,153,164,175],"input_token_ids_sha256":"1f8edd352b430cc0ec42d176428e4dc421cd81f094568d0cef527f3c289d718a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,33,44,55,66,77,88,99,110,121,132,143,154,165,176],"target_token_ids_sha256":"bd43a32b864185de3d60dd13a588ea5c44371aff58aa9439eee1749afedfd427"},"formatted_content_sha256":"ffcfea4052a70d3b74318b9aa27367efd191b6bb39153e56b0d8c6be270bf7dc","generator_receipt_sha256":null,"identity_record_sha256":"4303eb5887d9002f18afa44a53d86830e876c3605b807968cb2327574864c1ad","prompt_token_ids_sha256":"7e2722d66f52a4b3e9bc2b499fe7271693a49bfccaf075db172096a06590501c","ruler_category":null,"seed":null,"selection_rank":65,"selection_sha256":"2bdf2de5737b5e9e9c804506d9b637c0b3e4c429c713402914877db9e163698b","sequence_length":177,"sequence_token_ids_sha256":"1bd4f0e350599ac78e054ba1f3313444f17aca30b3ab8a7d1b97b0a4db36795d","source_content_sha256":"30117a9c1f5c8fa1a71aa5d47b10e7b6a6ed631272d09a212703db54dc4e2b48","target_token_ids_sha256":"8597d05e46c539c2c6176333f41f2d43081ba451957d9e6fb3fab21ad0354486","token_span":{"cache_exposed_start":177,"cache_exposed_stop":177,"prefill_start":0,"prefill_stop":161,"scored_start":161,"scored_stop":177},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"90e208ba484962acfe5a450f2238cad06fa1b1e78cb9b693ad7f8e119d289e81","anchor_positions":[9,20,31,42,53,64,75,86,97,108,119,130,141,152,163,174],"anchor_positions_sha256":"3a23db0bdf537a9cc8cf4af3c5b682f02edafbf8e11933d0867e7f5a84a5d5fc","canonical_id":"908","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,31,42,53,63,74,85,96,107,117,128,139,150,161,172],"fisher_boundary_sha256":"3ddff55453bdfe39ff17279167d0bf79d250df7e3566058fbc8400d75148e0cd","horizon":1,"input_positions":[10,21,32,43,54,64,75,86,97,108,118,129,140,151,162,173],"input_token_ids_sha256":"a57955a9e52e60a3866dc92a0785d9ec79382d7f1e9d9a6fbe931b41cbb23f05","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,33,44,55,65,76,87,98,109,119,130,141,152,163,174],"target_token_ids_sha256":"08a89b4fe30bb44a957a32e4dbff9cce9f766b22eb990ab1c15ed78cf1409870"},"formatted_content_sha256":"542026924d7c553982b9f71b589a6b03aa17c1a9071232b6b352a8d6de880b4f","generator_receipt_sha256":null,"identity_record_sha256":"9ab67c828f5ebad51f93eefcc25a6f64e7b2efa682ddcaa0b13cdfaae34bb357","prompt_token_ids_sha256":"92dc6abd8d2bcf9552cb9117b89f426545c93d4bf69ad859fe2e22691583d830","ruler_category":null,"seed":null,"selection_rank":66,"selection_sha256":"2bfb0c5069394a29410fe17d912939039348a3279be0932efb83b1a7c531f5cc","sequence_length":175,"sequence_token_ids_sha256":"470fc1e9439dda64ef76f847132fd61d9cc29b4d1eae89d0d27b45dcf9b147bc","source_content_sha256":"7f2ca7ee05543293bafe5fc5ea510834c2eadcb6c383bdf04c72ec8b68a255ba","target_token_ids_sha256":"fad64c8dc2d584216528c7db193f37eff8c35a166fa9f081fea4f50fc1596f9c","token_span":{"cache_exposed_start":175,"cache_exposed_stop":175,"prefill_start":0,"prefill_stop":143,"scored_start":143,"scored_stop":175},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"ae2a13b9227b88c92f44ed18f4a1e74a1af336a5e505f3993199f634831d84bf","anchor_positions":[9,20,30,41,51,62,72,83,93,104,114,125,135,146,156,167],"anchor_positions_sha256":"eaf3a0e814f88177633695c30e9a05b6b492aa64c90bf98da98cb9be3917c812","canonical_id":"695","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,30,40,50,61,71,82,92,102,113,123,133,144,154,165],"fisher_boundary_sha256":"00b58a22e1ec099f9f89a453c68d32a00b7de79e01353970212f4486aea7a91b","horizon":1,"input_positions":[10,20,31,41,51,62,72,83,93,103,114,124,134,145,155,166],"input_token_ids_sha256":"0d02b321a2c7a93e523c4102c0a515f78295358243305ecf63572a9b9251f793","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,32,42,52,63,73,84,94,104,115,125,135,146,156,167],"target_token_ids_sha256":"479412a3d9bc5c68516840b01dfe7229e1ffffaec904e9405309df32f66e5318"},"formatted_content_sha256":"2c108150869bff59ce632e2f1cf6126ae8d1e5449fb81fa352e6080ece8e18d8","generator_receipt_sha256":null,"identity_record_sha256":"b8d1fbbaf80e50801cbfefb607f87f2ee0be785618f9d8733e4ec1cec193070c","prompt_token_ids_sha256":"12d9523360c9b9bbbc77331eede017f4df2b2073b06793fb6caaa4264ed356eb","ruler_category":null,"seed":null,"selection_rank":67,"selection_sha256":"2c8457d859c30b49c20f86b5a6d57cb9d3f87babdd78e191666700bc6b19c2a9","sequence_length":168,"sequence_token_ids_sha256":"61a8f24e31170e6510a3fcb1c13e03df674e6054bc07e8c1b50b2869262279dc","source_content_sha256":"f26ba058dff18925efba9c51a7c584d713db98910e092b2b8efad4050d705a97","target_token_ids_sha256":"331d6876269e5c72808d45d0ff8043245793020b7f0c47f8dff86c3d5896440b","token_span":{"cache_exposed_start":168,"cache_exposed_stop":168,"prefill_start":0,"prefill_stop":124,"scored_start":124,"scored_stop":168},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2776f19ea8ad43a4ca532ba12548c47a68fb5d577db33f2001594d30daae4b47","anchor_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,127],"anchor_positions_sha256":"be2b9b89820981ce1f31651cf7733d2ed9a608d27c6984c26e1c299ac3d463c0","canonical_id":"674","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,30,38,46,54,62,69,77,85,93,101,109,117,125],"fisher_boundary_sha256":"9fb839c7596b94297977a40da9a8bdea3ac37720cc556e470eac01fad7d8fd26","horizon":1,"input_positions":[7,15,23,31,39,47,55,63,70,78,86,94,102,110,118,126],"input_token_ids_sha256":"4f531382690861c06fb87e927c47ddec1b08260da41fc61f8ade85e2dbfa2447","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,32,40,48,56,64,71,79,87,95,103,111,119,127],"target_token_ids_sha256":"58f5f25b01c19e81aae2e0327e1c394b668618a5926e174aeb8d4db536ad7b5a"},"formatted_content_sha256":"11e4332442cb1751b96af99218e5ed832e0741b4bd279519954cf8ff75df3000","generator_receipt_sha256":null,"identity_record_sha256":"91c60f95fb1de5699a93eac54f860ec4b636acdf00d4bf36d47cb7038568d0e6","prompt_token_ids_sha256":"fc30e67c09e80fb8040481e2b49df5d4de0abb2ca503e59d5f7fe81b9053ea14","ruler_category":null,"seed":null,"selection_rank":68,"selection_sha256":"2c910566f006288010eaa2a19e63636d5cdb71c4dbe532bbb2f800b8bc2d83e8","sequence_length":128,"sequence_token_ids_sha256":"c6637a5c180bd2cb7b6ae0114077150e40aa103770a4414d373677e6197d3eb0","source_content_sha256":"b085122555255bd8b48a5d7202b27dcfa83512407504a3a7d36c26ab0971701c","target_token_ids_sha256":"bf8896134a16b2defd1fd3ad3a1e041bed0c2522bb90fb8750e2d750970281f4","token_span":{"cache_exposed_start":128,"cache_exposed_stop":128,"prefill_start":0,"prefill_stop":92,"scored_start":92,"scored_stop":128},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"62ab5323d823671ba9386621641396cc44f5d6d879e1ef5cedfd73f5d42ea49d","anchor_positions":[5,12,19,26,33,40,47,54,60,67,74,81,88,95,102,109],"anchor_positions_sha256":"04b0f7fc4a0267db36244bf79b1775bc8efc674979a8a894100f0164758c7107","canonical_id":"937","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,19,26,32,39,46,53,59,66,73,80,86,93,100,107],"fisher_boundary_sha256":"d621358bf5b31a9a6e1bf8b7861671a5c3b7d8f4b833117f22528a32214d9059","horizon":1,"input_positions":[6,13,20,27,33,40,47,54,60,67,74,81,87,94,101,108],"input_token_ids_sha256":"6b99376f37916da3177e56d8dbe872f5b54a721dad5a56e3d0f7641537953001","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,21,28,34,41,48,55,61,68,75,82,88,95,102,109],"target_token_ids_sha256":"6caf9fd439e8d278a1020748e4978b74310b3608c9d071b94f12ce03d0b9be62"},"formatted_content_sha256":"3f1c5b9ff9268e2a55000da43f44ffd96c58159b9099ecb106bfc6d0e882d405","generator_receipt_sha256":null,"identity_record_sha256":"2aef9e476b0e90a8d4c54f89251fdd678232c30c0588a40c71c3ea135c3a140e","prompt_token_ids_sha256":"de6bf797fe9bca2c9736422f1027832ef62d4311eea81a92442bdf94a157a32d","ruler_category":null,"seed":null,"selection_rank":69,"selection_sha256":"2db747f786476cd53b2b76e66050c7c9524096bc2cc634dc8949c6700bbb91e9","sequence_length":110,"sequence_token_ids_sha256":"f66fae77a66de49a19604647a7a1dcef25341e73c3beeb2dfc54b05038100bfd","source_content_sha256":"9427a32773d2dda1a56f98cb0bdbee88ed5634a53a96ba4f2adf32764392e2d0","target_token_ids_sha256":"9d11355be764566fab4daa7e657723edeee80c1c0d05f8dcbfe888b93d368076","token_span":{"cache_exposed_start":110,"cache_exposed_stop":110,"prefill_start":0,"prefill_stop":73,"scored_start":73,"scored_stop":110},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"be38fbf54d72a7f68805fbe18ccbdf7981341fa3db7c90146ed5584b8a5632cd","anchor_positions":[25,52,79,106,132,159,186,213,239,266,293,320,346,373,400,427],"anchor_positions_sha256":"2c63cecba171fd759882d1a320b87c2dfaec15e4490a663f9e1099a229e8e934","canonical_id":"783","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[25,52,78,105,132,158,185,212,238,265,291,318,345,371,398,425],"fisher_boundary_sha256":"df1eb852686d1c5fa6f464d0608a8ec972c4dfdb569e90a9d61ac80e9dbbd261","horizon":1,"input_positions":[26,53,79,106,133,159,186,213,239,266,292,319,346,372,399,426],"input_token_ids_sha256":"b4cb4bc55e5f1df30d752a6409003f2a8cb96722509b10f768d46b67657faa6a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[27,54,80,107,134,160,187,214,240,267,293,320,347,373,400,427],"target_token_ids_sha256":"22fcb95368b58ec5a4c13f727526665b92519bc8752f8ab11b799ee2ec751029"},"formatted_content_sha256":"0d8b83c24ac54c8b3c1a41092dc210a7f174f870033838f69387ee0a959cb7be","generator_receipt_sha256":null,"identity_record_sha256":"fd3b5341382c42b888558f7ec9d99892ab4bc6c62b5fea552dfd262f705be6f8","prompt_token_ids_sha256":"3c17fc671c321556046900784e286f9b2efcfec63411dcd19bc16d80f81efb33","ruler_category":null,"seed":null,"selection_rank":70,"selection_sha256":"2deccaa412b35e79530ed8315fcf11416ca3ef9bf945badf8e69214d6c67f0e2","sequence_length":428,"sequence_token_ids_sha256":"923d6ae58844ab43d7c92cc96780db2f0a68359b436399f8f4b84e8bd73850df","source_content_sha256":"3bf5dd95dc2a714c277cb345634f0add55f38f0c23e30c02e2fac976cc5e3560","target_token_ids_sha256":"89d552cd5c74ddddddc518b6c4d8776915650ab6c296da75bf7d8dc8d5e8a3cc","token_span":{"cache_exposed_start":428,"cache_exposed_stop":428,"prefill_start":0,"prefill_stop":201,"scored_start":201,"scored_stop":428},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"d8cc6a91dfe0b8725032b9dbe17d00c2267d1f3903f0f46de298aef09a0eceb6","anchor_positions":[7,15,23,31,39,47,55,64,72,80,88,96,104,112,120,129],"anchor_positions_sha256":"c03e2e14b7ad2333a14fc199839382794d3c90c5842d21c3c2d26acb100d0b9a","canonical_id":"792","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,127],"fisher_boundary_sha256":"36cbcaa5f2a243260e4b704ad965c1eb8729226a615436df507566bab0a15d14","horizon":1,"input_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128],"input_token_ids_sha256":"43ea65b95cf29e7cb303d7b58a8466e49d86ff3b94f481b088aedd213565b50c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,33,41,49,57,65,73,81,89,97,105,113,121,129],"target_token_ids_sha256":"09ed1cfb8661ecaf52c064021fdaac9e5e45d85d244adc29abbadd32cbb23a1b"},"formatted_content_sha256":"84324342a9988ef788199714ffd6a7b7f6493516f982634aeb0fab6e891dca3e","generator_receipt_sha256":null,"identity_record_sha256":"9ea4728a95faf70fc41786042f177a0a108a52c817d7f798a50fa559e6bfd72f","prompt_token_ids_sha256":"e2b25c9110fe041500ee35bda9d717a5b8764c0af9684a331d986da4d503a12d","ruler_category":null,"seed":null,"selection_rank":71,"selection_sha256":"2e13ba7ae758055511fc93fd8b21f4c8279b266645abb65bad13f929bab6d4ec","sequence_length":130,"sequence_token_ids_sha256":"173382b09d1a8419bd02b4281c9d8ea5342d4cee26d9006629bdfe38b23fc3ea","source_content_sha256":"8b4ec72b56e7b421cfd5660acfeaf5323dcfcb622b00a3b4f8a55c2bb8a599ab","target_token_ids_sha256":"45b965687a80da78d6902f0642a6a41c15d5b6065995e352b178741376df5858","token_span":{"cache_exposed_start":130,"cache_exposed_stop":130,"prefill_start":0,"prefill_stop":117,"scored_start":117,"scored_stop":130},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b526b8b7bf92e5b467bf63c4c44c32a7d3baa99496c16d829dfc10c90ef8e179","anchor_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"anchor_positions_sha256":"debc2f860d51bf30c9ccfe7850d3428800e80a005f976ab2943a8642c33fb808","canonical_id":"848","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,30,38,46,54,62,70,78,86,94,102,110,118,126],"fisher_boundary_sha256":"eb5082f8394245be99a854bb9e95a0e8a24b830d5b4e12d9acf89075a71132d3","horizon":1,"input_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,127],"input_token_ids_sha256":"2488867fcffe8e28aca7f38c1051a25ba6e3576b5816a851d585a91e5d1fb9b2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128],"target_token_ids_sha256":"67aae898043f0810d7b6435c2dc11fdfee0f51cf372142272e7060f3d549bd8e"},"formatted_content_sha256":"030fb32fa54afa33ec26144f6963d266d6aa4b969906d8ea12cf708055b33d67","generator_receipt_sha256":null,"identity_record_sha256":"992345ddf332a9b3994c60bd506b547f20b7a90c821dab684435a5333bc84659","prompt_token_ids_sha256":"f4528ab0cef6a0aedb91c8fdfdd0fd4d1d19f26b5c42b52cbb5e9d55c87e496a","ruler_category":null,"seed":null,"selection_rank":72,"selection_sha256":"2e8927b105ba240d08737397b6e4ae7e731e45937512629b5d05c9e422358ce0","sequence_length":129,"sequence_token_ids_sha256":"bdb29154778fe78320ce09bc48fb722bce1a1cb08026d6843fa036de3c59c397","source_content_sha256":"2a1f6cc18c5e358f530bd92c65c306de3dce59f6ab5bf9e55ea9c27c9608156c","target_token_ids_sha256":"d4d73429c6a8b75c9e50462ee645fac5d41af8e10ac5bbeb75778385684a24e5","token_span":{"cache_exposed_start":129,"cache_exposed_stop":129,"prefill_start":0,"prefill_stop":96,"scored_start":96,"scored_stop":129},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"a8b715cff28765151d67536c865a8213c840d7a8b7b255ed28fa0f4f4d86e4c3","anchor_positions":[5,12,19,25,32,39,45,52,59,65,72,79,85,92,99,106],"anchor_positions_sha256":"1ec66db99ded67a6b74f7dd5259127f2ba61fd9a33265762c1ea04710558d7bc","canonical_id":"973","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,18,25,31,38,44,51,58,64,71,77,84,90,97,104],"fisher_boundary_sha256":"9a38f0cf45f0d18c982e6cd4e3041a5c05aaa808ae9c3cc11acd1e5bd81ba1ef","horizon":1,"input_positions":[6,13,19,26,32,39,45,52,59,65,72,78,85,91,98,105],"input_token_ids_sha256":"beda690cd5123bf4bea71d0d29b89a37efdf1c91f271681dad45a48428687cfd","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,20,27,33,40,46,53,60,66,73,79,86,92,99,106],"target_token_ids_sha256":"b2959f450541356aa81bb0e5965581933b267e9e8e2f7bff19f500f4677cd5d4"},"formatted_content_sha256":"03a49d9fc1d2e40de0d006e558a4cc366fe5fc106b78e9dcce28bf991852272b","generator_receipt_sha256":null,"identity_record_sha256":"92e9205a716392965076d0c07b2927b8ceb264553c8ddf5e51d643b2c4447675","prompt_token_ids_sha256":"a60f5de6590c6b61e5ed13b415d33c77fd42dcd611c101939112501a21966e85","ruler_category":null,"seed":null,"selection_rank":73,"selection_sha256":"2f2ec7824d30ee016341d9347bf3033c9e8baefcf7f210237aece22c50a0c793","sequence_length":107,"sequence_token_ids_sha256":"02a78c628534fda7b94bf5fce8dc4c42f06805ffe169c5e88e972974217f23cd","source_content_sha256":"535cf2dcfdda3927e31a863cc6592099702642031c1efc3a80742b6f3ea4493c","target_token_ids_sha256":"8ad5d65ce29243858983f2e234e934222f9ebfb842ba646ed2857ea2ba6d2d5a","token_span":{"cache_exposed_start":107,"cache_exposed_stop":107,"prefill_start":0,"prefill_stop":81,"scored_start":81,"scored_stop":107},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9356ecdbe5e79b88db404f8fcc1cc4066488ab817a3a8d2c38916554dca11b22","anchor_positions":[13,27,41,55,69,83,97,111,125,139,153,167,181,195,209,224],"anchor_positions_sha256":"1b59984280d92d0f5b008fda81e8c1ee99edd8adebbc3555a45f9e3b6ccc7b43","canonical_id":"648","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[12,26,40,54,68,82,96,110,124,138,152,166,180,194,208,222],"fisher_boundary_sha256":"6a4524107689341a41d25534167f49a57a6232f42670a7ccb3c3c53953ca52d7","horizon":1,"input_positions":[13,27,41,55,69,83,97,111,125,139,153,167,181,195,209,223],"input_token_ids_sha256":"5d718cfe7aa9550f8596ed8028ffa484e5dde6a58cf75d5e564d46b03ceb80e2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[14,28,42,56,70,84,98,112,126,140,154,168,182,196,210,224],"target_token_ids_sha256":"29ce1ee9257823ee60fc561f31cf4387cbb83ef9304ba18e1878faf829fa4c12"},"formatted_content_sha256":"03ce2215aec8d5a70b9b7d5d9457a3049c63a14d3fb41d268cb7d640203b379e","generator_receipt_sha256":null,"identity_record_sha256":"47caea8e58a24ad7e9189aecc0d5cd34719c3296041ecb933c3966a471cb646f","prompt_token_ids_sha256":"c856d4839a1ad8ebc26aefea0a9fb07e5436dd63c6a305c7b4112bee6193dde4","ruler_category":null,"seed":null,"selection_rank":74,"selection_sha256":"307a74054631fd4611a4637c7dcc3b5902cc9b2530ad55e1817a7b66e0244a87","sequence_length":225,"sequence_token_ids_sha256":"996568b0b6b68ffab6ac7da97e43dcd7e5fdaabcdae5248fcf638e8c03d9454e","source_content_sha256":"c078071a1981a4c9334368583a6c9b84c5bca8b86cdedb85a12e8be1b1203123","target_token_ids_sha256":"c8d5a3a6d636c7fc1ee25da058f4b6f9c67be6202458fb01fa42861b8939a7a5","token_span":{"cache_exposed_start":225,"cache_exposed_stop":225,"prefill_start":0,"prefill_stop":172,"scored_start":172,"scored_stop":225},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"806ebe5a2818da11223ca67ee7cf2265f233845126be6d099c9dcb15c55fa08c","anchor_positions":[9,19,30,40,50,61,71,82,92,102,113,123,133,144,154,165],"anchor_positions_sha256":"f50a8793906ae0ec9418f440c4efb82f7805f24bf465b189ea08ad0bcb356f8f","canonical_id":"772","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,40,50,60,70,81,91,101,111,122,132,142,152,163],"fisher_boundary_sha256":"1b7250139134ecd49bde104b1a6a2510822a0f9dfb9d2d4e9e5eecf763a8b9a8","horizon":1,"input_positions":[10,20,30,41,51,61,71,82,92,102,112,123,133,143,153,164],"input_token_ids_sha256":"3e9f5ed8311d8d669c372b59caccce107e54dd1559931f673bbb25676156cadf","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,42,52,62,72,83,93,103,113,124,134,144,154,165],"target_token_ids_sha256":"43edf81c690bf9f561ba94748611186c5eed3e9186c085b2808196492de0f6e4"},"formatted_content_sha256":"b065a8530085bc045a0533e1eeaf8aaf8188f595824a24047f8870dde69f04dc","generator_receipt_sha256":null,"identity_record_sha256":"9f12e75bacb719e1dd7725bb446586e29d53441710a6bd0bee305318116aa5ba","prompt_token_ids_sha256":"deba03b89b8f2bb57ae57611e69505484898b45ff63c512c33597cb6fcdbb1b9","ruler_category":null,"seed":null,"selection_rank":75,"selection_sha256":"3116bf2b1674d3472c0f8a996afd05f7fc073f3720ffdb297b5d89cbc3eccae5","sequence_length":166,"sequence_token_ids_sha256":"beeb3758c81c70dda2bba27ac2041d61adaa245c5d8d0edf701cdf0f91f45c4b","source_content_sha256":"1ae2ef8f6c027aabd4091686f02cddd085364dc8ca4f35da3877f375ddbecf79","target_token_ids_sha256":"920ea1a628b222d262ce367d32d278f0b295a5028fbca3295770bcff04e208d9","token_span":{"cache_exposed_start":166,"cache_exposed_stop":166,"prefill_start":0,"prefill_stop":117,"scored_start":117,"scored_stop":166},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9b187e5fe961b945d44953c99d7e41a4cbc730ca0aee47b0408c755b6bd9ad40","anchor_positions":[6,14,22,30,38,45,53,61,69,77,84,92,100,108,116,124],"anchor_positions_sha256":"fe7a10db15891cb54d23372b10af49697c0401f54d8f6b37c01417545e9ba930","canonical_id":"764","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,29,37,45,52,60,68,75,83,91,98,106,114,122],"fisher_boundary_sha256":"e6be121ec0065a1a1aaee9d9e86ba55d2577c56a8617ecadebe8c4e3e6fa2ebd","horizon":1,"input_positions":[7,15,23,30,38,46,53,61,69,76,84,92,99,107,115,123],"input_token_ids_sha256":"8b87308fede122de1697b74353b230a58de366e630c17539cb050893d296a569","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,31,39,47,54,62,70,77,85,93,100,108,116,124],"target_token_ids_sha256":"a3e5f3dfab994ff9d370511f35cb4ce1a6409b31076b0c2fbfc672290cd7dbf4"},"formatted_content_sha256":"dd3955ad349be1f3bbe53adc3231a0d62ae00e248214be3a7b5e980d326fc9fd","generator_receipt_sha256":null,"identity_record_sha256":"700a6d8a14194a1306ce6fb464aea0085a25f117d07fcf265169ed7ae1eb2406","prompt_token_ids_sha256":"66cf8821e075a67969485c8f8e2ae3f57b8384381fd08c0db59c080fc7ed3267","ruler_category":null,"seed":null,"selection_rank":76,"selection_sha256":"315964c2edf3979db8272d920dfef4d6429237f6cc495a6ee3cd3d5549a6b648","sequence_length":125,"sequence_token_ids_sha256":"516180a745f3962c6178902c00099642c3d74217bedb6c055bb189918a4cff6a","source_content_sha256":"690c9b90840fcf0cd5119cfd171717a21c533ed2145ba5803ba93e78d2229bc2","target_token_ids_sha256":"17e59e0812b0551e0a212266dfde313e3b8b138e97f059cabb2ac99726994b05","token_span":{"cache_exposed_start":125,"cache_exposed_stop":125,"prefill_start":0,"prefill_stop":75,"scored_start":75,"scored_stop":125},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e95a3332e3f7eb017e36ba83694e18fd59804b725bb678bfc239e300e0f3db3f","anchor_positions":[9,19,29,39,49,60,70,80,90,100,111,121,131,141,151,162],"anchor_positions_sha256":"937bf130ee4ebb63677095c86a2168cbc15816e3cb8813d07c1ce8340b5fe4d7","canonical_id":"710","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,160],"fisher_boundary_sha256":"6be547ce84d00ff5e76edeb90cf3d18f3f42ed8471ed55e075a934fcf78abcd8","horizon":1,"input_positions":[10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,161],"input_token_ids_sha256":"110af952e2d5d1fb75f8a65fa643ba6711e84effc65a18d37b8308133a6bfb65","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,41,51,61,71,81,91,101,111,121,131,141,151,162],"target_token_ids_sha256":"b28186d7106e163a34e3f412a706d4588d43b0357f9c57bee118d5b51160b7ee"},"formatted_content_sha256":"88240f422a8a69eea08b888c6b9b3c0bb24a6241a7a906d6c5bd4e4f75a9e655","generator_receipt_sha256":null,"identity_record_sha256":"1781d9f90e6dd08f1aba55a3d324d8fa14693f707ca4f071b3b392360bbe0408","prompt_token_ids_sha256":"be734d8f240da897ca816e1ebc35469363413920aebb0b334817102e135917d7","ruler_category":null,"seed":null,"selection_rank":77,"selection_sha256":"3214d5450dc1d7505961e5b95aa8e71c3b102461efcea184dc5d581da13805e4","sequence_length":163,"sequence_token_ids_sha256":"8b2dd644e6f00860a2723ce8ab4d5c0a9ab6aa316b27d063fc4a39356a29e075","source_content_sha256":"15fd5fe74d1087e4941f9d296c749df402bbce707b6f506e7dfb634abdac93fb","target_token_ids_sha256":"5b619991d998bbfbccb9502f7a36880ca09dab9061d34c4dd351784267f88e5c","token_span":{"cache_exposed_start":163,"cache_exposed_stop":163,"prefill_start":0,"prefill_stop":130,"scored_start":130,"scored_stop":163},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"8623504af00a9b25be7bc8e91298e9bf3d79e889943b43f81d79a76ff7b2b08e","anchor_positions":[20,41,62,83,104,125,146,168,189,210,231,252,273,294,315,337],"anchor_positions_sha256":"d86064c4e7cc49d03ee7921a0260f3727647f42da5038fce7e70f58100a9b429","canonical_id":"615","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[20,41,62,83,104,125,146,167,188,209,230,251,272,293,314,335],"fisher_boundary_sha256":"f7c95c163c354425c2add0a88c1c4185cf084e48e914cfdd216090db1b963bf8","horizon":1,"input_positions":[21,42,63,84,105,126,147,168,189,210,231,252,273,294,315,336],"input_token_ids_sha256":"9379deab7e7b61df7302e3f0d09b7e567d9ba26caa1886692eba4506c94310dd","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[22,43,64,85,106,127,148,169,190,211,232,253,274,295,316,337],"target_token_ids_sha256":"80d5c49f9ff9f2ebebef47d67d9b86438b015a070f8c6e2dc84883e9240b9ab1"},"formatted_content_sha256":"b9645bc5ec977dfe41735d4ce5d1f4a734b1f86ea78c4a5646565ebd8c15566f","generator_receipt_sha256":null,"identity_record_sha256":"9b128354be7b44adac95032db55fe4b66adf4ee89cf51ce82d87f3159b662f3a","prompt_token_ids_sha256":"51676de173a112846b34d4fba7305f57b81fb3f9d18e638cc2fa74f8e3a3e293","ruler_category":null,"seed":null,"selection_rank":78,"selection_sha256":"3285b6f67988631b43e377c409844a0c984cfd47e40b19a98584df2d8f61b362","sequence_length":338,"sequence_token_ids_sha256":"6bebf94cb834d1fcfe47dc541cf15d6fc87989ddf47fecdd7be2a90d0663d24c","source_content_sha256":"fdb94f9efd3d2f52a851821fe1708964cb54b4e59675eb6fbb5b37edf3910703","target_token_ids_sha256":"9714d922e9b518657f61cc75eda60ded6a3f1c50ebe5b59d1d6800481296fdd7","token_span":{"cache_exposed_start":338,"cache_exposed_stop":338,"prefill_start":0,"prefill_stop":310,"scored_start":310,"scored_stop":338},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"96cee3de971a921588b61de2e2cee579332da466f101701e569463c8c6a8fe9f","anchor_positions":[9,20,30,41,51,62,72,83,93,104,114,125,135,146,156,167],"anchor_positions_sha256":"eaf3a0e814f88177633695c30e9a05b6b492aa64c90bf98da98cb9be3917c812","canonical_id":"929","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,30,40,50,61,71,82,92,102,113,123,133,144,154,165],"fisher_boundary_sha256":"a4409da62e8624d4a36ab120ae46d80d5347e0949ddb59fa84433d595ab39c86","horizon":1,"input_positions":[10,20,31,41,51,62,72,83,93,103,114,124,134,145,155,166],"input_token_ids_sha256":"ef3502c0e2121d90870bb103716d7830c424bfec40f477dca54ebfeac256169a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,32,42,52,63,73,84,94,104,115,125,135,146,156,167],"target_token_ids_sha256":"0afc3393efde3a0d92114c3ef929fe4bb20b15a93254a54991f7854ac5d8fb1c"},"formatted_content_sha256":"8720c8227a224ac2e19aea8120b6280befcba03b75052032de9bf8e95ca77b04","generator_receipt_sha256":null,"identity_record_sha256":"1386ffcc0e108e711ebccb2df6fc089ab088b8bb5bf75534b414141f41a74a75","prompt_token_ids_sha256":"d3e5efc050ed2ac9f9ae235e854ba1d481168daa7af573882848337902db8cbf","ruler_category":null,"seed":null,"selection_rank":79,"selection_sha256":"3296d51e868118ed66c05092fa54df28b0ca574c48a0cba377e10c955ebf1545","sequence_length":168,"sequence_token_ids_sha256":"e74df24c78d5f48aa647690ed6d124d47ca43f1d31e9c157bbbd81fae882c552","source_content_sha256":"bc0b73708502c4d4ef970ea6606ef292432812c718cd0e89507be65055cb1c76","target_token_ids_sha256":"e0d8a0f88aa5d40291247005bc203563ae5853bbdc6122d0bc21fb681b59aab3","token_span":{"cache_exposed_start":168,"cache_exposed_stop":168,"prefill_start":0,"prefill_stop":145,"scored_start":145,"scored_stop":168},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9209ac953f696c6753bd12b7e3aee86341dec2daa6e7890b20b1f6ca9c923fa2","anchor_positions":[11,24,37,50,63,75,88,101,114,127,139,152,165,178,191,204],"anchor_positions_sha256":"48082390de232d2e911509462169a4b3ebe8a03329ea579a66fe38623c52b3bd","canonical_id":"872","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,49,62,75,87,100,113,125,138,151,163,176,189,202],"fisher_boundary_sha256":"099898e807c53f06b4fefe6112588ccaf699892b69781fb1af96a8f953dceff3","horizon":1,"input_positions":[12,25,38,50,63,76,88,101,114,126,139,152,164,177,190,203],"input_token_ids_sha256":"e90e1cdb40861c5ce546f0f793eca7fb22ff9a2d1fcd28ac90c1c3c41b2a9a8a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,51,64,77,89,102,115,127,140,153,165,178,191,204],"target_token_ids_sha256":"dfe340d203294bbebece21c27b39e76b677b6c6f99f69c75f1c05d25a052a6b0"},"formatted_content_sha256":"ae7cccb0f44aa7dd8ae1d59a7bb423feb46060df6bccc538c0d729941cf62f09","generator_receipt_sha256":null,"identity_record_sha256":"33a68c486b6fb35b93a963a3c1ab2bc5ff188b6005bdce71a974210943d77edf","prompt_token_ids_sha256":"c587ed25a2f2bfc478fbc770f969490260f64e578d5a442189b63d9a9743cf1f","ruler_category":null,"seed":null,"selection_rank":80,"selection_sha256":"32eea1bbd2a2735f50e8ed7192ad280642b4f864877e2e12c74229735f3eb364","sequence_length":205,"sequence_token_ids_sha256":"5cd1de46f6a6a54a60046c16ee57b387b99886733c7d130b2e28f1ae13d4ae9a","source_content_sha256":"7a15fb9218af2f77deb400a4399d8ac77545eab7979f345c0ea992388693aa5f","target_token_ids_sha256":"27908577789a6b8b4d0224aecb42ddeb2609c5ec146dca642424eb2a076e6e7f","token_span":{"cache_exposed_start":205,"cache_exposed_stop":205,"prefill_start":0,"prefill_stop":183,"scored_start":183,"scored_stop":205},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9e18bfe18f9c4da802617b4dee6331466e0e733317487a86f7aef33753525ee9","anchor_positions":[7,15,24,32,40,49,57,66,74,82,91,99,107,116,124,133],"anchor_positions_sha256":"6c1029515cd9fd840ee9baeeb3bbf16de22d914596dc66d6c0b00a017e82aef5","canonical_id":"804","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,32,40,48,56,65,73,81,89,98,106,114,122,131],"fisher_boundary_sha256":"0654213b540ddd6054c3a7a54b4ed68fe2ce52d54601d589e8dd8432c5e03c9b","horizon":1,"input_positions":[8,16,24,33,41,49,57,66,74,82,90,99,107,115,123,132],"input_token_ids_sha256":"e6dbcef9e02b4ece7b578fe3e362efe755dbd02afcae77d6d6f4edcd4de4f4cc","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,34,42,50,58,67,75,83,91,100,108,116,124,133],"target_token_ids_sha256":"0b20024476a22a5a23bff385afa2fb325aef7aecc21280ce17cd65007c86f1e0"},"formatted_content_sha256":"58c227c113ff5dfa4a446ad44a803eb54659e0fbefb3f5f80dbef45d0994bdfd","generator_receipt_sha256":null,"identity_record_sha256":"01f7ec235052c97847150a68e3de4b9c2d022df8fafdb46a04092b4a7ee18c27","prompt_token_ids_sha256":"927da8d69d8b88559b5811091f4fd329f0e47607f28c399059242149e3c7336a","ruler_category":null,"seed":null,"selection_rank":81,"selection_sha256":"3379395523a918b0a3ecf21510bcc8704655d52083f91c7d3bd323ce8ea3e272","sequence_length":134,"sequence_token_ids_sha256":"bc6201b3951727d59c9b6c88db4d0722a0b56896e2812e646426b637a86b085c","source_content_sha256":"0b30c40b1ec355476196d7e4b579e4c837c30947a206b5da87928bc338b51aac","target_token_ids_sha256":"ce0bec16e6ba57588fa2e620c280ea4fdf1f2da0b33b6d8c1bf55efd9f79a60e","token_span":{"cache_exposed_start":134,"cache_exposed_stop":134,"prefill_start":0,"prefill_stop":93,"scored_start":93,"scored_stop":134},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3078f2a2b5f48560ec6b5e149abdc2ec94224715ce8b8b558250fe31cd4dbf52","anchor_positions":[6,14,21,29,36,44,51,59,67,74,82,89,97,104,112,120],"anchor_positions_sha256":"7f8463e6889d5cbbd872abcc6f675ca04c3c0d8d69d266000933478b69489b8c","canonical_id":"713","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,13,21,28,36,43,51,58,65,73,80,88,95,103,110,118],"fisher_boundary_sha256":"12bd3ed950f8f6e2d1ea75538e2472a2bb7fcef26eeb014dce2f68e39c0ef080","horizon":1,"input_positions":[7,14,22,29,37,44,52,59,66,74,81,89,96,104,111,119],"input_token_ids_sha256":"8bf93fb6683667ca762c7e68944d126e5f2c2b6c44cd212a8000cdc09acb27e1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,15,23,30,38,45,53,60,67,75,82,90,97,105,112,120],"target_token_ids_sha256":"bb0137c2b21eefda36fe32ac45aa551c534cfa2ebce4a392aed849f5496ed9bc"},"formatted_content_sha256":"681d439ccf9ce8b1da99801aad0074b3790087d6efe044ae79d7e49571d76ac7","generator_receipt_sha256":null,"identity_record_sha256":"bf5c49f9a99a125e0728cdbdb1103ef16d8f41d27323feb6d66db4c28cbdf8e3","prompt_token_ids_sha256":"689f474ef4fa8ccced0bd23ab2314dc97972bacba54c2a54dc4d440098402b38","ruler_category":null,"seed":null,"selection_rank":82,"selection_sha256":"36a09fe9c145944020ea5abafade01ab7f4a1c538f55c32b2daf644698464fd3","sequence_length":121,"sequence_token_ids_sha256":"df03ea9ab5af6fe2ad270a5fbae4db0bf61e07b1919c60f124a8070bb517a052","source_content_sha256":"9197df3cdc3a76aced1eefeae0cf812e2f43b1c25afc3bd73c6e27fa82f1aeee","target_token_ids_sha256":"6b06cbab3e1a4e869733d23ddc800cba139bbf3020eda6b3ab38843880a60fa3","token_span":{"cache_exposed_start":121,"cache_exposed_stop":121,"prefill_start":0,"prefill_stop":90,"scored_start":90,"scored_stop":121},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"63b9e82e307a72c058464d1b1d8a7f3c58d7fb5152977d0a46c053ff527a82dc","anchor_positions":[23,47,72,96,120,145,169,194,218,242,267,291,315,340,364,389],"anchor_positions_sha256":"8e2b268c6192e66f463da8f9c055e0c9184253267e4f791d54da3db3eed553a6","canonical_id":"709","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[23,47,71,96,120,144,168,193,217,241,265,290,314,338,362,387],"fisher_boundary_sha256":"967b90e282da8354335e13a711b1db1fc3dc1cd2d5f3733153901609f0b7284b","horizon":1,"input_positions":[24,48,72,97,121,145,169,194,218,242,266,291,315,339,363,388],"input_token_ids_sha256":"14ec48de9ee49ea137494da470356e32fd2626ad45fbf1122fb508edd8791526","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[25,49,73,98,122,146,170,195,219,243,267,292,316,340,364,389],"target_token_ids_sha256":"eee798bbc46c0f86ea4c32017a079b8d80703d9f4c5b22ed2e2e6a9edc5c6096"},"formatted_content_sha256":"377241d6ba849220fead3711b06a3790ec65647bdb509adac60123fd052d34e6","generator_receipt_sha256":null,"identity_record_sha256":"1d437dc4e8346fe37a0eee9d9a423bd60824e5834a88120acefb46d1be5ecaf8","prompt_token_ids_sha256":"fd90d23dea7f7557e683b5a171cd89091ccc9e500b8c27afb3c9221f5d9788c4","ruler_category":null,"seed":null,"selection_rank":83,"selection_sha256":"372cfe33288b920e01911b67999a2534620fe6fd180024291e1b86f7f221909d","sequence_length":390,"sequence_token_ids_sha256":"c1e45e96c9091049c070bbc66c111935f57c6b5469213a4d3ca91d3d27085868","source_content_sha256":"1bd1bf4c9a1573325504ef4695c6a08c9641c60e85f7cf68942681af59f0c13d","target_token_ids_sha256":"2c233f19acde716535f8aed74a2b4c7a6ff031e2df4d090d3f77cfd880649eac","token_span":{"cache_exposed_start":390,"cache_exposed_stop":390,"prefill_start":0,"prefill_stop":309,"scored_start":309,"scored_stop":390},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"755396a4616427092f5857ae9f027899125cc2f2dc290df40ac96b2609861e3d","anchor_positions":[13,28,43,57,72,87,101,116,131,145,160,175,189,204,219,234],"anchor_positions_sha256":"019a1d4f16a491584b0b921a03e82e05ab8f21be64e2ef381494510440a2dbba","canonical_id":"854","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[13,28,42,57,71,86,100,115,130,144,159,173,188,202,217,232],"fisher_boundary_sha256":"3978f984679fc70aed874ca972bca2f58c59e79999bd8b5dbaaf51586a103e4f","horizon":1,"input_positions":[14,29,43,58,72,87,101,116,131,145,160,174,189,203,218,233],"input_token_ids_sha256":"f50bcffcfe9946342cfe243e55c6a21e7d36fb6cc50a570f25fa3180e679507d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[15,30,44,59,73,88,102,117,132,146,161,175,190,204,219,234],"target_token_ids_sha256":"d2eb23c855ed7281d5d4f214ea2ec6aa0309bff553e222cc09435b17b3a89dc7"},"formatted_content_sha256":"ad6817ab94e23314f897ae56c55d36b1bfcbf69a7b8d35fb818c81c03d39b3c0","generator_receipt_sha256":null,"identity_record_sha256":"341e83cf4392f6688e220933fa083c53bd3f177ce67aba89005be759ab1ef581","prompt_token_ids_sha256":"c469e78789638c0e407409fe9aebd074eba88c7033481e56ef2dec96fd271eaa","ruler_category":null,"seed":null,"selection_rank":84,"selection_sha256":"374c60a9450ed67d1bb35eb2406c9e00aaf02021508333ed279cecdacc65f059","sequence_length":235,"sequence_token_ids_sha256":"3434905939b518880cc623791311c79432ac4833d7f9739c8a2fd83b9590d85f","source_content_sha256":"2f5b2b2d153d57ad5deccfd519cfcd97185b0cedd8a8e32fbcc6bb3429ff8755","target_token_ids_sha256":"8109cfc76d8f3759529fc35b4e397ce62b4a87dd10005164d11c77adb9a1a048","token_span":{"cache_exposed_start":235,"cache_exposed_stop":235,"prefill_start":0,"prefill_stop":209,"scored_start":209,"scored_stop":235},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3125b8323a121faaca1072275076c923a67557e8121f55b5373de0a2d7742432","anchor_positions":[21,43,66,88,111,133,156,178,200,223,245,268,290,313,335,358],"anchor_positions_sha256":"8cbf29aa79890d46e276ccafa41f46a9c68b76575359288a1ba31b9a687111bc","canonical_id":"698","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[21,43,65,88,110,132,155,177,199,222,244,266,289,311,333,356],"fisher_boundary_sha256":"c83dfd1738f12889d5c8d8fa14927532b8398e8d2ad099c912c8a526d1c14c23","horizon":1,"input_positions":[22,44,66,89,111,133,156,178,200,223,245,267,290,312,334,357],"input_token_ids_sha256":"b4fa8aaa429dfdeb479beb6db13cf536dad62e221419e87d8cd4cc6e49562547","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[23,45,67,90,112,134,157,179,201,224,246,268,291,313,335,358],"target_token_ids_sha256":"0e14956d01089b21589756a95abe69218dcb1a764bacd1f5efde70f9667949f1"},"formatted_content_sha256":"829bb0068e2180b28903791519d5fd2ef9c4873981556f5c4fb97915943ceec4","generator_receipt_sha256":null,"identity_record_sha256":"92efe3c0141538bc1fe1ae06136691ba5bfdf72bae549e16e258bdad5bc9a4ca","prompt_token_ids_sha256":"e985782bb265579625ef2c2a88762e93185a635ff5c9104060000f4dffeff366","ruler_category":null,"seed":null,"selection_rank":85,"selection_sha256":"383631dc03a19eb73d617831ef3de08685e43b584ca707ed72bff063479a80f3","sequence_length":359,"sequence_token_ids_sha256":"4f8db022db57407107163fbccfb9fa14d20b6c9ebb4167358a92b76de52a8cf6","source_content_sha256":"040369c5072294811f7d2b172d9db244cffb410fb58a0ff0ecfd4ac76e9ecb95","target_token_ids_sha256":"df7cdff2b03251bf8483757a1fe829176f9debe6a93f970241f7d8a5e45d2923","token_span":{"cache_exposed_start":359,"cache_exposed_stop":359,"prefill_start":0,"prefill_stop":310,"scored_start":310,"scored_stop":359},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"27e366c7bd906286182fb5657b877a3d983450e230b7e29de32c33658d233999","anchor_positions":[7,16,25,34,43,52,61,70,79,88,97,106,115,124,133,142],"anchor_positions_sha256":"231f3d432eea6372c0ecf521691b115566b5045b38a4ebc776666c6712167443","canonical_id":"847","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,25,34,43,51,60,69,78,87,95,104,113,122,131,140],"fisher_boundary_sha256":"8329a9007c7d4b358ba485f2d16e86dbb70099e16057eba7d09ee5bccd1208a2","horizon":1,"input_positions":[8,17,26,35,44,52,61,70,79,88,96,105,114,123,132,141],"input_token_ids_sha256":"c5de0c4ff62e358587b0368de1f213abd9ae23772b8a469ddfe3d5703c9d1012","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,27,36,45,53,62,71,80,89,97,106,115,124,133,142],"target_token_ids_sha256":"98c5bf9c6ce5ebcc0886bb8c1a84fb66bffb85cf9331303d2fd8e801f62f2549"},"formatted_content_sha256":"072b0e2d3b510043865ddae77cb50d35becf17f9e8114b0d23ee65976876342f","generator_receipt_sha256":null,"identity_record_sha256":"45013aa8aca9961b02b0a4a7cb8344d4f74b6e33cd7e17bde68f4bcc9830f5e6","prompt_token_ids_sha256":"cbc2d13d695e49c699eabbb0c37174336cdd76b1c47e1f754455737973e861a6","ruler_category":null,"seed":null,"selection_rank":86,"selection_sha256":"3856a5bb20d2c0009500867054a1e894dca05be3fb5db8c3123f8dcc626b3326","sequence_length":143,"sequence_token_ids_sha256":"d3b45de44d96d01b589d50a86f386609c7543385f067532a7174592d3748d795","source_content_sha256":"06648ba18a8fa484b8f21f1e5f0581005f1c25c15cbc72488833b9a12475640e","target_token_ids_sha256":"0fec8926851dfdf2d26e890a92aff900619119eb83bb757d608676dac4ded5d9","token_span":{"cache_exposed_start":143,"cache_exposed_stop":143,"prefill_start":0,"prefill_stop":132,"scored_start":132,"scored_stop":143},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"a71bad1c8e08458e094f4e03eef72e9b22e929c4ebde165642691319a2f08b8b","anchor_positions":[11,23,36,48,60,73,85,98,110,122,135,147,159,172,184,197],"anchor_positions_sha256":"7e13e3a1913a1aa40d4138d69b9cd0e10357d1735e6198f7373eda076b3142df","canonical_id":"627","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,23,35,48,60,72,84,97,109,121,133,146,158,170,182,195],"fisher_boundary_sha256":"fa4d10a3491f2cbd7a778d5d9cca07773dcb82334929e2637a6782e847dbba69","horizon":1,"input_positions":[12,24,36,49,61,73,85,98,110,122,134,147,159,171,183,196],"input_token_ids_sha256":"44dc69fff0abd93231a72100f67a65e70f884191a50e851eec9626b5f6d3ecea","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,25,37,50,62,74,86,99,111,123,135,148,160,172,184,197],"target_token_ids_sha256":"737d45bbae07d3bc02b3df44539a526e40a1a65dfb9bbd68a8552b4e3b8a3396"},"formatted_content_sha256":"0a0b695997f5097537b328213722620e421fc13a6da1c75ddb049d61551d4ed8","generator_receipt_sha256":null,"identity_record_sha256":"a708abf4f23637f8abca094ec7198056051b374088f997be873b42a6e20807ab","prompt_token_ids_sha256":"f19ff189894482dc79997369f05a455e1a3ef3d68bf8187d519557d6d6c6ab5c","ruler_category":null,"seed":null,"selection_rank":87,"selection_sha256":"3878b1b7a110136224b32091f63168693baa7e86e3c51b5a5536e13dad73f9d5","sequence_length":198,"sequence_token_ids_sha256":"0f9fcb1d27f0fb0d37209deaa4fa835902344d189e823d011638a2011ede20bd","source_content_sha256":"12ac418a9b4a0d9ae374ce607cfe72460e1c30e426106ea273a46ac4ba95ccac","target_token_ids_sha256":"cf5bb6259eaf6bc72ef3d80bf3723b99a1c4a093c8a643a5b9a663a259d73de4","token_span":{"cache_exposed_start":198,"cache_exposed_stop":198,"prefill_start":0,"prefill_stop":111,"scored_start":111,"scored_stop":198},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9c95bca00064ff088717e40df85e1ee14f4f14ac42b57c865b2a27f9d95db37b","anchor_positions":[17,36,54,73,91,110,128,147,166,184,203,221,240,258,277,296],"anchor_positions_sha256":"92fd03c25ec622578164ea2f5bba6632c7a3ee726d6e1c9caa74a3fe54646d5d","canonical_id":"936","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,35,54,72,91,109,128,146,164,183,201,220,238,257,275,294],"fisher_boundary_sha256":"e2680422efa4a11ce200109baaec769ad06a6773088de15eee3df9953e7958f3","horizon":1,"input_positions":[18,36,55,73,92,110,129,147,165,184,202,221,239,258,276,295],"input_token_ids_sha256":"06925d4de4cf692d9d7ae2fb8defd1d78ee16de18017fea89fc095caf7b42551","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,37,56,74,93,111,130,148,166,185,203,222,240,259,277,296],"target_token_ids_sha256":"a529ab740b332e7c438cb708428ab5907c7a59afc2ac4ac51145f549dfdc5580"},"formatted_content_sha256":"afa72e90063431bd6756ace794a9050cc1610077201aea2fd9b4bda86c662ffb","generator_receipt_sha256":null,"identity_record_sha256":"92f30344725d406c50725f35a3f5b576e19bbd592073bffc0ff773be3cd5ef1c","prompt_token_ids_sha256":"6a725c770525d60d682f99ffbb259336c6f4946dc97f839cd4ca68ee7d0cf425","ruler_category":null,"seed":null,"selection_rank":88,"selection_sha256":"38fb5a0b645f175f62d3e012cf15aec73e7982d5713c59591a4c1d1a08561d18","sequence_length":297,"sequence_token_ids_sha256":"9f51cb1048ef1942bb0af6b3e3d4087006b31ee6c48ec3a0f524b1e08cc51f15","source_content_sha256":"1effae4b4c21d10e095a0a5f927b7cffd241b64be8df419dc05137e160edabc0","target_token_ids_sha256":"6a6840a3339ec51d008e341f8b430ed9e10a11c573b431942795dcfdb1e62c77","token_span":{"cache_exposed_start":297,"cache_exposed_stop":297,"prefill_start":0,"prefill_stop":255,"scored_start":255,"scored_stop":297},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3b13375846101ec060f2a2ecefd25b66c836c8f9c53225732689232378b5cc44","anchor_positions":[9,20,31,42,53,64,75,86,96,107,118,129,140,151,162,173],"anchor_positions_sha256":"5eb476494a95db7e5ff69d760daae6c1b59a438e7d071342cc2f8648e535c378","canonical_id":"827","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,31,42,52,63,74,85,95,106,117,128,138,149,160,171],"fisher_boundary_sha256":"d2950d5bb22979ced35eeb6c35b7e0f442e13245822cf644ec5b7300565507ee","horizon":1,"input_positions":[10,21,32,43,53,64,75,86,96,107,118,129,139,150,161,172],"input_token_ids_sha256":"995c75651110d6eeaf5f7790a1e5988aa866d3ec3f148b43040167cca7bd163f","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,33,44,54,65,76,87,97,108,119,130,140,151,162,173],"target_token_ids_sha256":"4d18d212a638d29f456dfa535bd673c510d66ec32e43594822d9bb9241ca20e6"},"formatted_content_sha256":"4790f2d60d6a241d89f28294cf7110652356521e1216cc3a9e2ce7e0b065bbe2","generator_receipt_sha256":null,"identity_record_sha256":"e93e8a672f73ac122851223c0114eb0974c76a960f1a880f4d4f64c9eb90f40c","prompt_token_ids_sha256":"9cc3fc3392444e349135b1065a5227e3391859f3554be674981d424cbdfa9fbe","ruler_category":null,"seed":null,"selection_rank":89,"selection_sha256":"39165caf08bb54ef81c9f6ea0a6ae1eac6a1c3f0b6f9748dcd2e2d953b5a44c6","sequence_length":174,"sequence_token_ids_sha256":"f5f5521cd3bd481ca316528bee26f7f9355182d73550ff14da89d91f6558600e","source_content_sha256":"c60d7f94f875619bd70591ab5f80b6e8fb8d336938e07fbeed1b32564c1a0b38","target_token_ids_sha256":"fac10609f7b6c41ca9b4bbbf7495988b94cc3b4df19e5dac66d84079bd3d4b1c","token_span":{"cache_exposed_start":174,"cache_exposed_stop":174,"prefill_start":0,"prefill_stop":148,"scored_start":148,"scored_stop":174},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"ad1b36a9a3ef4aa1d78cc9a2babd28dfdd79a75b4a7e69563454a05c10b6e2bf","anchor_positions":[7,16,24,33,42,50,59,68,76,85,93,102,111,119,128,137],"anchor_positions_sha256":"451653062c723e6fe2316470c66684b5f06f6d96d14e7b3bb2fb0c9e7d88845d","canonical_id":"930","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,24,33,41,50,58,67,75,84,92,101,109,118,126,135],"fisher_boundary_sha256":"a46d5960cd9fe3147cb09c59211821b555240915a05f4d76e9a91b70e253bf77","horizon":1,"input_positions":[8,17,25,34,42,51,59,68,76,85,93,102,110,119,127,136],"input_token_ids_sha256":"e59435b4be9b10629e72f1905eae42cfc9f3fd8b77cbff58f09164e96872cde6","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,26,35,43,52,60,69,77,86,94,103,111,120,128,137],"target_token_ids_sha256":"e841c04edad63a6a7f6c695115859daa7a4c3eeb9fe4f2f7094561aff448ee28"},"formatted_content_sha256":"dcb79d95145dcc945bd4f4ac20fe07c33528e7e8d8ef4cfea0312e5b02e817b5","generator_receipt_sha256":null,"identity_record_sha256":"5b49551d1068cd3633b13af41d2b4e4ab38c2383efee9d8c5370832e1a78e65e","prompt_token_ids_sha256":"2eeaa907d87be72e61b726a9ed4672463343033ccb6bf2376754766f298a566d","ruler_category":null,"seed":null,"selection_rank":90,"selection_sha256":"3934218a3cdeb9fe23fe0820bbaf2003e6426a07af797c8318d3df525a7b9d57","sequence_length":138,"sequence_token_ids_sha256":"5c11b18fe521da6db909428adfb48961064e310ff957a9d2a1ade8b72d391175","source_content_sha256":"435474a45f03a3373a0a7f5f78526f3621d01a4daf6907075965fc04440e323f","target_token_ids_sha256":"3fa7542edfba5ed95e02424207a7fc3d1bdc29ef1bb3fb055507740f8a0860d6","token_span":{"cache_exposed_start":138,"cache_exposed_stop":138,"prefill_start":0,"prefill_stop":90,"scored_start":90,"scored_stop":138},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"926ef081a870c670e181e6f21a6dbb6e67418bbe21df0f8841a9da25ad315d93","anchor_positions":[14,29,44,59,74,90,105,120,135,150,166,181,196,211,226,242],"anchor_positions_sha256":"3859b527bfcde30950aa1fd4919bae6af3d771f9dcd051365cd8ca16d468c677","canonical_id":"751","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[14,29,44,59,74,89,104,119,134,149,164,179,194,209,224,240],"fisher_boundary_sha256":"af0bf62935327f2467d479d54e944e5f444b6c444118efcf42d997ed4da7cc37","horizon":1,"input_positions":[15,30,45,60,75,90,105,120,135,150,165,180,195,210,225,241],"input_token_ids_sha256":"f52f3b35b365e905e8af0a0912e656ae9d1e3ad2906698dfa15eff817cdc47e1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[16,31,46,61,76,91,106,121,136,151,166,181,196,211,226,242],"target_token_ids_sha256":"c71c870c1fd6b3ebc3243a03cd21567b7db14cc45512773ebc666af0317504a3"},"formatted_content_sha256":"b9749edd133bc0cefb8007a7358f767d918467015bff6d845fa78b29bdd5b3c2","generator_receipt_sha256":null,"identity_record_sha256":"bb8bf4056e66c85336deaf5c6bf0614c6c6eea646b46a3dcc0b05e265ed2293b","prompt_token_ids_sha256":"4d217c659514cbec1da37d7ca1cf02b1de4d40d095c106f7fd89d4517b8e1800","ruler_category":null,"seed":null,"selection_rank":91,"selection_sha256":"3972ee7b79e05ce309d4c65459acc2d264f7821184dc4baf556f892f50507741","sequence_length":243,"sequence_token_ids_sha256":"22a137779b1349921b5784c3f3a7da9f3700715398d21e5c69536216ef7c5d82","source_content_sha256":"04de6155b488684214357c82595e2a1b0b4162588221bfc1f959c3ad02c64302","target_token_ids_sha256":"4a34e467583136f16ea6deeb64a30a0f625052cbead11e0c2460872fb5a60d9d","token_span":{"cache_exposed_start":243,"cache_exposed_stop":243,"prefill_start":0,"prefill_stop":129,"scored_start":129,"scored_stop":243},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"bc742a938d14512badf1f6f2dc626345160bae405ccb62cc78f8fcf2fbef198b","anchor_positions":[11,24,36,49,61,74,86,99,111,124,136,149,161,174,186,199],"anchor_positions_sha256":"cbd5b119373faf027bdb11d72340904f9e8568f13150612cfe8c0d28d26f0a32","canonical_id":"707","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,23,36,48,60,73,85,98,110,122,135,147,159,172,184,197],"fisher_boundary_sha256":"f539aaf9624257146284435f180cb1ef2b1c3d2448b93f936510956f316e60d0","horizon":1,"input_positions":[12,24,37,49,61,74,86,99,111,123,136,148,160,173,185,198],"input_token_ids_sha256":"53699344c41ad6728511aaab294416c9cb8ab020e2cce05973c6abf72fe3830a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,25,38,50,62,75,87,100,112,124,137,149,161,174,186,199],"target_token_ids_sha256":"c13ae7cca2c350903d6e54179df99c8f0ca7fdd40021679bdcd12e59ab7d3f00"},"formatted_content_sha256":"b7943712ec68b060c30f63f9c1e79415e5d98891bad66512dd84aae0ea33779a","generator_receipt_sha256":null,"identity_record_sha256":"7e8098446b3e22dff19a34b8e6cb18c072889a0914269f5e6c46eda3e278f916","prompt_token_ids_sha256":"aedaaa76edf70a4335d76e3753a2e74b6358828bd7fd27089a9349b82dbc4f04","ruler_category":null,"seed":null,"selection_rank":92,"selection_sha256":"39d4884c3293b83c7da73a4192c68ac6ed7dd2a2b8cb6535c8bc27dabefb2152","sequence_length":200,"sequence_token_ids_sha256":"f84c8958f4ae3235716092ff6d50ea9f9eeeed151677fd0291eefd3f4c046e46","source_content_sha256":"835bcc97198da2e9313e8115ac8c70845ef11e086ca2f36a6cbb1bc74afb15e6","target_token_ids_sha256":"63a1e5e3435f4582095fe5bf45dca9f77c0635e287377e887030050e9576dfbe","token_span":{"cache_exposed_start":200,"cache_exposed_stop":200,"prefill_start":0,"prefill_stop":82,"scored_start":82,"scored_stop":200},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b368a5e3bf6a4d6a9d65ed071a6ae293e6addf944c619d41b4db74fd479ea1c6","anchor_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,208],"anchor_positions_sha256":"e3c9aaa391385675ab3475417e405da92d531052d937e046ddc609185dc2c7d4","canonical_id":"946","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,50,63,76,89,102,115,128,141,154,167,180,193,206],"fisher_boundary_sha256":"6717c7a65af61cfd12e44550f289c401161af90d68d57dd17e3c519d22a1613c","horizon":1,"input_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,207],"input_token_ids_sha256":"35c588f0b227da0babb6432ce2d27c1f6f0ddf842b8da0c7b8c5def4f5567635","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,52,65,78,91,104,117,130,143,156,169,182,195,208],"target_token_ids_sha256":"3cc14e363f767b18bf8fb37c1c7a1681237d5f34c0d8244fad4f8ea124be7de9"},"formatted_content_sha256":"6672cb487be13a90610c98af3babace38c6413c484e2a543e785501685eea0d6","generator_receipt_sha256":null,"identity_record_sha256":"f4090de3fc3464141d40f4b023f18db5c2e540193c894d2b56d314001a2cc540","prompt_token_ids_sha256":"23c086ef1490fd86a6b3f23899489b782e5aef4d38689af88dbd19f7fb7b1d0a","ruler_category":null,"seed":null,"selection_rank":93,"selection_sha256":"3a31bfe17e95e3f782d9b40a9bad5670a4dbab98abaa87ce2414402b4d4c9d3e","sequence_length":209,"sequence_token_ids_sha256":"8002749920d9aa312887fb27e1c8d185977ea0ce5f145e2a0dc684e46b759834","source_content_sha256":"bd64b5ce22599f2fceb61fda2a89a9caed49e3e0b63694d888f83b548c800738","target_token_ids_sha256":"d1d65dd1bea87ea89e2cb46bc0c015ee3a4e8df03fe4a02a29bb30c5549b96d0","token_span":{"cache_exposed_start":209,"cache_exposed_stop":209,"prefill_start":0,"prefill_stop":178,"scored_start":178,"scored_stop":209},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"122dd2605483f8c364752af43aece851c5117635a2c390bbecc2d2c38ee203cc","anchor_positions":[12,26,40,53,67,81,94,108,122,135,149,163,176,190,204,218],"anchor_positions_sha256":"34e0f2438d49fed8c871a33d3818a40fc035b239ea8639ec66d64d1e59a9bb66","canonical_id":"853","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[12,26,39,53,66,80,93,107,121,134,148,161,175,188,202,216],"fisher_boundary_sha256":"41be834ada2a5acf9617b5b43955804d2e6a4ebe3234bb01e8e5dad38bb0a1a3","horizon":1,"input_positions":[13,27,40,54,67,81,94,108,122,135,149,162,176,189,203,217],"input_token_ids_sha256":"afd2edb984710f6fc20bd321b1036090c05a37d38b4b24a9acf50e7bd4160e2a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[14,28,41,55,68,82,95,109,123,136,150,163,177,190,204,218],"target_token_ids_sha256":"b5093288c3e5810d5c3d19010a1acdd8082bcba55f9489bdabb69e3196c53db6"},"formatted_content_sha256":"abf4e533a3c9388da04e8ab3ba22c031aaaeb56e90338a4793f35ac968218c95","generator_receipt_sha256":null,"identity_record_sha256":"e80d081ddd92bc2a2b3317fc55c504dc53c4973ed5fdd77bb04345fc653a731e","prompt_token_ids_sha256":"54d2e4ab6ff2869f39d935df89b9c87e44df50e0fe4ecf67a377211040ceb3dd","ruler_category":null,"seed":null,"selection_rank":94,"selection_sha256":"3a95db7ccef488fa57c922c5fa561a269f79fc7c743e4d7a0ba136629ed14154","sequence_length":219,"sequence_token_ids_sha256":"7c11f008ccc4d7cd53ee721463f7a84f45653c8215e85fce7deab133839b9b0e","source_content_sha256":"6e23a95d9b3ebcb264ca654a288e4f63ec19e20f70c8c2cde95ec7ec6f5b65bb","target_token_ids_sha256":"5da4da15ffc85692e7f49aaf3e35fa21d41d697277e633358e0285f47b164471","token_span":{"cache_exposed_start":219,"cache_exposed_stop":219,"prefill_start":0,"prefill_stop":83,"scored_start":83,"scored_stop":219},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4475867faff977bd5c0d96f97b251d767fa8320ea73f71a38a236d7f33bd9db2","anchor_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"anchor_positions_sha256":"debc2f860d51bf30c9ccfe7850d3428800e80a005f976ab2943a8642c33fb808","canonical_id":"824","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,30,38,46,54,62,70,78,86,94,102,110,118,126],"fisher_boundary_sha256":"652add26cddc5b2b11115550fc5858cab1d3a2bf00d5a3de3dd88ed6f9ee6844","horizon":1,"input_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,127],"input_token_ids_sha256":"f5e28e53573eb8607c244c66e29a9057150911f5b5348a120bb3a756c686e14e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128],"target_token_ids_sha256":"f2aa9adbe82c7f97119740cd60b33102619d0355f0db44358231d88ca624f304"},"formatted_content_sha256":"37af9d0afbb100c1787445b2ccb87b0a15ea15c7b2fac0560e9fc296bf6fe512","generator_receipt_sha256":null,"identity_record_sha256":"3933b2a829550c994be7ec3c13432e7fcc7eaa7429de8d86f09f2fdd0d54c382","prompt_token_ids_sha256":"e63b26e32e2d08ba0e1878bb31f438a8b4f0754fac3b288bf8ff4bc11ebc726b","ruler_category":null,"seed":null,"selection_rank":95,"selection_sha256":"3ae7adc084daf636812058b087ea0414390f1d765f67235a978166a970420f8e","sequence_length":129,"sequence_token_ids_sha256":"325eb23014b521a2ed7488ed8be2e2022f039e91aa7939745a5ea7decd8357c3","source_content_sha256":"f3f9375971e245fb7d4b9ecb4368b1469f96e8826d3d290349822de215c9524f","target_token_ids_sha256":"98ee353303b490751f3b0bf4a9db8ee9f144fd843e84aee5b61e6b8b237bdca6","token_span":{"cache_exposed_start":129,"cache_exposed_stop":129,"prefill_start":0,"prefill_stop":96,"scored_start":96,"scored_stop":129},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"95c07a4ae2c995c71644eb87f395b0201ab61998c92c88acd0deec2bdf52eb7f","anchor_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,208],"anchor_positions_sha256":"e3c9aaa391385675ab3475417e405da92d531052d937e046ddc609185dc2c7d4","canonical_id":"755","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,50,63,76,89,102,115,128,141,154,167,180,193,206],"fisher_boundary_sha256":"630fdfb50def6a0e9810e25416f083ca388d391614da06d1ea421eee032302e5","horizon":1,"input_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,207],"input_token_ids_sha256":"211b86c0afe55079edbfe3277616d686ed0a24152163e4ad422e5c2071214472","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,52,65,78,91,104,117,130,143,156,169,182,195,208],"target_token_ids_sha256":"279608d1d01474978b20236572dd00bca3894556f73197ecfd6a080fdab2cea0"},"formatted_content_sha256":"d7d00e78af67fa7b8d4f1123bc771c6c1838a42ca7940f09be045fa8e0258d3f","generator_receipt_sha256":null,"identity_record_sha256":"163bb04292002d23bbb74050efcec0fe6cb3a616a2ed635b0e83b30ad3599dbc","prompt_token_ids_sha256":"649e3b31340e844cf9aca294f09bbfb6ea6f59b8fa4339754a8040e2939a14ab","ruler_category":null,"seed":null,"selection_rank":96,"selection_sha256":"3b77b38c7af3c05a391dfb6e4427c1d6c8e1408b314a52ec893944d4dba7a59d","sequence_length":209,"sequence_token_ids_sha256":"5eadaf66f4c2a5b1419e30b0e9445bdcf5e76923985d3015fb3da1ae81a6112f","source_content_sha256":"f132344866b31da506e676ecd7f787c4f0b9c30ce84dbcf234d1bec372a4a3bf","target_token_ids_sha256":"b0d0429e68642809416cf5dc819cdacda9612ad5ec72ede704a6165361970f31","token_span":{"cache_exposed_start":209,"cache_exposed_stop":209,"prefill_start":0,"prefill_stop":107,"scored_start":107,"scored_stop":209},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e588a1ee6a1182055ad85a99122b37dca3e494260dd21a56a3847938dd6650d8","anchor_positions":[15,31,47,63,79,95,111,128,144,160,176,192,208,224,240,257],"anchor_positions_sha256":"58b00b7c1a9bcbca3b1a752019d06b8217b207235aef4a33f8862dbb76f94f4d","canonical_id":"705","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[15,31,47,63,79,95,111,127,143,159,175,191,207,223,239,255],"fisher_boundary_sha256":"7ddf7690ae9f4e218fef449a93f7970eefb48a98743e2cdab0de23b0a0dec118","horizon":1,"input_positions":[16,32,48,64,80,96,112,128,144,160,176,192,208,224,240,256],"input_token_ids_sha256":"fce5f6a7ef57e2e29ac18c5c07d311d066b45dcd8cc0d5feede90040805886a4","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[17,33,49,65,81,97,113,129,145,161,177,193,209,225,241,257],"target_token_ids_sha256":"7e886f55d5a1f767632ff6ce8330179e78261592f007a8d785bcacf263c6b5b2"},"formatted_content_sha256":"79b5d499f16fb4783be74bd200b7cdbb4738662932b42d91a8cb390646bd7282","generator_receipt_sha256":null,"identity_record_sha256":"fabe611b8a7c9e97b13d178a8020c782eeb92caeab58708b2acab9e36c7c9739","prompt_token_ids_sha256":"0906a739ae0536ce684360464813702b15ce64ce3043e3fad9895d8c7a910e81","ruler_category":null,"seed":null,"selection_rank":97,"selection_sha256":"3b7d0fef73308384ee4b6d98b48fefed969da32df7231a61067116bb7b9b77e5","sequence_length":258,"sequence_token_ids_sha256":"c89f8002d735b00177a7842bc2ce8332555e0cb94e7fc1f651c7d9306085ae48","source_content_sha256":"249e819abcbbc94c533d4966c357933213dbe7b58282534d182cfdd719ab0b02","target_token_ids_sha256":"f81a0d07e6c10d8a36f2da0ff7490f0c0aadc3fe1411b385e79f96372bff8f0b","token_span":{"cache_exposed_start":258,"cache_exposed_stop":258,"prefill_start":0,"prefill_stop":231,"scored_start":231,"scored_stop":258},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"21ba94b504ab5ded4ffa601325cf7df425e7f81789df808b86d2e08ba2117dcb","anchor_positions":[9,19,29,40,50,60,71,81,91,102,112,122,133,143,153,164],"anchor_positions_sha256":"e70afdac88bd21ff5ffd057462f0868a1aaa408a52296b58ecd4900f7f13759c","canonical_id":"614","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,39,49,60,70,80,90,100,111,121,131,141,151,162],"fisher_boundary_sha256":"b4910baa9b90efc3b7ffe399e1e19b050f46dd444b4bfa0faaadd835b9a3173c","horizon":1,"input_positions":[10,20,30,40,50,61,71,81,91,101,112,122,132,142,152,163],"input_token_ids_sha256":"09f0051a5092a284f02162790ff187253c2a8be10b80c2ff7b64476a0924f7b8","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,41,51,62,72,82,92,102,113,123,133,143,153,164],"target_token_ids_sha256":"9c3d0858143570f2b17fd9c211bbddf1beedb863bd39887f1982c64284451830"},"formatted_content_sha256":"b0e77c735851949333aa5a07a80654370a25c6f4c29b682d0a7296c28db21615","generator_receipt_sha256":null,"identity_record_sha256":"9e60edb916d1cba4b8692f32df62fc1c57894b2aa815769ff9a17da063de91f8","prompt_token_ids_sha256":"591d12041b2684415e1d05a0ea815d006597cc26105e19c6a0769dc0ea26a8bc","ruler_category":null,"seed":null,"selection_rank":98,"selection_sha256":"3c33175d44e2cc571159f6914ce6a7c57a1949ccdaa58585114ece582b46e4a7","sequence_length":165,"sequence_token_ids_sha256":"964232f668fe81da4ef5e6945e2164c11b771718b74fc029dbc71ae25a213e61","source_content_sha256":"10dc2964ee279825eee885f98c2883dcf718ce9958fca29aabfd2057996b5a19","target_token_ids_sha256":"fc3e58f7036836c2a7f90f009d9d954dbc547bc6a412334c087a495ec6166136","token_span":{"cache_exposed_start":165,"cache_exposed_stop":165,"prefill_start":0,"prefill_stop":140,"scored_start":140,"scored_stop":165},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"41e7124eadf2c8d65f12d91df35d39b81b0af62f8f0339d10d48d23001bed1f7","anchor_positions":[6,13,20,27,34,41,48,55,62,69,76,83,90,97,104,112],"anchor_positions_sha256":"311e7702507c904e70997e0b842193b8c519ddd45b6861c3e52fdd12fcbe662f","canonical_id":"835","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,19,26,33,40,47,54,61,68,75,82,89,96,103,110],"fisher_boundary_sha256":"ef27b3360779ff829870ac1205d40a423bbd1857f9d29564cf1a4bf273508b37","horizon":1,"input_positions":[6,13,20,27,34,41,48,55,62,69,76,83,90,97,104,111],"input_token_ids_sha256":"622f2d82715555a3a862da50633d47a53da0f6d05afb23f037425ec2460ebf97","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112],"target_token_ids_sha256":"e4f44986dfe9f6afb783e79bc0788f586424b03e794cc20ab1129e7a75ca4b91"},"formatted_content_sha256":"16149587d9ee5b76bb9036394de7cc48051ba6deeced83ecf9552046bdf3d6fd","generator_receipt_sha256":null,"identity_record_sha256":"bcd976206ac1782d1fd75247a66068233d32764eef93e916f4303ac7216894e6","prompt_token_ids_sha256":"defa46cd3ab312e32247db6a9f87d5520f8ebbaa993a9af787a0d44ecc31a91b","ruler_category":null,"seed":null,"selection_rank":99,"selection_sha256":"3e545f645fe778609ea47f9d9d70db3a2d6fbf7c1b704f5c4b892a732ca4870c","sequence_length":113,"sequence_token_ids_sha256":"d87292b8689b41a713b9f23d42d1936617675fed9a389358046c37dee67270a9","source_content_sha256":"a2d12e2a125b7130ed036625cab03ca87319d5c2bc2f78d56e5b06e5d48967bb","target_token_ids_sha256":"d18a4e331e8bc3c665aeec8b4e8173eeaf216f691cec5deedcce9b89d1eabcf1","token_span":{"cache_exposed_start":113,"cache_exposed_stop":113,"prefill_start":0,"prefill_stop":85,"scored_start":85,"scored_stop":113},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4428369199e6df8c36558c477449442560d7dbefe1b1a34029eb0d3a540e8850","anchor_positions":[8,18,27,37,46,56,65,75,84,94,103,113,122,132,141,151],"anchor_positions_sha256":"99b598ab1b26514b0793ce6e9d9b8fd5fa21cfd91c15cf4ba944b02280eccc00","canonical_id":"785","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,27,36,45,55,64,74,83,92,102,111,120,130,139,149],"fisher_boundary_sha256":"224129c3bde0a7dc09aba43d89bfcf9df160c62a8d883b2421405de57da83148","horizon":1,"input_positions":[9,18,28,37,46,56,65,75,84,93,103,112,121,131,140,150],"input_token_ids_sha256":"d1cac9f469f540d35dd51eee4d84dc6489bdfbcb831d778ca3b910593caad5ba","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,29,38,47,57,66,76,85,94,104,113,122,132,141,151],"target_token_ids_sha256":"293d996a68091856a066009f8299d24def6c9ecd1505302fd1289596b4ce5929"},"formatted_content_sha256":"55cc81d66bd2973ef60996d3a13b658ddb8e10ba036a3a91808a8c4ae225b19c","generator_receipt_sha256":null,"identity_record_sha256":"70a4011b9bb314496b859d408f1205aaa618d08b8ca5504e4b0b7a8b3a794737","prompt_token_ids_sha256":"b5c378f6ed8ac42eb8be052f6d123a663ddfcd36c4bbcfc5decae012cf1c1926","ruler_category":null,"seed":null,"selection_rank":100,"selection_sha256":"3e7cbaec0d7be5d2de99697b23da0c61163c5bc4acecf283c4d9167e7b780eae","sequence_length":152,"sequence_token_ids_sha256":"ea3cdc2b1f803377a8cdf0588f013cebc2f019e5ad60fdf01294bf2d04d429ca","source_content_sha256":"06e8fb9b807385f5950f7335cb79a3b8105f621f1f5ac389155c9e737ff13d4a","target_token_ids_sha256":"a21eb422039b5c07e6a0e5f71858bbc8eaa8c315d06b39437977061a7cfdd253","token_span":{"cache_exposed_start":152,"cache_exposed_stop":152,"prefill_start":0,"prefill_stop":109,"scored_start":109,"scored_stop":152},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1cad8beb17ffee15f4de17ab4607544233d3683ae14b0cba6d063ccc9f11ff77","anchor_positions":[20,42,64,86,108,130,152,174,195,217,239,261,283,305,327,349],"anchor_positions_sha256":"3e58027edb04b4c749c71fe1c1753ad8300a599facd28b3d327dffaa160dd92e","canonical_id":"662","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[20,42,64,86,107,129,151,173,194,216,238,260,281,303,325,347],"fisher_boundary_sha256":"a261daa2db8ba7f05e7ff89074fdc6a0533b335ee67208179b8229ca3e70980e","horizon":1,"input_positions":[21,43,65,87,108,130,152,174,195,217,239,261,282,304,326,348],"input_token_ids_sha256":"d7e08d7e4f81bf68a6cf07dae075daaf7f79121f48d39ad9dba80fb057031e6d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[22,44,66,88,109,131,153,175,196,218,240,262,283,305,327,349],"target_token_ids_sha256":"8501644795ff2f929a4924fb2a3a1644ea894aa81320b17182c421175240b4b9"},"formatted_content_sha256":"91184e28eff6c3890010723ccfc61b5b4736a602d677efdffe59a220b3bac9b5","generator_receipt_sha256":null,"identity_record_sha256":"45b6269ce5d7d2ccefccb9d3aff10c4cc786d4482f036ebc1e0837d56e6c1923","prompt_token_ids_sha256":"ee07e3808c4e1142a8b3f904c8582078b899979e541c1ebe2297a3670112077c","ruler_category":null,"seed":null,"selection_rank":101,"selection_sha256":"41900ba952796bb2c024b96db8f08d2e63a757587e61fd96909dc934d50da156","sequence_length":350,"sequence_token_ids_sha256":"6890b69767a6c590160f23954c0338831fa0e3912f06dd6514c5b1552317c006","source_content_sha256":"3cf31926336b0e44d3630a63a486831d0b79632309d78e0b65f28e1c6f37353c","target_token_ids_sha256":"38e5b68f608a60d1fc61bff0194cd835196d4f8e8acb129c9e0744850d8641bc","token_span":{"cache_exposed_start":350,"cache_exposed_stop":350,"prefill_start":0,"prefill_stop":319,"scored_start":319,"scored_stop":350},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7d64f9e8b46b0ba25c21f37dee8e110df8862303d52cb3c2697353097a97db76","anchor_positions":[7,15,24,32,40,49,57,66,74,82,91,99,107,116,124,133],"anchor_positions_sha256":"6c1029515cd9fd840ee9baeeb3bbf16de22d914596dc66d6c0b00a017e82aef5","canonical_id":"732","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,32,40,48,56,65,73,81,89,98,106,114,122,131],"fisher_boundary_sha256":"2ea117a68a1f560fe1206940c2b26cdf6971aeeecd14dde580d013efe50b498e","horizon":1,"input_positions":[8,16,24,33,41,49,57,66,74,82,90,99,107,115,123,132],"input_token_ids_sha256":"46da8cb7bee3ade0de904115c405d0a4e2c1e215d9814066bfc5c50885b092d7","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,34,42,50,58,67,75,83,91,100,108,116,124,133],"target_token_ids_sha256":"3dd8aae9f6aff8d199075c33306d8bb5bd3b5f73666f2d236bd066a6d55fcbe4"},"formatted_content_sha256":"cf5f521e053dcaa96c55370f299514c9de27ccc98b8d846b0a51c4d2450341e3","generator_receipt_sha256":null,"identity_record_sha256":"1ea4e6a64df86bf2e3771dfc41f1716146cfac365a658d3864c030013a4cd84b","prompt_token_ids_sha256":"d705f66b234374474a57207de30809b5832cf7196bccb9dd73addbb879937264","ruler_category":null,"seed":null,"selection_rank":102,"selection_sha256":"41c7c18c717892ef0cbe99444d2f1d3d16142bd3128dc408df64987e27a8531f","sequence_length":134,"sequence_token_ids_sha256":"c98483cae5d714b2a8de61cd02257cc0ab084c30a0673971a4460e6d07f59d2c","source_content_sha256":"e5aeda9a803b146c76d557a6c8dc84720e938726b6f35982a5e61465a9a9d11c","target_token_ids_sha256":"15d06316804c670aad3848c3d01e3abb336656533c4feb76f93e90a0399aa760","token_span":{"cache_exposed_start":134,"cache_exposed_stop":134,"prefill_start":0,"prefill_stop":111,"scored_start":111,"scored_stop":134},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"a1bc1d7610d52bcb926390555c1cab77e3b88be528f3e256ea9e9a8106902d2a","anchor_positions":[7,16,24,33,41,50,58,67,75,84,92,101,109,118,126,135],"anchor_positions_sha256":"9058652c46869f6bf5cde3a730d23fba0f016461c119f6bbc8af3fb1c3031545","canonical_id":"679","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,24,32,40,49,57,66,74,82,91,99,107,116,124,133],"fisher_boundary_sha256":"810b5e7562357db03cc4e0193687f43bf7f4a08839cf3425a51f4e629ca7f5f6","horizon":1,"input_positions":[8,16,25,33,41,50,58,67,75,83,92,100,108,117,125,134],"input_token_ids_sha256":"03872be7640939a2770876b1cc144c05f59d3e06a6418658db2bceeecdbb5f11","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,26,34,42,51,59,68,76,84,93,101,109,118,126,135],"target_token_ids_sha256":"5c71155464943a602fc92f92c7507287819905cfa27f665ae40e126b898cdc40"},"formatted_content_sha256":"ffbf833b467a2b56b26bc50bcb320d893f41f60eed6341ca4c61813965020c7c","generator_receipt_sha256":null,"identity_record_sha256":"13c42ddbb25f335f5eaa4ef5df7c2cecd1ea19af1f8972ade37312c4fce78802","prompt_token_ids_sha256":"df73b549576d1784c30bb45b9c873ebb80267ac6c56afb4335eadca3c7c56849","ruler_category":null,"seed":null,"selection_rank":103,"selection_sha256":"41da3df61a258a1897eeac98083550d328825fdeec35836584dc0b7a76b060b4","sequence_length":136,"sequence_token_ids_sha256":"fb871500f94a5409b4acab60ab3e2aa6955924b9ceb87214232c4099d302dab0","source_content_sha256":"39571a4165c49d0610e00d19d3f5b5e1879232f2d0429100e2aaf915eaca9ab9","target_token_ids_sha256":"d6eaabcd23781ac4a910aadc33823fd08ab066d01331a124767f6e84f181ffba","token_span":{"cache_exposed_start":136,"cache_exposed_stop":136,"prefill_start":0,"prefill_stop":118,"scored_start":118,"scored_stop":136},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b8947897cdec3fc049ba3461c75adcd685b2264e6810778decb68c1db034f7b8","anchor_positions":[17,35,53,72,90,108,127,145,163,182,200,218,237,255,273,292],"anchor_positions_sha256":"c72256a78f816fa7bf8e540cc3aaf3fc1795369a3859efe48a9c4a8cce0d2489","canonical_id":"895","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,35,53,71,89,108,126,144,162,180,199,217,235,253,271,290],"fisher_boundary_sha256":"d12370caffbc875b01d6a59c8cf4c99d8fb4c93c519041b5500142fdac3a8340","horizon":1,"input_positions":[18,36,54,72,90,109,127,145,163,181,200,218,236,254,272,291],"input_token_ids_sha256":"a316e2b2888cea8097ab8420cf22051694e1d976c3b5b32f83f5207f4007d89f","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,37,55,73,91,110,128,146,164,182,201,219,237,255,273,292],"target_token_ids_sha256":"ea285c83c7e7736e7a436adedf9c66203844bbe2521ff7ad148dd51b419e98f5"},"formatted_content_sha256":"9b9360fb9e789a22496cf1d9c91e89cf7895942447ca7e16f15acbf2fe6515b8","generator_receipt_sha256":null,"identity_record_sha256":"080db8e8894bfd652b320dbf19c1d4e53c8b6ed2626bcb8f8d3cdf87268005f8","prompt_token_ids_sha256":"3ed029adce95edd3406c2359cabf621905d3ecaceb70e519fb1cc89a66899110","ruler_category":null,"seed":null,"selection_rank":104,"selection_sha256":"43113137520f4bdb68e26d41a096fa093a2f995386d5473618f0bbe4b2a0f73e","sequence_length":293,"sequence_token_ids_sha256":"24d56709934330dbe30c61a49434a07def76fdd2b31e47f84f71b050fc7aff39","source_content_sha256":"d1edefa8e37752a9da687c2a2e51086bdfe6401714de0bd593a892ecd4fe7bcb","target_token_ids_sha256":"e894e28b0e66238fd8988d8b0b978628f3354d87898164223cb7c7dc2ce5a596","token_span":{"cache_exposed_start":293,"cache_exposed_stop":293,"prefill_start":0,"prefill_stop":161,"scored_start":161,"scored_stop":293},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7d9d79586d3fd7a1376390bd1f085ee9ec35cb401fa471d99ddf45417339ce49","anchor_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,160],"anchor_positions_sha256":"cea919870992245499fb8d8a858d848e1e76f02a4e2230ada935442533c03198","canonical_id":"727","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,18,28,38,48,58,68,78,88,98,108,118,128,138,148,158],"fisher_boundary_sha256":"768624e8d0cb29118f002fce04ec89945eebe14e7587f370283fa04b2d90cf95","horizon":1,"input_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,159],"input_token_ids_sha256":"93e60eaabf27cffd7d1fee3301e42b3f62e183c9e40317db494ede3450b1ea7c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,160],"target_token_ids_sha256":"507cebbcf35bc57eb21aeff74f69ab933d22c31de894e77b573193730da7662a"},"formatted_content_sha256":"934eb88ee283eea8ac09e46d9333c08ae4e41a6910aa75e0638bc73403184864","generator_receipt_sha256":null,"identity_record_sha256":"aac6297b0c851ed51271119b880a93cb272776311e2b9e97f498c76785255100","prompt_token_ids_sha256":"ad50cbf08781e9a1429766f1a21e3ed4a76d9fc30e43b5b8d968dcda018da4d4","ruler_category":null,"seed":null,"selection_rank":105,"selection_sha256":"4338156536b7925e6044f19142d418110ae27757c47306d15815155b2d682917","sequence_length":161,"sequence_token_ids_sha256":"422567bd8e19fbdf69ab47274fe1bda49c8ac4eae311d7232ee13e0888b0fa23","source_content_sha256":"400a362fb6664f9084e588aa605b6e7ad1505849ab7c769574e0996829f6827f","target_token_ids_sha256":"c8e9d40bce2584a16cdb8ee232a9e1f86dd9c58c6fee7999125c37073c1f5bb5","token_span":{"cache_exposed_start":161,"cache_exposed_stop":161,"prefill_start":0,"prefill_stop":134,"scored_start":134,"scored_stop":161},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b50fd03bddce1923b315763ae4ef228d2b6c9cce2b80954554946c98c92a3760","anchor_positions":[8,17,27,36,45,55,64,74,83,92,102,111,120,130,139,149],"anchor_positions_sha256":"27ff593f69b04d82cc98f1ee839db50560b1d26eb9ecd645cc99a4e74dade2d0","canonical_id":"629","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,26,36,45,54,63,73,82,91,100,110,119,128,137,147],"fisher_boundary_sha256":"d8f311b4a33da5b28927ed11e2b7efbb2a1812456e5cd54c884287cfc1ce634d","horizon":1,"input_positions":[9,18,27,37,46,55,64,74,83,92,101,111,120,129,138,148],"input_token_ids_sha256":"f56a22b67052689f706d00c90e6fc5541583c78a589b0b3a26c6b93eda4162d8","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,28,38,47,56,65,75,84,93,102,112,121,130,139,149],"target_token_ids_sha256":"93c8a6c8bdc5c1ed56f7db15ae4424e87803630ba80e3845d421f97169235f34"},"formatted_content_sha256":"9c29f28fcb8fb63c6faa67311c4d67889b21525c18b625ca3fbd69bb6611342e","generator_receipt_sha256":null,"identity_record_sha256":"dd6a317d0ffc82ff3c3b71c470d6f7f4ee8c3276d55ea892d77526d03f80719a","prompt_token_ids_sha256":"ae80eeafd124153f30025382491abc9552e2fb6ac10aa0e25acea39bb6cab5a7","ruler_category":null,"seed":null,"selection_rank":106,"selection_sha256":"4357903439ea78adb2c1be88172806c1a5e5d57f1bfa26229be4d3d56bf15b5f","sequence_length":150,"sequence_token_ids_sha256":"58d577039503bfee517e7ec9f595d3378f0520d7b10780457232e42ac1f8b9e0","source_content_sha256":"d75386bde56ac72aa78e5b13019f0613da914d25303ec7ae96f618caf713bf8d","target_token_ids_sha256":"25dde1ab97315ca73c3e47f45297c03f015715e1dbf87dc40a90d1fff93faa1d","token_span":{"cache_exposed_start":150,"cache_exposed_stop":150,"prefill_start":0,"prefill_stop":109,"scored_start":109,"scored_stop":150},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"743e09d0b63e0d1d69cb1f4d92a19f1b612ce8d5ed318965c7cbc5ef10da4ad0","anchor_positions":[5,11,17,23,29,35,41,48,54,60,66,72,78,84,90,97],"anchor_positions_sha256":"3a7e2557ca5cb4f3e315ceb26b0546b57f9a7cdbff08065541d63492d4059513","canonical_id":"789","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,11,17,23,29,35,41,47,53,59,65,71,77,83,89,95],"fisher_boundary_sha256":"03a7963bc281465bc3ed237e9329c2d675fc5a2743cb29e2366d18c053e4354e","horizon":1,"input_positions":[6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96],"input_token_ids_sha256":"51ddc5b200c5449da711c0cb31fd958274770a19f7df1e0a919bfde5c62fa46c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,13,19,25,31,37,43,49,55,61,67,73,79,85,91,97],"target_token_ids_sha256":"5b9e09d236094c93cd27152a8362d6fc0907405ed836f4c806eb6a49ce662637"},"formatted_content_sha256":"95cda84841f0ed6748d2aa3eab9b3f2a1d602bef0648c2354cb235fbc49eca99","generator_receipt_sha256":null,"identity_record_sha256":"1b904dd9e776f5697d87583519b28c5869436344070e3c267f5d2f9cbe24f679","prompt_token_ids_sha256":"59d140fe1bf3dedbe286b918f769e76d9062fc0a8677694f8a29a4264c3dcfc6","ruler_category":null,"seed":null,"selection_rank":107,"selection_sha256":"435d524443c963ac0697492db5d755ed96c21825c6c12686e14ad1216a9d98d3","sequence_length":98,"sequence_token_ids_sha256":"f843f9ee0ba37389203b844c5d69910ebcaaa1266a694030f5cd56fa427be6b8","source_content_sha256":"528cefc12ccc28d572fc35698c3e55d5fa17974dcd83ab45fe269d5d96166d59","target_token_ids_sha256":"e052b469e73f16555fd8aae9d7e12029ab50d45bb5400880bce25ee85f7c0582","token_span":{"cache_exposed_start":98,"cache_exposed_stop":98,"prefill_start":0,"prefill_stop":75,"scored_start":75,"scored_stop":98},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"56264414b48afbfd02d06a09b83695f1be6a109c5e5917043688018a40e3a3c2","anchor_positions":[23,47,71,95,119,143,167,191,215,239,263,287,311,335,359,384],"anchor_positions_sha256":"5ecef0009b1f2b5ca1c05ae59fc5945152ea0233cf7a7da1d75b9641a0da44d4","canonical_id":"819","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[22,46,70,94,118,142,166,190,214,238,262,286,310,334,358,382],"fisher_boundary_sha256":"83c8e0577de0e56d18d8697c6d54b87b15a0a615623c44993bd8b9c2163d30c8","horizon":1,"input_positions":[23,47,71,95,119,143,167,191,215,239,263,287,311,335,359,383],"input_token_ids_sha256":"e56c492befed4cd096e4eb5258b978187c18cba26036c7ea511ea7b3828aea8a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[24,48,72,96,120,144,168,192,216,240,264,288,312,336,360,384],"target_token_ids_sha256":"b5f4bffe29cf172dbb04ddcf7fd6b1f217b698f7016f31ff8f6e48c727e2c9be"},"formatted_content_sha256":"616a3535fe1ec791a293ac6da5c856a7712282d6db633cd638270d4013cec624","generator_receipt_sha256":null,"identity_record_sha256":"c33ba3115c44c968c4ff9d9c33e133ea4f975549b4e7717c6581b9f976c187f0","prompt_token_ids_sha256":"de127a7cae4376da7d66c0f3f1eae83db093b0eee64a2a21dde2ac28c80f0322","ruler_category":null,"seed":null,"selection_rank":108,"selection_sha256":"439c6f2d57f5500b6cc21860ee22a1ebce8b83eeeb2bb034371733828ffa361e","sequence_length":385,"sequence_token_ids_sha256":"5ff88ef079fd63302a6d16c9b8d2bb597b0886ed6975c9ee0d7005fd42a6852e","source_content_sha256":"1b57006191af74fc5cabf13ded0936d54e33d0eb2d92e12170ffbdf83e11e029","target_token_ids_sha256":"f0b593d02c8c7048d24a55e2360b1e42ecae542816dcf5a6d3a7a5f0d3baa103","token_span":{"cache_exposed_start":385,"cache_exposed_stop":385,"prefill_start":0,"prefill_stop":268,"scored_start":268,"scored_stop":385},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"f83dda5f76b6c77a2f8f34d32d59899a5438aa93e284ca72500f57061c684a29","anchor_positions":[8,18,27,37,46,56,65,75,84,94,103,113,122,132,141,151],"anchor_positions_sha256":"99b598ab1b26514b0793ce6e9d9b8fd5fa21cfd91c15cf4ba944b02280eccc00","canonical_id":"886","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,27,36,45,55,64,74,83,92,102,111,120,130,139,149],"fisher_boundary_sha256":"18cbe59ca514ee6ee4de7a9ff59d5f2173303011464fcc35cb15114c03969d2a","horizon":1,"input_positions":[9,18,28,37,46,56,65,75,84,93,103,112,121,131,140,150],"input_token_ids_sha256":"7e8df804a72a77f8e56f283f6f3d3e0ccf7bdd663763c7ba3cf49b6cf2c8c479","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,29,38,47,57,66,76,85,94,104,113,122,132,141,151],"target_token_ids_sha256":"fa410d4aeb53ce61e53c7bf6ae6bf9008a5152a79caa4db0b21713c71de06b6d"},"formatted_content_sha256":"1be2ddc47aaf0ef4c6ce6979f16a7daed6383578cbb0e12a913e8c76157fe911","generator_receipt_sha256":null,"identity_record_sha256":"16353e62969e446cb1fd7fc9ca7a99ffad2f73e31b73381c626a08c7293cb32b","prompt_token_ids_sha256":"fdf8cba22d99356b2668bf60f4e3c5f33e9981ae71ede27ef250e9945224d770","ruler_category":null,"seed":null,"selection_rank":109,"selection_sha256":"43d5d553cb2e28075980d0bd0920a10d225c7497df856fe96c4367fbe7e6bace","sequence_length":152,"sequence_token_ids_sha256":"8853c262834e9d1226bdfde776d4ef0e7a0e7c8778a464180efae64bfb4f3c20","source_content_sha256":"434c4f97415e9ae869612f384ad8eb1d4459b91e71891d13598de2c0ecf58ce9","target_token_ids_sha256":"c73711379fdd03655ee8104d211db2039b2bd1b1d8757202f91122e2e8800f82","token_span":{"cache_exposed_start":152,"cache_exposed_stop":152,"prefill_start":0,"prefill_stop":120,"scored_start":120,"scored_stop":152},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9bfe15074048c1b3cc4e9a64f9f82a7762f5e3fc7b6286e9505bebdfc8ce78e6","anchor_positions":[17,36,54,73,91,110,128,147,165,184,202,221,239,258,276,295],"anchor_positions_sha256":"54ac14bb1cffe66a9bd2a063df3998236174c0775660ded8a46c6b4f3b34b4be","canonical_id":"669","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,35,54,72,90,109,127,146,164,182,201,219,237,256,274,293],"fisher_boundary_sha256":"e64ce615c2670462c794e8f67c34ff9ba26d820a1d46601098cb0321f7249f65","horizon":1,"input_positions":[18,36,55,73,91,110,128,147,165,183,202,220,238,257,275,294],"input_token_ids_sha256":"2147411fed9b01bc77a54da5a575818dd51a16507fe185b59c616201c9dbe92e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,37,56,74,92,111,129,148,166,184,203,221,239,258,276,295],"target_token_ids_sha256":"d9cc42057cf2f44e2193c2fff4ec48c3d54db14f1369ce5096749eb0a37be2eb"},"formatted_content_sha256":"fe004ffe4d515aa06edc2d7a6ff8f2900da04c2c06fe055101a34ba872ba040d","generator_receipt_sha256":null,"identity_record_sha256":"5a4fdfb2bb3bdb18316e26b3524adf61c217b8661a9502ef4888ec80cf44a517","prompt_token_ids_sha256":"545545ebb1988d32125cc7c0b6360272908d9ddf46dbea3b96b9923e8f61a5da","ruler_category":null,"seed":null,"selection_rank":110,"selection_sha256":"454527a603157418a471cc4d85900b7bc64b30edc7df17206aecd43f4b706a81","sequence_length":296,"sequence_token_ids_sha256":"0c56d0026a937592d71b7773e5acad5e8774badceb5806f30884aac130918445","source_content_sha256":"76e713c3e769794fd21cc2b169d3fd25155269e05a87202fa2ac7130dd7d9534","target_token_ids_sha256":"e5afdd086f0ae8795afe5c271dd9f9a4fae2f74fda8cd0e04e3f697b2d39bb25","token_span":{"cache_exposed_start":296,"cache_exposed_stop":296,"prefill_start":0,"prefill_stop":113,"scored_start":113,"scored_stop":296},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"bba06f38d7ef46d2274fcc3c62d6b99545aea08fc9a970bb2cf4112497f86d55","anchor_positions":[18,37,56,75,94,113,132,151,170,189,208,227,246,265,284,304],"anchor_positions_sha256":"94f33a91bb15b5b174db14bd5a18a220943ce79ec8eba0ee2655b965e84d64e0","canonical_id":"846","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,36,55,74,93,112,131,150,169,188,207,226,245,264,283,302],"fisher_boundary_sha256":"e56eaa1d53ef9cac5e4ea05eac5fae5d2ff32456ec4fd16aa432bb0ba682df16","horizon":1,"input_positions":[18,37,56,75,94,113,132,151,170,189,208,227,246,265,284,303],"input_token_ids_sha256":"b57a987371cd39d03b724b08c3b6837260a5e8ad281a18e85c9901110811e452","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,38,57,76,95,114,133,152,171,190,209,228,247,266,285,304],"target_token_ids_sha256":"adb018d6d0b2d7716ff0afe479b6ee4cb1e352197e41003e0e3962d666cf8a0f"},"formatted_content_sha256":"d783edb3ae5e40760a243586ed62d48e4d1fab758d36793b9e509e3e34c74a3d","generator_receipt_sha256":null,"identity_record_sha256":"3c715e7b82c94eee9feb5fcc68ac303cd191e6ec2068a3d300d3ca8e4dd6e5de","prompt_token_ids_sha256":"604c21c3a8e0575c89ca1b7911771a2a7ef7b4a915f2b165be5a3a2c56b206e1","ruler_category":null,"seed":null,"selection_rank":111,"selection_sha256":"45dda3a343ef8f05e6b253db4cedda79669dd8cedb0368cd7ba98b9b569952cd","sequence_length":305,"sequence_token_ids_sha256":"b0ea9f94ab26fc47af030e71000bddd166cb8e1001febd6121f29acbdf792437","source_content_sha256":"e03f7274bd5dfa833b5d69f144a32a2866a4d5e2df73779e455ab0a797e8b9f6","target_token_ids_sha256":"7c55e00d0effcd9ceef682e102dd49697117fe7a162f11e2bcd35a18523b96dc","token_span":{"cache_exposed_start":305,"cache_exposed_stop":305,"prefill_start":0,"prefill_stop":182,"scored_start":182,"scored_stop":305},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2f002330a14ccd49bbdf1186b94d7f4a660f3c79ca0935f6733429506fbac4dc","anchor_positions":[9,20,31,42,53,63,74,85,96,107,117,128,139,150,161,172],"anchor_positions_sha256":"9bb32a96de9c1958e4d75c68046a769ab85020ffa742f7286cba9f5cfe66172b","canonical_id":"809","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,31,41,52,63,73,84,95,105,116,127,137,148,159,170],"fisher_boundary_sha256":"b791fc560913737b45746c61831af949c9b84432325347422a7a866d99f55180","horizon":1,"input_positions":[10,21,32,42,53,64,74,85,96,106,117,128,138,149,160,171],"input_token_ids_sha256":"35bdf9fe7875294a945c0b7fa771287b6e892aade75b41750c2e0144945a8d4f","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,33,43,54,65,75,86,97,107,118,129,139,150,161,172],"target_token_ids_sha256":"901d22acbaeea26debc37b388abaf43ac50248b0422016d07fd3a292798c4cd0"},"formatted_content_sha256":"7d79ec67027ef058aa16aa49df49541f7821cc1e06a8283c93d05218bee5c30d","generator_receipt_sha256":null,"identity_record_sha256":"00931944a29fecfc4c8c6b60832e5269dd3d431f1fc0fec8f21cfc3d5b2e56c5","prompt_token_ids_sha256":"d2c4a745f10a0a0263c72f290f8ef232ab4aafe62f10358bad7a317ba0990e25","ruler_category":null,"seed":null,"selection_rank":112,"selection_sha256":"4639d4a26db93822b4b2e18576a8169bd874398e85f3a592904ffc3f972fbb17","sequence_length":173,"sequence_token_ids_sha256":"97308fe18d6a1e87ca3681748a23734c46402fa3f1992ea007b186d80d049540","source_content_sha256":"197f56fd23a55154213be6170da63f959f105eb46b7cd9cb707820e74a06529e","target_token_ids_sha256":"c3f58b0c22170cfdfc5245c921af88598385a2623da14949ba7503f68723c427","token_span":{"cache_exposed_start":173,"cache_exposed_stop":173,"prefill_start":0,"prefill_stop":128,"scored_start":128,"scored_stop":173},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"696c3eff19f8cd3ba60be744325a381bcade3a5bde2058ce2b7057bd86a602e7","anchor_positions":[8,17,26,35,44,54,63,72,81,90,100,109,118,127,136,146],"anchor_positions_sha256":"95e4d2457eadd7ff2ebe49af36c11afbb6034a7c8d08013efa9de5325f17bb5d","canonical_id":"797","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,26,35,44,53,62,71,80,89,98,107,116,125,134,144],"fisher_boundary_sha256":"0f290f46b842b2d9555c6342ee65ad579f7f6195e2840a98edaf138b7f77e879","horizon":1,"input_positions":[9,18,27,36,45,54,63,72,81,90,99,108,117,126,135,145],"input_token_ids_sha256":"45d165859afaa71c112d2d4db0282de95319684d4d160e6d34be3255f19e378c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,28,37,46,55,64,73,82,91,100,109,118,127,136,146],"target_token_ids_sha256":"dd11e1c91e7cf97f459b2696ad0d206a7e54b2e224552b8590c9c6431e8584a1"},"formatted_content_sha256":"8e313e746cb70b0f39a63b254855866da5978faa757f114d540a3e7ae9d63cbf","generator_receipt_sha256":null,"identity_record_sha256":"6952ce6716bd8931f4ca2c58ccaf72f466062aa0785261289d59a5104678a7b5","prompt_token_ids_sha256":"04b1d2f628e08d770e64feea8b70cbb7a64b347ed50b692622c6c3e0c7c8029d","ruler_category":null,"seed":null,"selection_rank":113,"selection_sha256":"46503a4924e57013ffda3af84d33a2db536b465afca8ee866ff2a2e87961e472","sequence_length":147,"sequence_token_ids_sha256":"dbf6194f0a08074850f461642c989526f8fe9049d05306dfd6a2da9a9987f8f8","source_content_sha256":"aa80156f16bf70bd62cebda84762b9a1a8ae437dc8871c0acb7d61e9d83b1b8a","target_token_ids_sha256":"d91b2a60c8ed8407d0c17107efc9a4cc4e719d8a094196cb04d089c6d8aea011","token_span":{"cache_exposed_start":147,"cache_exposed_stop":147,"prefill_start":0,"prefill_stop":91,"scored_start":91,"scored_stop":147},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b907107ca4056a13d77b90b11cffa6104f002742e48e4d73d84701f35cda3ce3","anchor_positions":[7,16,25,34,43,52,61,70,78,87,96,105,114,123,132,141],"anchor_positions_sha256":"4a3ecd9aedde633817cc01e0837692eec5dd57faddeca3f7bcf5e18333ad7c64","canonical_id":"803","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,25,34,42,51,60,69,77,86,95,104,112,121,130,139],"fisher_boundary_sha256":"8c5dd3de4221407fb31879a8831c58de078b7bd96edff29efabeb0d4f941ca5d","horizon":1,"input_positions":[8,17,26,35,43,52,61,70,78,87,96,105,113,122,131,140],"input_token_ids_sha256":"612f42435d82b79fa57bcb8dc07f12f964553d764d942f85ac5c989daa0ac60d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,27,36,44,53,62,71,79,88,97,106,114,123,132,141],"target_token_ids_sha256":"20ecbcd0e6f4d0f38e6e34055d4306d4556676db803bddcbf4efb50dde35e518"},"formatted_content_sha256":"b80b90161540d8a9b1c0ba6896342db1e035ccefbe96663db9df9b74d82a3eae","generator_receipt_sha256":null,"identity_record_sha256":"cdbc48a4424d2515ce6141290d69310f055340010d03518c808f7666de9eecb6","prompt_token_ids_sha256":"9a94491891f327307e6c2d541dae50c594d32f7e7113f10a4ad421cd16f7efee","ruler_category":null,"seed":null,"selection_rank":114,"selection_sha256":"47a81fc27a92568642839ad9294c190dcd0b21f98ad4eca4472567609f81bac4","sequence_length":142,"sequence_token_ids_sha256":"4e6d4b6ce7a5a64ad6b5d03e8b6c0786f2a1206a63cd36a35f88e41282093685","source_content_sha256":"fcc558736fc46d07648e59309cc19de0322f28060b2265985cd5cb46f9ccac97","target_token_ids_sha256":"16fcd9ca012cf831e086d70b1c2f66e51e595f470fb2610a600125f50807abae","token_span":{"cache_exposed_start":142,"cache_exposed_stop":142,"prefill_start":0,"prefill_stop":82,"scored_start":82,"scored_stop":142},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"61f0fff09b788c1dde584ff85418eee082c77947d695e4df4e831eb33cf048eb","anchor_positions":[10,22,33,45,57,68,80,92,103,115,126,138,150,161,173,185],"anchor_positions_sha256":"fde5a3b64a487819cb5be9d572133781e4744390c11836e2f115ba2608fe7216","canonical_id":"644","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[10,22,33,45,56,68,79,91,102,114,125,137,148,160,171,183],"fisher_boundary_sha256":"3c448eaba6e976f4d8540a8018f1b6ee02f1a078e5ab97372fe15de20e37c79d","horizon":1,"input_positions":[11,23,34,46,57,69,80,92,103,115,126,138,149,161,172,184],"input_token_ids_sha256":"61a5b967b5a2460665ba8ac148a2ed8fda9b1a0e95c8d66a509db1c89e384bb5","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[12,24,35,47,58,70,81,93,104,116,127,139,150,162,173,185],"target_token_ids_sha256":"e6285297df83884b53ce8b429f39bfbc76734d91c897b1ef6b45e02c90bdc53e"},"formatted_content_sha256":"cc789f8b54535a69935595ebbc2d15ace8da8a517f3e3a766f4c9801ab975e73","generator_receipt_sha256":null,"identity_record_sha256":"add76763c425e50ca33682da8956ff29cc4547518dd84c9fea5b339de6f6125b","prompt_token_ids_sha256":"a672313ea736e1353ebd2a843db4d5ae378ddbec92e54cb81d21376a13087e88","ruler_category":null,"seed":null,"selection_rank":115,"selection_sha256":"49614c4b89fd4e064236884bd54d2b6744d30da34cae36de98a3f8d6dc005295","sequence_length":186,"sequence_token_ids_sha256":"2a629a0ceb8ed135d2d3814402c87b6b4d4925c16ec65dcf756469c7f70d28dc","source_content_sha256":"6ca1519c2cb7b633366e44afc37a261e40427c751412998cceb863343020cf4e","target_token_ids_sha256":"147bf03e5bc17953126d30a0640ae17e4b57cbaf4d65322290bee14c6f8c12a8","token_span":{"cache_exposed_start":186,"cache_exposed_stop":186,"prefill_start":0,"prefill_stop":160,"scored_start":160,"scored_stop":186},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0a09087a0815d40c6da7249de80b96629effaed44659b49eb51e112de7acf916","anchor_positions":[5,12,19,25,32,39,45,52,59,65,72,79,85,92,99,106],"anchor_positions_sha256":"1ec66db99ded67a6b74f7dd5259127f2ba61fd9a33265762c1ea04710558d7bc","canonical_id":"647","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,18,25,31,38,44,51,58,64,71,77,84,90,97,104],"fisher_boundary_sha256":"bbab335f18e2c0ccb810bd85c071826d79d9b490e590cef0c6d681341c0cc08e","horizon":1,"input_positions":[6,13,19,26,32,39,45,52,59,65,72,78,85,91,98,105],"input_token_ids_sha256":"46b16340d489ab9d5e99f73c42d4a7005f7436243eef5fc34fa92767754e2622","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,20,27,33,40,46,53,60,66,73,79,86,92,99,106],"target_token_ids_sha256":"761af83bc90049bcb9676e87032fb087e505ce54ccdace5bd3157cfe531b0e6f"},"formatted_content_sha256":"892bdd77965161d3f33ecae4371aaab245852c1a733575ea82a03f4c5162b6e9","generator_receipt_sha256":null,"identity_record_sha256":"096bf170f6323441d41cfdcd82e991e5330ce43bbca166fdaff8e2867cb3901f","prompt_token_ids_sha256":"9d1498f0e195703f3d0a9bab6ab2b95760b77762a386814e263445e614eebe8e","ruler_category":null,"seed":null,"selection_rank":116,"selection_sha256":"49bf4e9209616a951134062596b8e1762dbe5ae3d6768f1c37fc3ca1a831c8d8","sequence_length":107,"sequence_token_ids_sha256":"e9307774f416105649348d7b962aed8818afdd9670718378a8011aab53794a9f","source_content_sha256":"a4c7ff187da9b88bed24a5ce4e33b1d245e061cc2f7fb731157aeb3dfaf250ab","target_token_ids_sha256":"d3f2b417944e705153e271af9eeac8e255eb09fdbc9a8241cedc49106c05cb62","token_span":{"cache_exposed_start":107,"cache_exposed_stop":107,"prefill_start":0,"prefill_stop":82,"scored_start":82,"scored_stop":107},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4294762e8478186f4347187eec4203bd212ed4716abafee16d6c831b211a92d0","anchor_positions":[6,14,22,30,38,45,53,61,69,77,84,92,100,108,116,124],"anchor_positions_sha256":"fe7a10db15891cb54d23372b10af49697c0401f54d8f6b37c01417545e9ba930","canonical_id":"605","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,29,37,45,52,60,68,75,83,91,98,106,114,122],"fisher_boundary_sha256":"c098a5d59dd920acf5e6ecb372e0ff5df7d0b3ee1501bb011f699481cf66e206","horizon":1,"input_positions":[7,15,23,30,38,46,53,61,69,76,84,92,99,107,115,123],"input_token_ids_sha256":"faf2e5d622e9951baec003f38847af5d1319f255169b8c041a433d5b46952e3f","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,31,39,47,54,62,70,77,85,93,100,108,116,124],"target_token_ids_sha256":"26f6b36890856124d1f7f7a676b24ea8aeea3382d92cd1489d141c22982a041b"},"formatted_content_sha256":"80aa72fd69bd10f4c2e5403802378e6fb00d15a1e298252ded65c6ce760ab095","generator_receipt_sha256":null,"identity_record_sha256":"ee34d2c3691827aa084c190eddddebe9d1345888c7f48e25c87d60d765f6dfcc","prompt_token_ids_sha256":"9fd32241d4f2b36ee9f01d68e556fe7d2b96791f996bcb4bddb46b56225c1199","ruler_category":null,"seed":null,"selection_rank":117,"selection_sha256":"4c310c24967e920116a6713c985dc574fc27f51aa0e5f96102923b5848a2fea8","sequence_length":125,"sequence_token_ids_sha256":"d5c71f5ac8cca279a5091bc56ba019e4c7f2a97540c957843ce3658f1edad1ac","source_content_sha256":"dcf527a4f6986a9167a459248d6567bd87701d5fa34856a31f9be5ee02c91168","target_token_ids_sha256":"741ab7638f051c1d782ffdca7cafb68a673af7ec2773fdff58abe66e46d7eef1","token_span":{"cache_exposed_start":125,"cache_exposed_stop":125,"prefill_start":0,"prefill_stop":68,"scored_start":68,"scored_stop":125},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"05c8e3cbead3af43c5c1e6e8d87467ab8c03ec4648a4298153b82d936448e5ad","anchor_positions":[19,39,60,80,101,121,142,162,182,203,223,244,264,285,305,326],"anchor_positions_sha256":"f3e5839dc1785a2c00f87ad2ea0e816ae94270b3238251e62817f25d09a21cbc","canonical_id":"630","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[19,39,59,80,100,120,141,161,181,202,222,242,263,283,303,324],"fisher_boundary_sha256":"de76b71c7d9a5873e8f18d38d9bfe274b70bf5ccb7ad95749ffff39ac8269bce","horizon":1,"input_positions":[20,40,60,81,101,121,142,162,182,203,223,243,264,284,304,325],"input_token_ids_sha256":"b71f4ccff2a296d03b9b8fada60f1b7538853521101cb758ed05e1b6e41628d2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[21,41,61,82,102,122,143,163,183,204,224,244,265,285,305,326],"target_token_ids_sha256":"939dd654b199e687770731bddc84e26956b5c40203089d198687d9c12778ed94"},"formatted_content_sha256":"b39458254cec5c52b4b1ffb7f1dc476b0e99bab62c4d66d0998f147e9cd767cd","generator_receipt_sha256":null,"identity_record_sha256":"d20e005d97d9c08185e9a25884618ab3451741d9e00a6f7cf4842daa416fb1a1","prompt_token_ids_sha256":"92effc02a358ea378746b418ac301dfe5bacb980d7a8a9bfee9363f21a656396","ruler_category":null,"seed":null,"selection_rank":118,"selection_sha256":"4d4a3d035c7868292b96361ee3387aa0653c72bf82ff2ffbcdeee1c544f9b310","sequence_length":327,"sequence_token_ids_sha256":"11215c622904358795cd7771cb56cd3e68586e31dea00e30330fdece99fcd790","source_content_sha256":"d026e046adcf26a5522065e389d1e74a8d7ef963fd53e4f8db8952c3c3058226","target_token_ids_sha256":"59f027a42d76a9550f2a9d95174c87b9fd402df01c12dd6433c83e8bbb1834ec","token_span":{"cache_exposed_start":327,"cache_exposed_stop":327,"prefill_start":0,"prefill_stop":236,"scored_start":236,"scored_stop":327},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"865d6b59e048496b3bf7c4b287421ac71174d11943f8a146bf5f215f6e662bb5","anchor_positions":[7,16,24,33,41,50,58,67,76,84,93,101,110,118,127,136],"anchor_positions_sha256":"b394d1b64998869298fd2b292d4ab84e6faf0b5a40eeb811c668f2563bb4df64","canonical_id":"868","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,24,32,41,49,58,66,74,83,91,100,108,117,125,134],"fisher_boundary_sha256":"7f4687722337e0596bbc45f63cbf6b70e8988c03af12154439f27a9db973cb09","horizon":1,"input_positions":[8,16,25,33,42,50,59,67,75,84,92,101,109,118,126,135],"input_token_ids_sha256":"8d01b29f328a257d664952e75b1923e83d2777cbcd4b7dd6cee1855ba4bd0e23","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,26,34,43,51,60,68,76,85,93,102,110,119,127,136],"target_token_ids_sha256":"8b8ebd92cf550b5df42ff3360fbfe39d8efc120a7e745a701b2476dc01b5b305"},"formatted_content_sha256":"44d9a5afa4c9790a32cb254c2d7e1c8151d16636b7c10318c889176cec4bab1e","generator_receipt_sha256":null,"identity_record_sha256":"fa45bce52100c237b5694c601b0c6d6d61bafa11dc08e55e2816e2dc5931e145","prompt_token_ids_sha256":"b9f642149302b4aad7f0cd78c10cba633079e023b978bce75611464d15bbca52","ruler_category":null,"seed":null,"selection_rank":119,"selection_sha256":"4d7d9814d186699e85046419caf35e8bacfed3ab6650cbd36ef4705e2fe2367b","sequence_length":137,"sequence_token_ids_sha256":"b282f3d8b38678f37b26da12ae374fa98bd55719947e98f5670fa8a8530270c8","source_content_sha256":"eae1ffd3380c73bbc35bb8929b99e7549668c0ee1f54da998081ad923ebc2b98","target_token_ids_sha256":"92e8679c291f99696b3ac568510fbe994ddeef46b7ee37dd2252d2f7a9c4bf84","token_span":{"cache_exposed_start":137,"cache_exposed_stop":137,"prefill_start":0,"prefill_stop":78,"scored_start":78,"scored_stop":137},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3053356b37e0ec018499d9df09100d5d459ba083855ec3be6383459c76336c2f","anchor_positions":[18,37,57,76,95,115,134,154,173,192,212,231,250,270,289,309],"anchor_positions_sha256":"1d1fbb7874404570c3a9cce52a9bba801a787fd59ed00a041f9aca9922c13383","canonical_id":"902","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[18,37,56,76,95,114,133,153,172,191,210,230,249,268,287,307],"fisher_boundary_sha256":"b03addf9ce28714390d0c8aa72e4cf5bfc4d998e1e89125d619cc5de86bb2d27","horizon":1,"input_positions":[19,38,57,77,96,115,134,154,173,192,211,231,250,269,288,308],"input_token_ids_sha256":"a9d774c7e1c1f9d59bd7b757d0d21d9849e8eec8052afe6025037cde871559f7","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[20,39,58,78,97,116,135,155,174,193,212,232,251,270,289,309],"target_token_ids_sha256":"52cd56fb6fa5b1573d45b6aa37a594fb645c879bc5efa69d6edc441f4e348a6c"},"formatted_content_sha256":"5c1d9ec33b3a94931255d1220cce10d4ee097868c37884a8e19ae9f9a448966a","generator_receipt_sha256":null,"identity_record_sha256":"11c597e2cf580e07f77afc5041e62139cfccff5f5ab2c548565673fb0b500832","prompt_token_ids_sha256":"bf15bcbdbc30699f9cd0081bb00efafed21ea1abf4e5381da4e70c8a8ea2907e","ruler_category":null,"seed":null,"selection_rank":120,"selection_sha256":"4daa86751a582244f7d590b6ba730fc4e01e1d3eb676941a3552f1d490faba8f","sequence_length":310,"sequence_token_ids_sha256":"376f09bf804c7e1d57cc7e70ae46ecf6050c3c815f6a7cee4a95bd11199c01d1","source_content_sha256":"57ceedd084b1dac9a04f6d5be3b4a7e63c4474855a5cbb2577214912ffc07f8e","target_token_ids_sha256":"49502604cb2f2efcb1c5cee4993331ea6bb17ff438c88e61241bb8472ffbd75a","token_span":{"cache_exposed_start":310,"cache_exposed_stop":310,"prefill_start":0,"prefill_stop":278,"scored_start":278,"scored_stop":310},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"afd54f87610f85707a02f20f8b511f975ef944c51b61fa08f447b37ed40c6a78","anchor_positions":[6,13,20,28,35,42,49,57,64,71,78,86,93,100,107,115],"anchor_positions_sha256":"5af6cea1df262184ac4897d6d91d25605392e5fb08e41b5742c64172e9670d87","canonical_id":"955","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,13,20,27,34,41,48,56,63,70,77,84,91,98,105,113],"fisher_boundary_sha256":"1c62dce0ea57f80fb1a56a3ea511f2ca9f646f6413e8db3d99f5d99de6521811","horizon":1,"input_positions":[7,14,21,28,35,42,49,57,64,71,78,85,92,99,106,114],"input_token_ids_sha256":"ef01c4ac7ec6c6b186fdd4eb31a7f21ee8d3e4dc566355d91c713471887db336","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,15,22,29,36,43,50,58,65,72,79,86,93,100,107,115],"target_token_ids_sha256":"785091482f15a4ef0949a945686a1778f4de4cf07b8104861aa31a9db7b18f4a"},"formatted_content_sha256":"ef1384fcf9c059d64323dd5fa9880b686c5b7ea953ce70d245534bb1c6ed7137","generator_receipt_sha256":null,"identity_record_sha256":"f520632d9fe13e3b9ffeebdc9ea540d828dee798f54926584e9872d0fe7b87da","prompt_token_ids_sha256":"010438e2e3ae3ee5626141c874feb0f74e50fca32fd27c7d8e49a07e790ec560","ruler_category":null,"seed":null,"selection_rank":121,"selection_sha256":"4dd5c5d56c9d92dfa81aec10aa5beae38c2061e96e799b813b238234a0ce8e4a","sequence_length":116,"sequence_token_ids_sha256":"fcc6077ed67145358daf5bcf3eb1ece0e289f68f139666cb89192eec34ca4386","source_content_sha256":"8408dff25eaf845d972ce9ae510b9d76eaa3e7a6bcfb8b8c7a628d70d2dd6cf6","target_token_ids_sha256":"76411fdcb68c0c051819b5bc2c0fed5b4048dcb9000413fb60e5de16d537d568","token_span":{"cache_exposed_start":116,"cache_exposed_stop":116,"prefill_start":0,"prefill_stop":72,"scored_start":72,"scored_stop":116},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"6bfb1e634cbb65a8f9c6d30dcee64be9d19b2bf1774c4fa850c4b7b5ce5abb9f","anchor_positions":[6,13,21,28,35,43,50,58,65,72,80,87,94,102,109,117],"anchor_positions_sha256":"f40be3d1cf7064d7e86e3a810371d8f2d368583e1dcff4660d5d02fc4a37f43e","canonical_id":"640","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,13,20,28,35,42,49,57,64,71,78,86,93,100,107,115],"fisher_boundary_sha256":"261088f8bd3fe25be23146357491bf2699628f4ce493685b028cc1ddbac01eb3","horizon":1,"input_positions":[7,14,21,29,36,43,50,58,65,72,79,87,94,101,108,116],"input_token_ids_sha256":"db9064f4846367e8570c2c84fe797b835d18960e9211681a5e0daac0654b2347","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,15,22,30,37,44,51,59,66,73,80,88,95,102,109,117],"target_token_ids_sha256":"0a9ab2151f96a33137ea1ad001a0d76f499d390891517e6c83bd2ac4188c1207"},"formatted_content_sha256":"06826303346d9d86d61fc5abbffb92ebac62b2e090a9f70b82927eb026441a29","generator_receipt_sha256":null,"identity_record_sha256":"7b2ff5b497e0a82c3b0d260a880af1a82e4a4c08c8185dd3ce186597015d3a20","prompt_token_ids_sha256":"05a2a260c33e6e500d66729f3e56aa8f9ccd92ecf9825c6449d5f8fe340e67a0","ruler_category":null,"seed":null,"selection_rank":122,"selection_sha256":"4ddb9210eb7ae0fdab8792259d081bdba6eae451141c47a811a0245cec7ee62d","sequence_length":118,"sequence_token_ids_sha256":"20509fd474053e85eba21ed705bd626c6c6f337445f4daf2177bf8b506211954","source_content_sha256":"6438ac844fd844aa7880ed46075764fa684488592b7d562f4ae65218f8b35da2","target_token_ids_sha256":"fd3de534e5c7f8e743fbfaeaa6eb5d1a4c3d2acfdfd4a83cb4e9f6b0a9383173","token_span":{"cache_exposed_start":118,"cache_exposed_stop":118,"prefill_start":0,"prefill_stop":85,"scored_start":85,"scored_stop":118},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"f5ff8bcd127367dd786bcc2b0c253bbddf5577a9692a04a945dfd7611e658813","anchor_positions":[7,15,23,31,39,48,56,64,72,80,89,97,105,113,121,130],"anchor_positions_sha256":"4b59ffbfdc6171d05db7e4ce13fd81c13fb1c2704ef54cbfaede26eafe2b1382","canonical_id":"602","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"fisher_boundary_sha256":"4366416cc3edb9f1e2ee5e0a55178de659ab8d05d59a8cbbce27f688cce7b2e6","horizon":1,"input_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,129],"input_token_ids_sha256":"2db46a104b4295959d6a16c851488604e471fef800ab5243426a1d23cce719c5","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,33,41,49,57,65,73,81,89,97,105,113,121,130],"target_token_ids_sha256":"2b0416f82f51105c2f972ed61b19f300631a81a49326605f7f5bf45fc3d5957e"},"formatted_content_sha256":"48559307e617b830c4e7a2ef8b20df2f6a53a7ddd15f766c7415c7b8597ff2f9","generator_receipt_sha256":null,"identity_record_sha256":"baa5a63154617acc29a3b5192d3459d2edae4c0959c00d24e2dadeeb3837d757","prompt_token_ids_sha256":"0bb478e0d8162f79fa8ab3da8baed5b87413927171e2f2c8071c49ab30da5521","ruler_category":null,"seed":null,"selection_rank":123,"selection_sha256":"4e02d4ed1f0c6d170eb518cc4020d82ff4ab1bd18d4949c5913e319fa7cc9e83","sequence_length":131,"sequence_token_ids_sha256":"e7b30f61231d01e87b3afa4dee3faf2008c8e20ffd1b3a478e7a94932567aac3","source_content_sha256":"59b47992e5e199dff2a5b0af7ea0a02713ea803a5dcc6fc3db6ecda2c4879288","target_token_ids_sha256":"78bd9c18ec9dfb40ae35c333bd6dd45162c4cc291b415b84d5ae2a87dd1ea334","token_span":{"cache_exposed_start":131,"cache_exposed_stop":131,"prefill_start":0,"prefill_stop":86,"scored_start":86,"scored_stop":131},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"bf2d404007f3e1af973b816cfc665fdfc8257f337deb35322ac0077058cea4d9","anchor_positions":[12,26,40,53,67,81,94,108,122,135,149,163,176,190,204,218],"anchor_positions_sha256":"34e0f2438d49fed8c871a33d3818a40fc035b239ea8639ec66d64d1e59a9bb66","canonical_id":"712","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[12,26,39,53,66,80,93,107,121,134,148,161,175,188,202,216],"fisher_boundary_sha256":"957de9da750e31aa4980cb34b1fb160bb11d67e08faf67af18d3cd3e5289f959","horizon":1,"input_positions":[13,27,40,54,67,81,94,108,122,135,149,162,176,189,203,217],"input_token_ids_sha256":"f56a9ec29643c9eba2eb44791cb09e84d9f2b231c902427ad5756e74ec534316","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[14,28,41,55,68,82,95,109,123,136,150,163,177,190,204,218],"target_token_ids_sha256":"c0a62a7a97676edbafdec6adb2cfe3c67719d5ccdcfcc3a0ddba0c6331624ede"},"formatted_content_sha256":"b47cde36a934283ca86c0366ca05754f6267c0fa319aa8c9ab18bc5ff0e5d9b5","generator_receipt_sha256":null,"identity_record_sha256":"4cc5c35280b65e6893e928c4b6a5b2caa568e303f0840d0df3438ecc6b258676","prompt_token_ids_sha256":"4ef7f3aa586f42e443494b0dc4987d24f11b6cc50acb5df627b135819de32092","ruler_category":null,"seed":null,"selection_rank":124,"selection_sha256":"4e132dddf48254f73a1575b1a311842bc8036e6c5310eeac68d27fcfa9705827","sequence_length":219,"sequence_token_ids_sha256":"45c3e814d2308b690e19384d22ca303456c689d9292a9292a9974a96d2c27dfb","source_content_sha256":"5355ae6f3f743535c93a53ce5c0f05092f5c9e89585eeb219a5a424a29b6a20e","target_token_ids_sha256":"8c478cb16206b569ca9079b27faaad0f0bf99e2182d4dd8ff62cd8328afac5ca","token_span":{"cache_exposed_start":219,"cache_exposed_stop":219,"prefill_start":0,"prefill_stop":183,"scored_start":183,"scored_stop":219},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"36c3b31211a995bad7f56d142b515472a78c8c0bab6bb376f23f53c90f2b7139","anchor_positions":[8,17,26,35,44,53,62,71,80,89,98,107,116,125,134,143],"anchor_positions_sha256":"486e7f47d72e2b3d4a862156a2b41e9dbb05a70d78349fdcac0b01e09667750a","canonical_id":"606","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,25,34,43,52,61,70,78,87,96,105,114,123,132,141],"fisher_boundary_sha256":"bcfe34f783ad5a75f3e21e034bca0957d966c976fd05656202a26e596ce4d242","horizon":1,"input_positions":[8,17,26,35,44,53,62,71,79,88,97,106,115,124,133,142],"input_token_ids_sha256":"2b725b30a4b9b4a7e69480982037a0a07c57e454ad8e62b46c0f0d34a3b1774a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,27,36,45,54,63,72,80,89,98,107,116,125,134,143],"target_token_ids_sha256":"872de4a20f2b57da5f4caff92476bd417ca5b3e9d701ef7d101a9fd65f42abed"},"formatted_content_sha256":"c6fed4dd2cb64cca6e833bf23797bb7c1d862a9c875ccc65acb9b0897618cc79","generator_receipt_sha256":null,"identity_record_sha256":"a9e7062d3f002e0a39343fd4f410be18e7409cd9643a09c6dedbf4bf79332f30","prompt_token_ids_sha256":"67adef290bf71e36a6af78dc46a182ea33221b75714d4b89977dfbfb25d08ad7","ruler_category":null,"seed":null,"selection_rank":125,"selection_sha256":"4e211b7dc81626479060883502f3e73d8b09c6e4da35910a1305a8e23d6e76a7","sequence_length":144,"sequence_token_ids_sha256":"e35fd3a90e9e66af3e142095f74dbbe6c47e3a63d75cede17749f5f6d49e6dd0","source_content_sha256":"3a119437b4f93db7da36fbe2d787607dde294a4a838f8b29d327bc68a0eb0478","target_token_ids_sha256":"764d32b0ec71bb9f1d078ccc8e4a39c3ce7dd359fa503899ebfcef2b1653ed9f","token_span":{"cache_exposed_start":144,"cache_exposed_stop":144,"prefill_start":0,"prefill_stop":117,"scored_start":117,"scored_stop":144},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7d1f918958bc691d31836ac50e195b86b748a5336a8a243a1b427eb6b289303d","anchor_positions":[9,20,30,41,51,62,72,83,94,104,115,125,136,146,157,168],"anchor_positions_sha256":"a040c10bf4081d1ff5f0840fe94899e3d5f4cc8b3513ac78d8caa23e146870a7","canonical_id":"656","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,30,40,51,61,72,82,92,103,113,124,134,145,155,166],"fisher_boundary_sha256":"c9142db1a8573b215599b5dc489eb1c6d88615d3cbf8ad1c104742ef7bfc5041","horizon":1,"input_positions":[10,20,31,41,52,62,73,83,93,104,114,125,135,146,156,167],"input_token_ids_sha256":"c32bee20d7de9784fc14e7f94e2b1e165c75b8c09339f3ecc44be1cbad01f8e1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,32,42,53,63,74,84,94,105,115,126,136,147,157,168],"target_token_ids_sha256":"abafaed5c5aabca6d515ea1a705691d431f604b08089f1f0b25c6df8b45c8305"},"formatted_content_sha256":"374f6e67c0234e75d236ea8f2069dcdf8a631ce916aaed870d7810e3347cde44","generator_receipt_sha256":null,"identity_record_sha256":"770edcf1d6f1f10e0e3b4c54ae7a7e8a532a07b1a3d17003f426dc095bed8679","prompt_token_ids_sha256":"b45a43c9c890b88c42ea8fe0a8f0a5d71c55ec6f01a772f30a952fad7f8431f5","ruler_category":null,"seed":null,"selection_rank":126,"selection_sha256":"4e7a2318f515292e8fc2e6b77f63c13218916891ac0b3f71650ab59f87fc1a5c","sequence_length":169,"sequence_token_ids_sha256":"c61b8bdf63e3f2d95107fee666460e20eb1e4a5d5420d41d523360fc8916c990","source_content_sha256":"7164e42f57848f0ec61d9fcc2a78bec0f09e6a54082574820bd69a7607d84721","target_token_ids_sha256":"52ef9360d90d7b7b754a31958af231bec9dfcb171dd416721cfb316918352b68","token_span":{"cache_exposed_start":169,"cache_exposed_stop":169,"prefill_start":0,"prefill_stop":118,"scored_start":118,"scored_stop":169},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9613ae532d1ac6180936e714bd20e0522fe6966715f1926f3a3b8b7af31b80be","anchor_positions":[11,23,35,47,59,71,83,95,107,119,131,143,155,167,179,192],"anchor_positions_sha256":"97c036645b9b6a78e53e848879a7e3ee08a70ff938ed6e98aa7098b21ab4319c","canonical_id":"671","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[10,22,34,46,58,70,82,94,106,118,130,142,154,166,178,190],"fisher_boundary_sha256":"730eb3d8f69aea1c1666e7307d73b83daf05f241bec613d1628b92a50ec3bcaf","horizon":1,"input_positions":[11,23,35,47,59,71,83,95,107,119,131,143,155,167,179,191],"input_token_ids_sha256":"1100089821e0f1740660a914b8b15a1db0534566177c6e09bcf34f670905693c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[12,24,36,48,60,72,84,96,108,120,132,144,156,168,180,192],"target_token_ids_sha256":"8d49d123f2607a365571c71836e7acda4676f3df261e0a4254afd127a53299ec"},"formatted_content_sha256":"22bd9c0d68e48a76c31dfe254036ce6008c9abd1f71c4c290e76fb4135d09ad0","generator_receipt_sha256":null,"identity_record_sha256":"4cb68a3b9440b00f53a5e655ff7024c46d2ecdc277ae03326b75f41268b173cd","prompt_token_ids_sha256":"63bd2370a08ace97ff440307492d1bee8479e8e14cd40ba9b40b1487a9ad53fa","ruler_category":null,"seed":null,"selection_rank":127,"selection_sha256":"4f4ba031f801028fbf78f774438c1e6ef3c15a58f65dccc230632c90a7fe6b5b","sequence_length":193,"sequence_token_ids_sha256":"d653c8bb051679b00cd66c5370bff1e217b990629e9ca260192942d1541dc892","source_content_sha256":"85295da947527bd6a8cb51dbe0f81aed64a26a245e12c289dab42daa0830ecdf","target_token_ids_sha256":"5233a54b205f9eef3872026a1b044403750ba40d368571e658bd36b909a1a85d","token_span":{"cache_exposed_start":193,"cache_exposed_stop":193,"prefill_start":0,"prefill_stop":89,"scored_start":89,"scored_stop":193},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"797956c3bb60ce0f54962ed315c69368b4d9168e36abfd675d14ccda36e8b2fc","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/36808","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"7e362e252ec4500305135b0f2ef134e2fe129c787fb0a31326c35439cc9ff4fc","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"d0e5311a6d6bde4f201973a4428fa08ba4fa97dc6cf2580fce9e2a4604922291","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"5b2da07174c560f02d4940468e335ad6777df3f471b2be9a3550cae36b1fd62d"},"formatted_content_sha256":"f6e3c61d5723ac6698db28d6198db5b270f51e83e2c8800f5ab2cd181a993a21","generator_receipt_sha256":null,"identity_record_sha256":"e7274663db91b5411d8af35ca6979c6b6038b747658e034486c7161db217a12d","prompt_token_ids_sha256":"5233822ab87654bb7050a9eff9f77a58c8ca4a74c176420316866641587b5ac1","ruler_category":null,"seed":null,"selection_rank":0,"selection_sha256":"0000d64fc1c2791eedccba1b0a014ae30d36a69ee95be69a7d2dc5541d9ff914","sequence_length":2304,"sequence_token_ids_sha256":"5233822ab87654bb7050a9eff9f77a58c8ca4a74c176420316866641587b5ac1","source_content_sha256":"ec7b13f2bb2766533058782bbf50f269e8c0743722581ba193e8dcf47008ada9","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"5ca98bf769dab4ebd4692e9fbb560efcc0f0f474c5a49e0732d605a2683da881","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/32024","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"2b3d247e247806b4dc881d8599ac43f4212a4abfdf27c4a4984cc7b6e286bcde","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"231119f3847a33472d36eeacd144d75fbf1c16d0971ecebd977dec5f84685ed7","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"bcf8ceb1b07052b45eaec59a1762d47a3fd14770e2014af6c7462da3ae912a8b"},"formatted_content_sha256":"056228535dc5d8d08c833c37eb05eb57a524d567c7137e2d8314c181aeca5137","generator_receipt_sha256":null,"identity_record_sha256":"48c93886d09d16bbd4b1c300f10b64a7a5908445969fd866133d7dbd65c13133","prompt_token_ids_sha256":"20d38634a5307135f41953611e0e9d2a8e22a29a1f912d78af0414bcca3e0dc2","ruler_category":null,"seed":null,"selection_rank":1,"selection_sha256":"0002ed283941f3da5106f697d697735d5064de6f22aa84e512fd5fb4792c49a8","sequence_length":2304,"sequence_token_ids_sha256":"20d38634a5307135f41953611e0e9d2a8e22a29a1f912d78af0414bcca3e0dc2","source_content_sha256":"eb06d835db68d2ccee63052eac9d025e120ca9331c84186892dbfca0b8c7b5e9","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"870cf6deead4c720c1131fda048c46f654bb50af3e403058e9370e06d21978fa","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/20279","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"9a2ec9cafe47a0a911539d7cf75996655cafccb2071b17063c4d74a9a215b6bf","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"b6c9a50b23af8aa52a00688ce9f3e2f1b86afb30dba305513f107d56499b7109","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"484ad2459b1ac7e91b8a406d092192990e1eb1f2ef4295cbc9ae9b0d8db832fe"},"formatted_content_sha256":"ad22fff4026303176b09131a60d9a8a6d1a6385ccaa59768e5f6a72ff8117f2b","generator_receipt_sha256":null,"identity_record_sha256":"01c99745bb7d41ee57ea15450fc853f25c79f09d2f8bc063c0bdd3bb66dcc3da","prompt_token_ids_sha256":"aa29883de5d02657b8eddaf1e1caece900a2d6c56fcf63a868f35f285132dee9","ruler_category":null,"seed":null,"selection_rank":2,"selection_sha256":"000d6a7d93865e2955a912d7fcafdf0e79cc24fdca9b059114ada982813dedf6","sequence_length":2304,"sequence_token_ids_sha256":"aa29883de5d02657b8eddaf1e1caece900a2d6c56fcf63a868f35f285132dee9","source_content_sha256":"b899a1b6c5b3ba61252a07cb53ea7d0f15f0c63315df7b872a9c7cacc58017fd","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"c3a77259d6b11a251eaddefe696ec8873dac79b38e498e4c19008e424264366d","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/35786","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"049cec2caddadd6c5a22d5eebe0f37af9a7d660e1f8a0134c85404ef60e34076","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"34af22d248242c1ad9044c3b1cc5988e257371a5867a4bc4f686e53596cf32a7","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"e400ca0c9462a4d8e94a16cb560f6a54f948fead3ee81377ae34f4b435d27bf6"},"formatted_content_sha256":"7e0c728729a3f6e258bf44c314097e48e5bd7a3d2ed6a9f7c33b0dc333de0830","generator_receipt_sha256":null,"identity_record_sha256":"182e8520c6e45d8da1ae7d54afd83292ddcbb68a76cccbeb96d22ffdde1b9464","prompt_token_ids_sha256":"a074a6e0e6624af64b673167b490ce4de6ff1fc5bc6a5a0429a482706d5047aa","ruler_category":null,"seed":null,"selection_rank":3,"selection_sha256":"000f8dbba0aa87b15dcf3bb9e900e6a1c95b96affc23284c0aad2871def7d41d","sequence_length":2304,"sequence_token_ids_sha256":"a074a6e0e6624af64b673167b490ce4de6ff1fc5bc6a5a0429a482706d5047aa","source_content_sha256":"d99d106567585d3e0dbdbd545794876258507c655b4b92af1dd705f6e4c984ce","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"6e96a1becbdeb74006e814bfe6a5fb240a5de9f6134d609b13117dbe7211fb78","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/29413","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"f68c33e180124100446c92def52974f274452469befeff1421363223dfd67323","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"9c4a80f3b25c53840a44de74f52c06907a16b90bcc7a8c077dc4eea1fccba752","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"85a930c5f1ed93345b9259991c2d9c7a90faace00b3ca7248e61b28fbbf27372"},"formatted_content_sha256":"aa37e0852c2a9d938e44d1f6050bbe8db2ca60ee6d89c2e40ed5ddc5f9c36a77","generator_receipt_sha256":null,"identity_record_sha256":"c8f11e6e9703b0eb65e3a6f5f79135730de277373c9e3a94e68eab94f5054d7b","prompt_token_ids_sha256":"5f2026b82faac8988768fc7330d60a392741c3a5f7b364d0b87bf8d596e74bdb","ruler_category":null,"seed":null,"selection_rank":4,"selection_sha256":"001145473f2b3b765745639cd14dac4b421dffa50f63df360f9059d362dc9b5a","sequence_length":2304,"sequence_token_ids_sha256":"5f2026b82faac8988768fc7330d60a392741c3a5f7b364d0b87bf8d596e74bdb","source_content_sha256":"12b0ce7832facf193492ad1f568ff0a0b25bc5e52381a9690df089bc8a9b1db8","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0244917a01282995b95e9ac9304e280f4253bfbab1626b6c1c57bf69a558b519","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/24903","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"d50289a8f2b5eec9977321d118785e94090fabd5a2f4938b5e6f1c63045b237e","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"ee472d2e8db03638388106e721863599f643316a3422a5245d0628a1edb69010","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"4b9781c37044a00962c0018332d1f3f966c9e3b9ff49f267b124cbe927314707"},"formatted_content_sha256":"32ac15efb7073af0ae11d3032d1da75babff6e7bd6406334879047edbd6e821a","generator_receipt_sha256":null,"identity_record_sha256":"b26e48a4638561a3f7f57dc2f6915accfeabce22cb26f87ddd65919ddf7a653f","prompt_token_ids_sha256":"7b786394510a818db1c521f6d71eb3b4052359b0d28d0a57d166057d80326905","ruler_category":null,"seed":null,"selection_rank":5,"selection_sha256":"001280ff6af9eba470ab76ff48bc2983cffe1995efb051d68cffd40cbaac4d3e","sequence_length":2304,"sequence_token_ids_sha256":"7b786394510a818db1c521f6d71eb3b4052359b0d28d0a57d166057d80326905","source_content_sha256":"14074cbd779ec8e06e0e5e91c6c6e6c02ebe2d185bb5310ab0dfdf64230019d7","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"67b1ede620a9bdb86c619e3124eef21332be86df3795cb4169fdf93c1bb4c46c","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/10609","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"51f6c6102f8fa772b2f8e1665ab91ad76a5f4f11205009bb3a8af2e9c9d4c06e","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"3d06c4debf0e8c528b85d65d243bbe561bcd13afdcb83d09b2d604fed1853965","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"9bdcf5cea67426ee4ade841fed2e6833ccbede63b7b5497449fa2279f5e54182"},"formatted_content_sha256":"8639254a29be3154bbc110c673738678316be09fe6f456878dac249bdeb1f4e0","generator_receipt_sha256":null,"identity_record_sha256":"a21cb3bd2e34a22b91569c2942d19d8bafaed2f11827ee9c58af3cbe366d479a","prompt_token_ids_sha256":"0697f0eea8df7d85bbeb6a688ab15d4294212694298ccd864b429ddac3ed7a4e","ruler_category":null,"seed":null,"selection_rank":6,"selection_sha256":"00131239a626272348186654735088a9b4c813c3bd83827262f7891e6056d43f","sequence_length":2304,"sequence_token_ids_sha256":"0697f0eea8df7d85bbeb6a688ab15d4294212694298ccd864b429ddac3ed7a4e","source_content_sha256":"49ff1f749aab76c1df7c47e6c79fdee9dd3fd191d073320f0ab1f5f361059fc3","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0d6376f0009c55e4f4efbf0d650cfe974856c049c8b14c303c991dd3f6c05c19","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/39447","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"7914f39477c00d3c1516c2e43c441114314ef252fe82526ad293b346ab359284","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"7d6865c318074116ceae5d7735c27fed5066ae326d49cbc3280f9f0289cfb7cc","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"ec95c049c76add2ddc5fa9d5301e2bddb64906cc8e42e8438a5c2aeeffb46499"},"formatted_content_sha256":"033b6ca5d9572a314560961826373bc34657c08759a1919296e81a4cb84c6471","generator_receipt_sha256":null,"identity_record_sha256":"f59cda6a20ec53af40493bf6be8ad0f898023c32531b81fe0ff0fb3b13340ea9","prompt_token_ids_sha256":"81890a7b8319487c5f4c312ca211202afd2d039e46e4931b65e09c6e446fa634","ruler_category":null,"seed":null,"selection_rank":7,"selection_sha256":"00166693ce38f4c5185c40c1258a1a602a290506112c351fa067daa039c2d960","sequence_length":2304,"sequence_token_ids_sha256":"81890a7b8319487c5f4c312ca211202afd2d039e46e4931b65e09c6e446fa634","source_content_sha256":"2d19f7657219986df6eb9bcf188544ab7660f95cb5521668082b48da38ac52bf","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"6f5f79cc3c1180f2cf00501aeafbc8d0f1dd6e6e7ccc69a8648c032e3469df45","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/33622","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"9b4c1f8e6fd129beb4bc474e6514815a95af657b4e2bc5fe7a7f32362dfb40bb","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"dcd27f6003c74d9c8a85711934161ee0f2636d2f0d27c4137e156f0d5064649e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"536e8bc4cb2331a406028d335233022e273cb3c563ac84e3db83762692164b14"},"formatted_content_sha256":"de944eeb38c1a6f2dfea1cb1657adb820f3d66ebe3aa361c72a39823aad3fa73","generator_receipt_sha256":null,"identity_record_sha256":"344ad881a15a994684e1ad39c738fc64b472325c7c46c63c837f255455d9f5e0","prompt_token_ids_sha256":"93be74d13c5df9550e8923cb9af9086891766e59060bc9997c00a1c14458302a","ruler_category":null,"seed":null,"selection_rank":8,"selection_sha256":"00191187cdc6cab1097f558bd88f30baeb7d1f08e9eaffd47828d8a1345ed53b","sequence_length":2304,"sequence_token_ids_sha256":"93be74d13c5df9550e8923cb9af9086891766e59060bc9997c00a1c14458302a","source_content_sha256":"dcf794cb44d57515b528594025b879dcacdc8e06e892b1a0e62d26319aa8d8d7","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"f602f879f02588bab48170a79cd2e407d35c19e5a4e05337a2615b62c1d0c64f","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/28065","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"3ab0b82eda5a33b4272654c9de3e352bda591b27e5499e28d3642e5a138ec7f8","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"f29080ea5125a83d114344a4897dcb5e6cf6135a3d28a1b55374cdb2836dd1db","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"34dd2e61b1ba68a0bd87b1f9e8650c6869d66f35aa98c549feab516d17c3afd8"},"formatted_content_sha256":"f500fe5cd15c60256f64965b23d5784b6badce6cc0da7c4792ac3017524b173a","generator_receipt_sha256":null,"identity_record_sha256":"73be35853dc6427eeab216a511cbdeba7dbfbab63feb81bd2642b1d99bbe5f6e","prompt_token_ids_sha256":"1c701cd7ff6691ab6d74de0874349dc37eac3d7dd6450cdb43b876a5aa7ea76b","ruler_category":null,"seed":null,"selection_rank":9,"selection_sha256":"001ac4d7e72067b598bc36c7582913c54df342db338498dd73a106eec9c8ffb6","sequence_length":2304,"sequence_token_ids_sha256":"1c701cd7ff6691ab6d74de0874349dc37eac3d7dd6450cdb43b876a5aa7ea76b","source_content_sha256":"0beeb0285771e4bc2cee42c31f1b35d1570cda724294b426b193cdac54b83393","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0d2186f26771b127e5a2350edc81554ef21f6eb9da3e69d77f7377a307b8a2ec","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/1253","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"4db1f2865c58e3a6faa8055be8d46b6c3f57be277f07fb5e39670a879cc599df","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"e20502727d9945c19d9f78c314ab8a1be19622e5a4316dd86e9823225d38135d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"be880e0cd3b16a5b9d737b8b2b7cd71353c9aad69f7dd469e024c5803d69f411"},"formatted_content_sha256":"a6e5b442be8b8fd1b45487216940754b214f379ae9a2021d30f82598049ca5f2","generator_receipt_sha256":null,"identity_record_sha256":"02492d4c2ba2a4f9435fd461ed91ea404250adb616636b5c5f6930fc4c350595","prompt_token_ids_sha256":"c55c7896671653ced092d5fc45ea5527e9457780b57375667e77dd586f161da4","ruler_category":null,"seed":null,"selection_rank":10,"selection_sha256":"00223684c9261647b075a573c94999d1d273100f31524ec62c5e6c73b4787787","sequence_length":2304,"sequence_token_ids_sha256":"c55c7896671653ced092d5fc45ea5527e9457780b57375667e77dd586f161da4","source_content_sha256":"6d1b9548166f4f9d82806a3af5ceb3ef3a1f644c2c946deb8aa86ec78d2c2fe5","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2608f932273743963113f1c10bf0e1c74703b9252284d4ab16394eea6ee00a1e","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/17774","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"ea92971f2ed12679772d304c1af6ce6e5d20a0a07f9c6e29ab3b8b95eb0c9c7a","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"de2670faad91b518e2bdc4f93edd92e3cf3b6d9e12fe24803a9dbb2d6ee7d8c3","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"d1ca5e91572227a6a1cdbeac2c3e0023a33d11172238246640c98d613aeb61a8"},"formatted_content_sha256":"2a4bbc348f9010bdfc1eec2d9ac77a487ad6f51468cfdedc35c10316e95564da","generator_receipt_sha256":null,"identity_record_sha256":"aba94b34901cf76209e4f0517738794062a4e8978f75d8d8e46bd007b32f2d61","prompt_token_ids_sha256":"25656de4135fa58df4d27bf5e1bb9122210cd21754c03b8b5f83f9412a6e2f5e","ruler_category":null,"seed":null,"selection_rank":11,"selection_sha256":"002494cd7b130c2e8101df6704ee2628630640a6ea6d8c21e9a0419c810d8f25","sequence_length":2304,"sequence_token_ids_sha256":"25656de4135fa58df4d27bf5e1bb9122210cd21754c03b8b5f83f9412a6e2f5e","source_content_sha256":"c71f6b5999e91ecf1857fb9dfb3f2573827e4ae4c5c3c4af2b08e3ba6a349e07","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"112223ea4e8c7a17eec0768b0bef76c663105f97785440668c062c5080b3859e","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/26008","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"2eb3fc663ac2fcca42c750d5cba63ad3d0d118921b38a48382c1cac4734faa8b","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"ecda06dc336cf17705f48dcde65a60f0b7ff432af083c07918dd0f1dbf09d79c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"e7eb1385302da9031a1227b768aaabecafafdb5614b3a0e2873e45ccfa3f7325"},"formatted_content_sha256":"fb608858dedf19c2f0a8e14d8710aa1d1c393be5a72ac8554731f0af33714904","generator_receipt_sha256":null,"identity_record_sha256":"af695a09abd9495d4e87989cbe7b352aebdfea88272863b3cade1adbd9d9c1a1","prompt_token_ids_sha256":"8eac346c569d9163ec8a9bde2355446690ca3c72a9a9f5c9bc00b745b6648f63","ruler_category":null,"seed":null,"selection_rank":12,"selection_sha256":"0027603c9d271c580c43673b3431be3adf061650ddff27e01b7869ea18673d9b","sequence_length":2304,"sequence_token_ids_sha256":"8eac346c569d9163ec8a9bde2355446690ca3c72a9a9f5c9bc00b745b6648f63","source_content_sha256":"469e817fab0a0502150988f96b4b16d5c4de03fb7f7bd185e1cff0715aee4b26","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"c447c8e41acc4393a51daf3ef28ed95971628266812cc78c05b887e341a38f02","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/38160","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"849f6474e4eb97b56439c66b9aaf2ac1fca275e49c29e33c48b2de0e806d1193","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"d11cb3a40f35a5034111c7bbbf1a8ff895d15396fea10ddb0cdbd97e8a47a5bb","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"ae53490b8b58ad6da63a300819db55d4a14e7b1bf25c07bbcd54b06469cc8b62"},"formatted_content_sha256":"b87156fd6d95b79ffe144b1b7448682cc84e8c2d06709db21214d82439b58315","generator_receipt_sha256":null,"identity_record_sha256":"f629ef55f22acb269a1825e56238e5770bf1bffc327d1d71d03370911d984632","prompt_token_ids_sha256":"55fb1408407a280bc2c65e78161f5cefb0ceaa4ceb04073817e0973d6a982b53","ruler_category":null,"seed":null,"selection_rank":13,"selection_sha256":"002a54c1b2c80d534795a77a68dfa9f3411107c3896ba33d31a868339d813293","sequence_length":2304,"sequence_token_ids_sha256":"55fb1408407a280bc2c65e78161f5cefb0ceaa4ceb04073817e0973d6a982b53","source_content_sha256":"f59885bfc6d88898f97c7bda7625006b3be0651da5929a1e272907e85a9772e3","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"14e443d1ba9ca076a4a9d065c90f0b3ccb2c24defaced1591fe7cc77f71e7527","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/2773","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"5a0b47a49833f2e6114f9b3e2d475b69a889ed7945db09507580c9536ef76448","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"f0dbd8116db99645f9271fcd040a856c59b111c6243748f64ddc1422928c64ff","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"78a78a32fdab2f986b289b32d0a4612fb33ea2014dccbbce3a60d65c16e765ac"},"formatted_content_sha256":"68c11a24ba674cdac7966cbaf0f71a9d2fc30a62a0d90a649a6554907ef00125","generator_receipt_sha256":null,"identity_record_sha256":"ddbf6728a3fc2c07defaa4546c34ea401f4fd05c829819301903562cd04a9519","prompt_token_ids_sha256":"8b935a7676a8137367066b0b1af7a39f635e0e3592fe3c1982a28bac24eb96cc","ruler_category":null,"seed":null,"selection_rank":14,"selection_sha256":"002d0ca3ff81dfc79dea1f54c4f28547f367149283e13dc83a14c114322353bd","sequence_length":2304,"sequence_token_ids_sha256":"8b935a7676a8137367066b0b1af7a39f635e0e3592fe3c1982a28bac24eb96cc","source_content_sha256":"be1775efe2cd9be13c65cbd65200b0cd27ae7d29b52ff493b27ed85e4b549c61","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1fb610bd6210663f10a294d4b45df223b7ee4e3b6a8c0de390f3e50612ea1219","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/1219","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"92e822869a8cdca6f8c09e9495406404c8a345c0a99d06dc98a948b7bcd010d6","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"7ea3f80cfbc818f9210b87b0153e90af5720869a1fe3f26b0d5f723bfa9eead2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"c1725ca6a384a182c8f164ec35f4faded719127f4cef2baec3d812545eff3e6f"},"formatted_content_sha256":"cd30711364e668a3275113e8a3f92ecee4b268935f5a095cdb0f0c1f11205c1c","generator_receipt_sha256":null,"identity_record_sha256":"2d205b87fc59eabeed0bda29df3e0789eef230026c46ad11796d41b866d18b2e","prompt_token_ids_sha256":"65e443512311bce75a506b694195aa105b7831541ecb92c1768fe5a0ce285ef2","ruler_category":null,"seed":null,"selection_rank":15,"selection_sha256":"0032c753604107e647b4ccd7ef877562bc0ca9c0e232a26a761388befd002fea","sequence_length":2304,"sequence_token_ids_sha256":"65e443512311bce75a506b694195aa105b7831541ecb92c1768fe5a0ce285ef2","source_content_sha256":"6cb2a106560ad166c025fb7a83708e519f95bca68cecb9a7bff61e261a0b8384","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"52b78dddc7c06fdc5e5ab3937802742ef3b3414f9743427d2bd71994c5cb1671","anchor_positions":[113,227,341,456,570,684,798,913,1027,1141,1255,1370,1484,1598,1712,1827],"anchor_positions_sha256":"9289ece2ebec2ce8ec20865a50beecbecea8ea6a3dc4b5c5c8ed9f6ad0d784ac","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multikey_2:length=2048:seed=12340:sample=0","config":"niah_multikey_2","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[113,227,341,455,569,683,797,912,1026,1140,1254,1368,1482,1596,1710,1825],"fisher_boundary_sha256":"a9016c04cfbd31eb6f72106eb0693c1545bbaa0451e228e6e447a7814d921eb7","horizon":1,"input_positions":[114,228,342,456,570,684,798,913,1027,1141,1255,1369,1483,1597,1711,1826],"input_token_ids_sha256":"dfd87b8cdcdc45c3aed78576c14b5dd53c5f3a8de7a4a69358857943c1f8ba1a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[115,229,343,457,571,685,799,914,1028,1142,1256,1370,1484,1598,1712,1827],"target_token_ids_sha256":"4ed30d7e42a7866174c94f532bdcc6947a15ad58245c18f3aba71a4612f05990"},"formatted_content_sha256":"d1e07461677a5fe92f27926505552cc8a343cd60a5b5be86c2fa00090fae27f0","generator_receipt_sha256":"801e44bdb609773d985e8fd74f1d853f1afb30cf0a66d2cee0f7013ae8749c9c","identity_record_sha256":"a1cb4798036f5d2c63f1da08e09141e23d4fdec224e854eeb2d770ac8ba11721","prompt_token_ids_sha256":"793e38d71d7e91af366df94fb359ffb203e20cafb77f7afbb412db3ac5cfc5d2","ruler_category":"retrieval","seed":12340,"selection_rank":0,"selection_sha256":"26538bdeb4a99f8904324b26add0826a13744b51ec94fd9a004c72a2ba68c287","sequence_length":1828,"sequence_token_ids_sha256":"793e38d71d7e91af366df94fb359ffb203e20cafb77f7afbb412db3ac5cfc5d2","source_content_sha256":"801e44bdb609773d985e8fd74f1d853f1afb30cf0a66d2cee0f7013ae8749c9c","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1828,"cache_exposed_stop":1828,"prefill_start":0,"prefill_stop":1828,"scored_start":1828,"scored_stop":1828},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"75123434e18fce7bfe0e4b995fd12769a05a08d35fc315c68b8438e286e23f77","anchor_positions":[246,494,742,990,1237,1485,1733,1981,2228,2476,2724,2972,3219,3467,3715,3963],"anchor_positions_sha256":"dff504ff5227ef8cb24123e368d13dfb8cef71f15a3e715472122831820e3e93","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multivalue:length=4096:seed=12340:sample=0","config":"niah_multivalue","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[246,494,741,989,1237,1484,1732,1980,2227,2475,2722,2970,3218,3465,3713,3961],"fisher_boundary_sha256":"b36b4cfcb1258559a6eeafb235f1efb9a3bc5d2d3ec31389b39290f7e7209d82","horizon":1,"input_positions":[247,495,742,990,1238,1485,1733,1981,2228,2476,2723,2971,3219,3466,3714,3962],"input_token_ids_sha256":"dd9ed3a5d8b1bac29e9c251660e29d7a61b1d0cbb9846d333768dd85128cd708","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[248,496,743,991,1239,1486,1734,1982,2229,2477,2724,2972,3220,3467,3715,3963],"target_token_ids_sha256":"80f466fa56b4c9b1c24b8b51bb8cacb08c7290a7b0f1fd9ceeb119534231b14d"},"formatted_content_sha256":"f6bd8b2f8c5eb25a8ea8e08f2f4fb8e581a8d2080ff40194bce0a86f5deccc0e","generator_receipt_sha256":"fc75f4bdae8929325589536065fb7c8f5bb1fd737001da616d339365aa94920a","identity_record_sha256":"8091f9201ab1213aa14972869be942841763c3897bee5b30b77fb3d34877dd86","prompt_token_ids_sha256":"d498355b8380354e768abc57a79afe8cc73c5849b98f01081fd945329db5e77a","ruler_category":"retrieval","seed":12340,"selection_rank":1,"selection_sha256":"34e79c93add91edaaf2f631ffbc9e4c5350f61dc96769cf971f03ae113ccd1b5","sequence_length":3964,"sequence_token_ids_sha256":"d498355b8380354e768abc57a79afe8cc73c5849b98f01081fd945329db5e77a","source_content_sha256":"fc75f4bdae8929325589536065fb7c8f5bb1fd737001da616d339365aa94920a","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":3964,"cache_exposed_stop":3964,"prefill_start":0,"prefill_stop":3964,"scored_start":3964,"scored_stop":3964},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"532ac060a0cb86e868a3d0d5f0e7217475df142f3480655a5f9cc923addbf97e","anchor_positions":[124,250,376,501,627,753,878,1004,1130,1255,1381,1507,1632,1758,1884,2010],"anchor_positions_sha256":"baf07dbe6e81bfe17433570a4cb75b5ed267ca19947bc93119fe6c078f612806","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=2048:seed=12339:sample=0","config":"vt","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[124,250,375,501,626,752,877,1003,1129,1254,1380,1505,1631,1756,1882,2008],"fisher_boundary_sha256":"9a1bd7b90a51362eb876693f97e6c20ec0fa6bff1077ed5acd1237a823804b3a","horizon":1,"input_positions":[125,251,376,502,627,753,878,1004,1130,1255,1381,1506,1632,1757,1883,2009],"input_token_ids_sha256":"d43a3eb571f5bfc8ea7dc04191b8336f22f066f154918abe65e057791d13a43e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[126,252,377,503,628,754,879,1005,1131,1256,1382,1507,1633,1758,1884,2010],"target_token_ids_sha256":"453114002e40922d3386b14ea8e4915c4aa872297dae554dfbf91fe2f3b3e9ec"},"formatted_content_sha256":"221e10f0973c46ef1f20a688046e4bffae24656998c1059a831d7b20726fede0","generator_receipt_sha256":"b9a9b27500644e9aa2532304d577b85ab988c4359dd5edf7630fc8a8bee4b60e","identity_record_sha256":"3c519c0d4df294b9bb4d57dfaf44f3fe4934ae8e697dc924d17dbf39644026e8","prompt_token_ids_sha256":"cd95e7b2f95f4b705b1b43dc6b201b8307b182fe379566cb2e900ef21733b21a","ruler_category":"multi_hop_tracing","seed":12339,"selection_rank":2,"selection_sha256":"48145a1ddcbd5ae2492d753c9ffdf69047346dcba3d4c88e131c036be6968d9e","sequence_length":2011,"sequence_token_ids_sha256":"cd95e7b2f95f4b705b1b43dc6b201b8307b182fe379566cb2e900ef21733b21a","source_content_sha256":"b9a9b27500644e9aa2532304d577b85ab988c4359dd5edf7630fc8a8bee4b60e","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2011,"cache_exposed_stop":2011,"prefill_start":0,"prefill_stop":2011,"scored_start":2011,"scored_stop":2011},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"deb02a9e929ebd4d409deb018ab87063b47401d9993356ae1689a17e44d9121f","anchor_positions":[248,497,746,995,1244,1494,1743,1992,2241,2490,2740,2989,3238,3487,3736,3986],"anchor_positions_sha256":"a5ad1bc8d8afa780351af0b99deecbf3b31c05390a133ecb9438c32584b13b51","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:fwe:length=4096:seed=12339:sample=0","config":"fwe","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[248,497,746,995,1244,1493,1742,1991,2240,2489,2738,2987,3236,3485,3734,3984],"fisher_boundary_sha256":"3ce3ab1d00aff5ae9a8f93284d14aaa6f1744a79acc88d0772877941983042b6","horizon":1,"input_positions":[249,498,747,996,1245,1494,1743,1992,2241,2490,2739,2988,3237,3486,3735,3985],"input_token_ids_sha256":"87a331f7d1b5d5467dcce6012238bb065bb86881942f6c071315f3911a04e910","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[250,499,748,997,1246,1495,1744,1993,2242,2491,2740,2989,3238,3487,3736,3986],"target_token_ids_sha256":"0c3be397f087a6670d953bee273680288fc932be2e83d55d3086f5f6090398bb"},"formatted_content_sha256":"0fead2f4a48fe6c038641241dc6133e35cf0290088d4b06bf388bbf505180d9d","generator_receipt_sha256":"7fe1d0a912ee13f2dc1aae6839a7f8e87a623d5df30a3408ac5c6059f88aed13","identity_record_sha256":"bef04af160238768f27800f450d76a76013d80a3c96282a8a684900ba5807d4d","prompt_token_ids_sha256":"1610d294ca00f9745a960e213dd10784d8a523eca88bdaded2c7bde663d09a1c","ruler_category":"aggregation","seed":12339,"selection_rank":3,"selection_sha256":"51065012a292ff5d1577ac8988306f3ebc15a9c955ac9c104f135c18f7ed7cc9","sequence_length":3987,"sequence_token_ids_sha256":"1610d294ca00f9745a960e213dd10784d8a523eca88bdaded2c7bde663d09a1c","source_content_sha256":"7fe1d0a912ee13f2dc1aae6839a7f8e87a623d5df30a3408ac5c6059f88aed13","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":3987,"cache_exposed_stop":3987,"prefill_start":0,"prefill_stop":3987,"scored_start":3987,"scored_stop":3987},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"8a6b31f4f399feceed15eefe68a896dcd67ad7e19c3225837e50ada41fc0a847","anchor_positions":[252,506,759,1013,1266,1520,1773,2027,2280,2534,2787,3041,3294,3548,3801,4055],"anchor_positions_sha256":"e3a8a2871a86f2cf789e76ba6df6ede1df61047b14b69480ff1217c0a911147f","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=4096:seed=12339:sample=0","config":"vt","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[252,505,759,1012,1265,1519,1772,2026,2279,2532,2786,3039,3292,3546,3799,4053],"fisher_boundary_sha256":"159f8ad58c26093c2f75744da0f9502f7a2906b033bd5fce2665cf525fefb38b","horizon":1,"input_positions":[253,506,760,1013,1266,1520,1773,2027,2280,2533,2787,3040,3293,3547,3800,4054],"input_token_ids_sha256":"e8a2ff60bb805b206d15a2e0bf12885fedc537e7082f756e2c445a6bf70c326d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[254,507,761,1014,1267,1521,1774,2028,2281,2534,2788,3041,3294,3548,3801,4055],"target_token_ids_sha256":"49fc01abb0da9ff437c1e5a9aefc9158a1f8db110f275f2946581b8be9491118"},"formatted_content_sha256":"946e2f3a8214ecc488a65888ebeb1dc5b42d3027357523816a5ce8d3414b899c","generator_receipt_sha256":"8ea9a21c91b1b5203697a120d21824bea8dc810455d228e58a7ae319dc72dd82","identity_record_sha256":"5556eeb12c7c154e817d9c46859eaeb64f6a54512b787f1b1f38bf9fcf284542","prompt_token_ids_sha256":"9973907d38c28c89e9a53bec04ecb4c695cc4c94f35050f705a22d2e6ab2c897","ruler_category":"multi_hop_tracing","seed":12339,"selection_rank":4,"selection_sha256":"5c5a0c61d80b5c3a76e5bbf1062b35594e4a4995d482c429634e381eaaadcf21","sequence_length":4056,"sequence_token_ids_sha256":"9973907d38c28c89e9a53bec04ecb4c695cc4c94f35050f705a22d2e6ab2c897","source_content_sha256":"8ea9a21c91b1b5203697a120d21824bea8dc810455d228e58a7ae319dc72dd82","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":4056,"cache_exposed_stop":4056,"prefill_start":0,"prefill_stop":4056,"scored_start":4056,"scored_stop":4056},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e6965757239ac98f811d9377515f2e5110de3926f0390ca3416c9f2e3ef74bd3","anchor_positions":[79,160,240,321,401,482,562,643,724,804,885,965,1046,1126,1207,1288],"anchor_positions_sha256":"a02a7b889641d261dbd8927a54b0e9bda873159838da4973760162d67a2adbcd","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multiquery:length=2048:seed=12339:sample=0","config":"niah_multiquery","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[79,159,240,320,401,481,562,642,722,803,883,964,1044,1125,1205,1286],"fisher_boundary_sha256":"57d7fe986325f9a24836dcf6f4afdba3cd327289b6aa8077b19ec4aaff566ce9","horizon":1,"input_positions":[80,160,241,321,402,482,563,643,723,804,884,965,1045,1126,1206,1287],"input_token_ids_sha256":"0a50d970537d979d05cfd20781db749fecbfb1eb42f1426875d990cadb4a07ee","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[81,161,242,322,403,483,564,644,724,805,885,966,1046,1127,1207,1288],"target_token_ids_sha256":"88866037b9a7485b03068ce42d898b0e5484251bc087329d421a95fea8993da1"},"formatted_content_sha256":"bfb4cc6fdedde1a2578a09daf8fa4e779725a810722080b59de46588e1d5ab2b","generator_receipt_sha256":"0e8034964c18899e91a435a5737bfa29f7791fc41c29f42e4f2d9d75fa3a98ff","identity_record_sha256":"6bb7156902c5f36fc7c3e3c92666c510e1a2d572d3c6c557cc98ab440dd94db2","prompt_token_ids_sha256":"c68672f5e746622c7ef9c8d45f9312057365f04182c7af0576723d8089244ca1","ruler_category":"retrieval","seed":12339,"selection_rank":5,"selection_sha256":"65f813250e251ff50bf38bf6b227abf56f6e43ea5dedef6b18e6cf94874e510d","sequence_length":1289,"sequence_token_ids_sha256":"c68672f5e746622c7ef9c8d45f9312057365f04182c7af0576723d8089244ca1","source_content_sha256":"0e8034964c18899e91a435a5737bfa29f7791fc41c29f42e4f2d9d75fa3a98ff","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1289,"cache_exposed_stop":1289,"prefill_start":0,"prefill_stop":1289,"scored_start":1289,"scored_stop":1289},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"128a207210316772af37d86a91c97448a5eec69418db26c573a15daa57fff7db","anchor_positions":[252,506,759,1013,1266,1520,1773,2027,2281,2534,2788,3041,3295,3548,3802,4056],"anchor_positions_sha256":"ab52ebab779359ace36255ddbc6dfccb71733c90923120903537310d833e1be5","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=4096:seed=12340:sample=0","config":"vt","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[252,505,759,1012,1266,1519,1773,2026,2279,2533,2786,3040,3293,3547,3800,4054],"fisher_boundary_sha256":"c02daef8e0847ca8ef8f6795ed064d750cbfb33070533ceaf84d3f199f95bdf4","horizon":1,"input_positions":[253,506,760,1013,1267,1520,1774,2027,2280,2534,2787,3041,3294,3548,3801,4055],"input_token_ids_sha256":"758c575c40dc855f729daac2fb4c427f22776d59517e67e7dcd8ad13b9015cd3","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[254,507,761,1014,1268,1521,1775,2028,2281,2535,2788,3042,3295,3549,3802,4056],"target_token_ids_sha256":"e7ee891d8494cf6174e6a03c1da26e912c52473297bcba994663141abda1e73e"},"formatted_content_sha256":"84db9b2d31c7c5cfb2df3227d24ed8fa03d50cc474dbaf560e4f7aee2e52bb97","generator_receipt_sha256":"5b6cd316a368db4420845994f9423aaeedc0fd75802aba9eafb7deb4f4eddc73","identity_record_sha256":"ce937620a8ea8bbf5af1e1b522055853a1ec44ac92520d74486cc8e8f0bae348","prompt_token_ids_sha256":"42167ea36f4f43f21c1eadf1bd9fb391cb7d16904f553e91eac1401c360e902c","ruler_category":"multi_hop_tracing","seed":12340,"selection_rank":6,"selection_sha256":"79c3cb58b42a4b44fb604b28cd22ae1aa71bc49e05a3eb8399ca698c0044ee51","sequence_length":4057,"sequence_token_ids_sha256":"42167ea36f4f43f21c1eadf1bd9fb391cb7d16904f553e91eac1401c360e902c","source_content_sha256":"5b6cd316a368db4420845994f9423aaeedc0fd75802aba9eafb7deb4f4eddc73","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":4057,"cache_exposed_stop":4057,"prefill_start":0,"prefill_stop":4057,"scored_start":4057,"scored_stop":4057},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0fe15004ffd8e39b3cd00a623cb86c9ae2e0570216811c075442338f6c8c6963","anchor_positions":[141,284,426,569,711,854,996,1139,1282,1424,1567,1709,1852,1994,2137,2280],"anchor_positions_sha256":"9ea473fa5cae3aef9908a7ca4fa168866d469f3650f18f07e5dce4ee02714d7c","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_2:length=4096:seed=12340:sample=0","config":"qa_2","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[141,283,426,568,711,853,996,1138,1280,1423,1565,1708,1850,1993,2135,2278],"fisher_boundary_sha256":"12c58ae371c707d325c0b7db5868000c046d213a4b92b2344f6482266e812bec","horizon":1,"input_positions":[142,284,427,569,712,854,997,1139,1281,1424,1566,1709,1851,1994,2136,2279],"input_token_ids_sha256":"e6b8490ddd4e543ee4bbe0a2096b3a34397a5c6e4ee16c9efb438fba25a387d5","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[143,285,428,570,713,855,998,1140,1282,1425,1567,1710,1852,1995,2137,2280],"target_token_ids_sha256":"644343be1688f80c9846724beff4e6d47423e4ced93613951f85b480fb1aa6bf"},"formatted_content_sha256":"1d3faed8d004df01dbd7309b8dd1ed0e326c1bd1c5fa54306abe22855487372d","generator_receipt_sha256":"fba640bf322cb8c1f6175cc1a73e6267b18618e8555d9601e202268427ebabe8","identity_record_sha256":"1e32be0f57d7bfa1580c73b4f1c5e8a06ff8f429ff4611e766534940463ad32e","prompt_token_ids_sha256":"dab144649219fd1b15323997ba14f284492db0e725562192c789c0f766c8de3c","ruler_category":"question_answering","seed":12340,"selection_rank":7,"selection_sha256":"7a527495d7e22511d3058e2b2a4a846cd8b244d93a021ee51622ee975c2530ea","sequence_length":2281,"sequence_token_ids_sha256":"dab144649219fd1b15323997ba14f284492db0e725562192c789c0f766c8de3c","source_content_sha256":"fba640bf322cb8c1f6175cc1a73e6267b18618e8555d9601e202268427ebabe8","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2281,"cache_exposed_stop":2281,"prefill_start":0,"prefill_stop":2281,"scored_start":2281,"scored_stop":2281},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4a35e619dc657f107bad1efd0356762a6276d17f1c44fe7c843454ce15350f14","anchor_positions":[84,170,255,341,426,512,597,683,768,854,939,1025,1110,1196,1281,1367],"anchor_positions_sha256":"52762b5ae990ebf341f9864370415ddc0be98c34b1415f0fb53c8f4bb5237780","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_1:length=2048:seed=12339:sample=0","config":"qa_1","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[84,169,255,340,425,511,596,682,767,852,938,1023,1108,1194,1279,1365],"fisher_boundary_sha256":"71d2d40de65a4fd2d9d9e691302e52d3a9fc059b1629b02c3b6543f8201e463f","horizon":1,"input_positions":[85,170,256,341,426,512,597,683,768,853,939,1024,1109,1195,1280,1366],"input_token_ids_sha256":"673327b5691ed16299c8dd34b55f88b87d1fe642535708588295ca0f00f89422","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[86,171,257,342,427,513,598,684,769,854,940,1025,1110,1196,1281,1367],"target_token_ids_sha256":"aefebb06df48a7ba94a6d5de230228ea289d46553d344edc66924e98dea4b336"},"formatted_content_sha256":"9463b4927878ab07df8a32bd0b0584b76f5d42182dc21d90193275757cadc022","generator_receipt_sha256":"8d5f10bad274c66452a465f737da20099210de1d161c27a36c6331783da5f4f2","identity_record_sha256":"9198d8dbcec2c86f25636e37bc394d9d761cea2bb93559b1786b527f69e7fd7c","prompt_token_ids_sha256":"1102e50da7352084f75b9fa23ab6cc2d70708530625fc296126f13ff696d77e9","ruler_category":"question_answering","seed":12339,"selection_rank":8,"selection_sha256":"8bfbb4cad87302a003026059836508ab6e2d89448865d0217704654cd5b9bc8e","sequence_length":1368,"sequence_token_ids_sha256":"1102e50da7352084f75b9fa23ab6cc2d70708530625fc296126f13ff696d77e9","source_content_sha256":"8d5f10bad274c66452a465f737da20099210de1d161c27a36c6331783da5f4f2","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1368,"cache_exposed_stop":1368,"prefill_start":0,"prefill_stop":1368,"scored_start":1368,"scored_stop":1368},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"56eb209f76f5b84c51f3a6dfa390051c3e6179a589094e42a630548e33c79ab6","anchor_positions":[117,235,353,472,590,708,827,945,1063,1182,1300,1418,1537,1655,1773,1892],"anchor_positions_sha256":"0f267c5222d70787fd8238bd3de2e5bcbdfa19d30f6519cbc9abe6eb49aff8f8","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_2:length=2048:seed=12340:sample=0","config":"qa_2","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[117,235,353,471,589,708,826,944,1062,1180,1299,1417,1535,1653,1771,1890],"fisher_boundary_sha256":"433027cb3aa35bbaaa74d9db1d137366f2b08b50e51181e868b602a2a55b6952","horizon":1,"input_positions":[118,236,354,472,590,709,827,945,1063,1181,1300,1418,1536,1654,1772,1891],"input_token_ids_sha256":"eef4188a93a14820c4b8529c64e375c3dc36f29c2314d289c81684b9e3d1db9d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[119,237,355,473,591,710,828,946,1064,1182,1301,1419,1537,1655,1773,1892],"target_token_ids_sha256":"828ccb325b0d5b2cbe71be59f39c620de4093e298de9f0dda10d92926f2263fe"},"formatted_content_sha256":"696b7eb132f4a3d5f49052ac7680047c4a0a1ad1fc05f0bfa2bb25b4ddd67353","generator_receipt_sha256":"e58aa114cbdb8502b1dff17517a049edca17022eacfe90cd5afc0e2eda097c3e","identity_record_sha256":"67d8a257b752e3b5c57062870f7fde1fc0efbf2427ed7c5aaa4b1f1b94419ddc","prompt_token_ids_sha256":"ca3e9180e70700d36dc0d15f1ca4e7e4febee3a9058814b3f575e89f58f04daf","ruler_category":"question_answering","seed":12340,"selection_rank":9,"selection_sha256":"8fd992ea1ad818ee9cb171e5d9070ef59e24c95c1dfb3549cdc3963f904c96d3","sequence_length":1893,"sequence_token_ids_sha256":"ca3e9180e70700d36dc0d15f1ca4e7e4febee3a9058814b3f575e89f58f04daf","source_content_sha256":"e58aa114cbdb8502b1dff17517a049edca17022eacfe90cd5afc0e2eda097c3e","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1893,"cache_exposed_stop":1893,"prefill_start":0,"prefill_stop":1893,"scored_start":1893,"scored_stop":1893},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"cc7b46a7e50bc99f0cdb680c8631375304bb7983eedd6e6bec9dbbd2fbcfbdf5","anchor_positions":[251,504,756,1009,1261,1514,1766,2019,2271,2524,2776,3029,3281,3534,3786,4039],"anchor_positions_sha256":"7360c0b207ce24b236c422e4e664bff53fcafb7e524bd09aaebed6ea7c3ec7f8","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_1:length=4096:seed=12339:sample=0","config":"qa_1","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[251,503,756,1008,1260,1513,1765,2018,2270,2522,2775,3027,3279,3532,3784,4037],"fisher_boundary_sha256":"e551a79a205e6b53d8d809cc13da1c1d6f98faf520fcf9e40ed135aee7b9bb8e","horizon":1,"input_positions":[252,504,757,1009,1261,1514,1766,2019,2271,2523,2776,3028,3280,3533,3785,4038],"input_token_ids_sha256":"eb303aed866e00a6665d3e56e5ccdf9775fbd1e8a9bb7f5652c3fd7bd9707424","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[253,505,758,1010,1262,1515,1767,2020,2272,2524,2777,3029,3281,3534,3786,4039],"target_token_ids_sha256":"fdd7b7b8e4d53d09f5fa32c50c573deb2dbc0fe9474bed57416b8fac4ca54c1e"},"formatted_content_sha256":"9c7b681fe9e8d8f416aceaac08bdfd92d885e97dc140320d76ff0d9fefb803db","generator_receipt_sha256":"2dd908d09753092d8a358262f5073246919dd0069dfb0a60eca406aba764caa5","identity_record_sha256":"de00cb025c394a25eaa32bd222daf198e16413a112f3bccd5a72b628d593cd51","prompt_token_ids_sha256":"a089b295fbea580bdf8d050e97c37b242dfd30da2c3c7a3e620b76a54ed9c79f","ruler_category":"question_answering","seed":12339,"selection_rank":10,"selection_sha256":"a114fee36328bb536916462de6cc07e89d3dd7b3809c5295bc63e2663d7973f7","sequence_length":4040,"sequence_token_ids_sha256":"a089b295fbea580bdf8d050e97c37b242dfd30da2c3c7a3e620b76a54ed9c79f","source_content_sha256":"2dd908d09753092d8a358262f5073246919dd0069dfb0a60eca406aba764caa5","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":4040,"cache_exposed_stop":4040,"prefill_start":0,"prefill_stop":4040,"scored_start":4040,"scored_stop":4040},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"271a989874c4cc17ca1a739347bba23628a9e467e85d3219bb363f9ed7b07bf0","anchor_positions":[115,231,348,464,580,697,813,930,1046,1162,1279,1395,1511,1628,1744,1861],"anchor_positions_sha256":"cf6b395abd6929080fc35256e6179d2e3c657bc70c02f16093cb0533b0035bcc","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:cwe:length=2048:seed=12340:sample=0","config":"cwe","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[115,231,347,464,580,696,812,929,1045,1161,1277,1394,1510,1626,1742,1859],"fisher_boundary_sha256":"4015c686931b188bc2932b79e3bfa9ac30c8393959c5972c66fb8b82816a3bfd","horizon":1,"input_positions":[116,232,348,465,581,697,813,930,1046,1162,1278,1395,1511,1627,1743,1860],"input_token_ids_sha256":"7a7df1462084ce8ab589c85b605a1f4311c0ec91dc306a02673a84285b192131","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[117,233,349,466,582,698,814,931,1047,1163,1279,1396,1512,1628,1744,1861],"target_token_ids_sha256":"77743f96c0e111a7c8ad0cb12178be0abbe535b25edec949d8812bb2522c07a0"},"formatted_content_sha256":"3b5a450e8c283afa9103e3393bb8414b1d833a42c88cc5d7d4bd3f26a6120467","generator_receipt_sha256":"1c23f946536f8b05ad81d768edf4b1a5cc74ce39a3e9048124593635eb45ad57","identity_record_sha256":"21e8e431c773084606e1bb52807ebb59255d6a84e9bff884ebb333b51b0833f7","prompt_token_ids_sha256":"52bbc9b8942f395aaf5ad8a6b712b521a6a64d665cf085cb5fba76caccdf5b8f","ruler_category":"aggregation","seed":12340,"selection_rank":11,"selection_sha256":"a4f7cd62de6cc19b84e61c4bbedb2f095811b5b15e0045c070a67a003ff2110a","sequence_length":1862,"sequence_token_ids_sha256":"52bbc9b8942f395aaf5ad8a6b712b521a6a64d665cf085cb5fba76caccdf5b8f","source_content_sha256":"1c23f946536f8b05ad81d768edf4b1a5cc74ce39a3e9048124593635eb45ad57","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1862,"cache_exposed_stop":1862,"prefill_start":0,"prefill_stop":1862,"scored_start":1862,"scored_stop":1862},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"134c9730064b61e43d80673f0cb931058bd1b3199bca91ad233138c0226ce11c","anchor_positions":[120,242,364,486,608,730,852,974,1096,1218,1340,1462,1584,1706,1828,1950],"anchor_positions_sha256":"1e64ef366c8307e3a3410c30274175d81a4400831d9e53231d196d85e95f722d","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:fwe:length=2048:seed=12339:sample=0","config":"fwe","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[120,242,364,486,608,729,851,973,1095,1217,1338,1460,1582,1704,1826,1948],"fisher_boundary_sha256":"6f240a219eecca56c401dc3488a6be62114cf717fb5d3354698e9b9bd01cb16c","horizon":1,"input_positions":[121,243,365,487,609,730,852,974,1096,1218,1339,1461,1583,1705,1827,1949],"input_token_ids_sha256":"cb97fb7a28acb3307ed6ee8d8c153283de017a8ff009aa8791bb6bcb457ee5f6","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[122,244,366,488,610,731,853,975,1097,1219,1340,1462,1584,1706,1828,1950],"target_token_ids_sha256":"317374ba87916932cb7eb42091c9f2e9f86931a246150c2f199b1253ca4deba1"},"formatted_content_sha256":"6133db426215f641d319c7f7a251c50725f112b597bacfa38b5b397f559b9bc6","generator_receipt_sha256":"bb2518d8cd4ac6e41fa7aafc28a2951da762de64e8c2843d63b20f731b44ed69","identity_record_sha256":"fda1b358c53c26d14ec53f9f1307c3a894b7bd9b2265f34dc4835ad657df108d","prompt_token_ids_sha256":"5dc1507b9a08d033e3d20d61fa912a594f04959a9046d7a45a38e8828444ba30","ruler_category":"aggregation","seed":12339,"selection_rank":12,"selection_sha256":"a802c5ae87d368a5b0da4639128597266434ded4c54487021d47309dd1c7d305","sequence_length":1951,"sequence_token_ids_sha256":"5dc1507b9a08d033e3d20d61fa912a594f04959a9046d7a45a38e8828444ba30","source_content_sha256":"bb2518d8cd4ac6e41fa7aafc28a2951da762de64e8c2843d63b20f731b44ed69","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1951,"cache_exposed_stop":1951,"prefill_start":0,"prefill_stop":1951,"scored_start":1951,"scored_stop":1951},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3a9bc7a5fff3d80eb04fe1bfebb5c1e9543f99c72342a46af4069af857693b87","anchor_positions":[222,446,669,893,1116,1340,1563,1787,2010,2234,2457,2681,2904,3128,3351,3575],"anchor_positions_sha256":"f02061347ae4b281dab9fe2e3493b73bfb716ffe8fbf61cda899a3009674e03c","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:cwe:length=4096:seed=12340:sample=0","config":"cwe","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[222,445,669,892,1115,1339,1562,1786,2009,2232,2456,2679,2902,3126,3349,3573],"fisher_boundary_sha256":"44f51387dfbe3723bf9be8b472cc53ae9a75aa3358aacbe82ce774bca098b167","horizon":1,"input_positions":[223,446,670,893,1116,1340,1563,1787,2010,2233,2457,2680,2903,3127,3350,3574],"input_token_ids_sha256":"7d7dec828b0e79f43e590036098693e354e401b432f63c491d9b74ef31439c57","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[224,447,671,894,1117,1341,1564,1788,2011,2234,2458,2681,2904,3128,3351,3575],"target_token_ids_sha256":"8a65c1ab28b83d19798e54204b1019dbe72fa0be387f3d6e56ba27d82d13e7fa"},"formatted_content_sha256":"bbb3633bd44e955d1f33e80a2c1f71f0ae0e778f671a8a282b12b48eb76e3e53","generator_receipt_sha256":"3a6698b2ea721d388d72f28e1c4d4646616d6952fe01ed8038c01dc53dee3a72","identity_record_sha256":"cad2c6009d9d8080acff02b0fc66babc81f232bac219039d597c2889f240a957","prompt_token_ids_sha256":"679f814d93763a3c0db75fbc0cbf70d684bfd0ccde3a62fe15ff9d09dbc1a014","ruler_category":"aggregation","seed":12340,"selection_rank":13,"selection_sha256":"e09c4998af87e1e33e97059c684d3b0558e6e1e107295e2ccf3655bc3d70e7c8","sequence_length":3576,"sequence_token_ids_sha256":"679f814d93763a3c0db75fbc0cbf70d684bfd0ccde3a62fe15ff9d09dbc1a014","source_content_sha256":"3a6698b2ea721d388d72f28e1c4d4646616d6952fe01ed8038c01dc53dee3a72","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":3576,"cache_exposed_stop":3576,"prefill_start":0,"prefill_stop":3576,"scored_start":3576,"scored_stop":3576},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"8046d74f592f47985b3070756d1e18a618db723abbd079cab7a5a30d4efcf596","anchor_positions":[124,250,375,501,626,752,877,1003,1129,1254,1380,1505,1631,1756,1882,2008],"anchor_positions_sha256":"d296c9d479bcd5abca26602f7c8ff6f9e1668dd7bcce7512dc2b7a9f6612803c","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=2048:seed=12340:sample=0","config":"vt","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[124,249,375,500,626,751,877,1002,1127,1253,1378,1504,1629,1755,1880,2006],"fisher_boundary_sha256":"bddd9058cc4355edbe7a8c638de06d849c2134d8813b5401ee6957f2d201a918","horizon":1,"input_positions":[125,250,376,501,627,752,878,1003,1128,1254,1379,1505,1630,1756,1881,2007],"input_token_ids_sha256":"56e2b2e1f3b426b0a06e44e2cbb64564b09850f5d87d64bf62b5cea5466bcee1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[126,251,377,502,628,753,879,1004,1129,1255,1380,1506,1631,1757,1882,2008],"target_token_ids_sha256":"bd6a1f6ec03f5d3adcfb37a0b1ac11dadaa147d56888994f667aa7841dc1257b"},"formatted_content_sha256":"b17100d16d1771ca1122c2b9937cacb984da6216ccf38bef7be0228979e43248","generator_receipt_sha256":"e8b50ab62299b2f7306b66c83a8c6f2b471e1198bc02aba090f1ab406da24110","identity_record_sha256":"ab943760500ed5247a613c9eb87ac98c621b16c8919c9b1bcf5bebb2f1da258f","prompt_token_ids_sha256":"c6dfdc419e73d825ef29b21a8e2369076e96964690bfab85b8262a2299d05ce2","ruler_category":"multi_hop_tracing","seed":12340,"selection_rank":14,"selection_sha256":"f833f67c79e0d9e87d7d872ac8220d32c02483b56c47dbcd2385976fff75fe5f","sequence_length":2009,"sequence_token_ids_sha256":"c6dfdc419e73d825ef29b21a8e2369076e96964690bfab85b8262a2299d05ce2","source_content_sha256":"e8b50ab62299b2f7306b66c83a8c6f2b471e1198bc02aba090f1ab406da24110","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2009,"cache_exposed_stop":2009,"prefill_start":0,"prefill_stop":2009,"scored_start":2009,"scored_stop":2009},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"163e62428e8cb8f598c255e524c87ac6fe75f5451263fe61445eb8cafaa35b23","anchor_positions":[246,493,740,987,1234,1482,1729,1976,2223,2470,2718,2965,3212,3459,3706,3954],"anchor_positions_sha256":"c09190aec7245025981cb99d1fb2f1b133b5cd4db4f1f1baa5e107f0f2266b39","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_single_1:length=4096:seed=12339:sample=0","config":"niah_single_1","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[246,493,740,987,1234,1481,1728,1975,2222,2469,2716,2963,3210,3457,3704,3952],"fisher_boundary_sha256":"5d449cbf648406313b9dd1abfb6d827d86ea00767627eae59437398324207334","horizon":1,"input_positions":[247,494,741,988,1235,1482,1729,1976,2223,2470,2717,2964,3211,3458,3705,3953],"input_token_ids_sha256":"bc6efe8e70dc84d76b6d63607de536e9dad39e3e692c1153b17d01d1f948f5b4","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[248,495,742,989,1236,1483,1730,1977,2224,2471,2718,2965,3212,3459,3706,3954],"target_token_ids_sha256":"8b9cd390ee896f79de83be50c222152af4aff55c3013e4e55bd32df875a3182e"},"formatted_content_sha256":"5044e6b8cb4a5c93592f73b69fc42041fad9c54b2ba7d53216fc486266f9f8ba","generator_receipt_sha256":"bd8f1387bc9aca3118ced5d9287c08f72c670bfbe6ef88729d36d6940cc2a22f","identity_record_sha256":"9209f85027779d40b0ad2a15080aef80e042f89c3c96760320f1648c2f18b461","prompt_token_ids_sha256":"786505e0d874cafdc0c5a41e3e0922fe96e048b0fb0ed0d4c098605d9bf54be8","ruler_category":"retrieval","seed":12339,"selection_rank":15,"selection_sha256":"ffbb6f30b667ab8eb83eff25cc9f46717b4a36965d1f9c15e08b6633fe46e13d","sequence_length":3955,"sequence_token_ids_sha256":"786505e0d874cafdc0c5a41e3e0922fe96e048b0fb0ed0d4c098605d9bf54be8","source_content_sha256":"bd8f1387bc9aca3118ced5d9287c08f72c670bfbe6ef88729d36d6940cc2a22f","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":3955,"cache_exposed_stop":3955,"prefill_start":0,"prefill_stop":3955,"scored_start":3955,"scored_stop":3955},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"}],"resolver_version":6,"schema_version":5,"selection":{"humaneval_plus_stage_a_b_namespace":"recurquant.experiment013.humaneval-plus.stage-a-b.v1\u0000","humaneval_plus_stage_c_namespace":"recurquant.experiment013.humaneval-plus.stage-c.v1\u0000","pg19_test_namespace":"recurquant.experiment013.pg19.test.v1\u0000","pg19_train_namespace":"recurquant.experiment013.pg19.train.v1\u0000","pg19_validation_namespace":"recurquant.experiment013.pg19.validation.v1\u0000"},"source_manifest_sha256":"fc3240bc893839ee550ec412daf71e515c452819df9055dcf583360b1d3f96cf","status":"frozen","tokenizer":{"class":"Qwen2Tokenizer","file_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef","files":[{"name":"merges.txt","sha256":"a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d","size_bytes":3353259},{"name":"tokenizer.json","sha256":"fe000e3ed39ed12b8d2481d527d44f93c65d37e87645d2dcc80d1bf9d50d2927","size_bytes":12807196},{"name":"tokenizer_config.json","sha256":"e611fbccc7c29ef3b1cafb1cb7ea548d189968632901d678fd62be68c47885de","size_bytes":16712},{"name":"vocab.json","sha256":"ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003","size_bytes":6722759}],"revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68","source_id":"Qwen/Qwen3.5-0.8B-Base","transformers_version":"5.14.1"},"upstream_tool_contracts":{"evalplus_formatter":{"id":"evalplus/evalplus","revision":"26d6d00bb1fd0fa37f39c99d5290da67891d1c5e"},"ruler_generator":{"id":"NVIDIA/RULER","revision":"c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a"}}}} From 0f3ea5e86e5d2ec13d5c5836540ce105e41ad02b Mon Sep 17 00:00:00 2001 From: HiddenTrojan Date: Sat, 15 Aug 2026 03:22:25 +0800 Subject: [PATCH 20/24] Repair Experiment 013 runtime context contract --- ...EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md | 75 +++++- scripts/run_static_q468_calibration.py | 69 ++++-- .../experiment013_calibration_api.py | 3 + tests/test_experiment013_calibration_api.py | 31 +++ tests/test_experiment013_qwen35_adapter.py | 2 + tests/test_run_static_q468_calibration.py | 232 ++++++++++++++++++ 6 files changed, 384 insertions(+), 28 deletions(-) diff --git a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md index 92692cd..2ef21ef 100644 --- a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md +++ b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md @@ -1,24 +1,25 @@ # Experiment 013: static RHT-Q468 packed-native adoption protocol -> **Status: replacement-H0 candidate after a preserved pre-model authorization -> failure; not yet re-preregistered.** +> **Status: second replacement-H0 candidate after a preserved pre-model +> sealed-smoke contract failure; not yet re-preregistered.** > > This replacement working copy becomes the next frozen Experiment 013 > preregistration only when its exact bytes and dependencies are committed in a > clean source commit H0 and that H0 is bound before any further identity -> resolution, policy fitting, protected materialization, model-weight loading, -> or quality measurement. A dirty or unbound working copy is not a frozen -> protocol. +> resolution, model staging or loading, policy fitting, protected +> materialization, or quality measurement. A dirty or unbound working copy is +> not a frozen protocol. > > The amendment history is retained below, but its rules remain candidate rules > until that H0 binding. Prior identities, token spans, tokenizer-file hashes, > and content hashes remain preserved as superseded evidence under the disclosed -> reuse rule; the replacement identity has not yet been resolved or promoted. -> An identity candidate is not authorization to load model weights. +> reuse rule; the next replacement identity has not yet been resolved or +> promoted. An identity candidate is not authorization to stage or load model +> weights. Protocol draft initiated: 2026-08-02 -Current replacement-H0 candidate amended: 2026-08-15 +Current second replacement-H0 candidate amended: 2026-08-15 Pre-resolution audit amendment: 2026-08-02. The amendment corrects a cache-exposed-span off-by-one, binds the Stage-A calibration chain by exact @@ -322,6 +323,62 @@ promotion-hash cascade may differ. Any inventory, byte, hash, version, or semantic-identity mismatch stops reuse and requires a separately preregistered fresh complete batch after the replacement H0. +Fourteenth pre-resolution adapter-context amendment: 2026-08-15. Under source +commit `85625a5c4e4d7c6d1b015c0f3cccffea5c3d71c3`, tag +`experiment013-h0-85625a5`, and identity-only descendant +`84edf4299e8a5b3af970f74f03abc099d3696904`, the identity-bound `stage-model` +step published a local copy of the exact three-file pinned model. Its frozen +identity file SHA-256 is +`e401a3c18a002626da096ba6ba86aa5d297d16b5c8ab76711658ce730e5a5f77`. +The first sealed `--fisher-h1-smoke` attempt then authenticated the sealed +runtime, H0 source, the frozen identity bytes committed at H1, and all four +bound manifest byte strings. It parsed and matched the public model-file +metadata but failed deterministically +while initializing `AdapterConstructionContext`: calibration-runner v3 supplied +the inert absolute `git_executable` path that capture procedure v6 requires, +while the authenticated calibration API's exact runtime-context key set omitted +that key. + +The failed smoke did not reach reviewed-adapter loading or construction, +calibration materialization, tokenizer, dataset, or RULER access, staged-model- +root traversal or file hashing, model configuration or weight deserialization, +CUDA or Fisher execution, or output-directory checking or publication. It +created no smoke output or staging sibling and produced no smoke report, +completion marker, score, policy, calibration binding, stability value, or +quality result. The separately completed `stage-model` action did access and +publish the model payload under H1; the failed smoke itself did not read that +published model root. This is a downstream execution-contract defect, not an +experimental result or infrastructure interruption. The H0, tag, H1, identity, +and staged model root remain unchanged as superseded incident evidence and +authorize no further official execution. + +Calibration-runner v4 aligns the authenticated API with capture v6's existing +five-key runtime context. The API now accepts, validates, copies, and retains +the absolute inert Git path while preserving exact-key rejection and recursive +mapping immutability. Production `_official_main` and the regression share one +context-construction helper; the regression loads the exact authenticated API +and reviewed adapter and checks that construction creates none of its model, +cache, RULER, or bytecode sentinel paths. A separate no-data boundary +regression carries the same context through the real adapter into the +manifest-bound capture module and stops before artifact decoding, Hub, +tokenizer, or dataset access. No record selection, dataset revision, token +span, Fisher boundary, quantization policy, metric, gate, model contract, or +protected-stage rule changes. + +Runner revision advances from v3 to v4. Identity schema v5; capture and +resolver procedure v6; adapter revision v2; RULER launcher v7, generation- +manifest v2, and runtime-manifest v3; calibration runtime-manifest v4; source- +manifest schema/profile v2; model-manifest v1; frozen-contract and model- +staging-authorization stdout schemas v1; run-report v2; and Fisher-boundary and +smoke-marker contracts v1 remain unchanged. A new clean H0, repository-source +manifest, promoted calibration identity, H1, and identity-bound model +publication are required. The exact authenticated RULER v7 batch, sealed +runtime, public model metadata manifest, Parquet manifest, and shared Hub cache +may be reused only after their existing byte, inventory, and semantic checks +pass unchanged. The old H1 model root may be treated only as preserved incident +evidence, not as the official model root for the next H1; a fresh no-overwrite +root must be published by a newly authorized `stage-model` execution. + ## Question Can a calibration-frozen, static Q4/Q6/Q8 recurrent-state layout satisfy the @@ -717,7 +774,7 @@ Before committing H1, the exact promoted identity bytes in their ignored, no-overwrite precommit location must pass `verify-frozen-identity-contract`. That read-only command authenticates H0 and its source manifest, loads the exact H0 resolver, and consumes the complete record inventory through calibration- -runner v3's identity view. It accepts no H1, model manifest, Hub, cache, or +runner v4's identity view. It accepts no H1, model manifest, Hub, cache, or output argument. Its non-persisted canonical JSON stdout document uses artifact kind `recurquant_experiment013_frozen_identity_contract_verification`, schema version one, and binds the H0/source contract, portable Git identity, all four diff --git a/scripts/run_static_q468_calibration.py b/scripts/run_static_q468_calibration.py index 9e7b599..8f47b19 100644 --- a/scripts/run_static_q468_calibration.py +++ b/scripts/run_static_q468_calibration.py @@ -56,7 +56,7 @@ CANONICAL_ADAPTER_MODULE: Final = "recurquant.experiment013_qwen35_adapter" CANONICAL_ADAPTER_PATH: Final = "src/recurquant/experiment013_qwen35_adapter.py" -RUNNER_REVISION: Final = "experiment-013-static-q468-calibration-runner-v3" +RUNNER_REVISION: Final = "experiment-013-static-q468-calibration-runner-v4" FROZEN_IDENTITY_SCHEMA_VERSION: Final = 5 FISHER_BOUNDARY_SCHEMA: Final = "recurquant.experiment013.fisher-boundary.v1" FISHER_BOUNDARY_NAMESPACE: Final = b"recurquant.experiment013.fisher-boundary.v1\0" @@ -5486,6 +5486,43 @@ def _install_authenticated_recurquant_namespace(repository_root: Path) -> Module return package +def _adapter_construction_context( + *, + calibration_api: ModuleType, + repository_root: Path, + model_root: Path, + cache_root: Path, + ruler_root: Path, + repository_source_manifest_bytes: bytes, + calibration_runtime_manifest_bytes: bytes, + model_file_manifest_bytes: bytes, + parquet_materialization_manifest_bytes: bytes, + runtime_context: SealedRuntimeContext, + interpreter_path: Path, +) -> Any: + """Build the exact authenticated context consumed by the reviewed adapter.""" + + return calibration_api.AdapterConstructionContext( + repository_root=Path(repository_root), + model_root=Path(model_root), + cache_root=Path(cache_root), + ruler_root=Path(ruler_root), + execution_binding_artifacts={ + "repository_source_manifest_bytes": bytes(repository_source_manifest_bytes), + "calibration_runtime_manifest_bytes": bytes(calibration_runtime_manifest_bytes), + "model_file_manifest_bytes": bytes(model_file_manifest_bytes), + "parquet_materialization_manifest_bytes": bytes(parquet_materialization_manifest_bytes), + }, + runtime_authentication_context={ + "base_runtime_root": runtime_context.base_runtime_root, + "git_executable": runtime_context.git_executable_path, + "staged_interpreter": Path(interpreter_path), + "package_runtime_roots": dict(runtime_context.package_roots), + "package_import_paths": dict(runtime_context.package_import_paths), + }, + ) + + def _load_adapter( specification: str, *, @@ -5923,24 +5960,18 @@ def _official_main( raise CalibrationRunError("runtime authenticator returned a different manifest identity") model_manifest = parse_model_file_manifest(model_manifest_bytes) _model_contract_matches(identity, model_manifest) - context = _AUTHENTICATED_CALIBRATION_API.AdapterConstructionContext( - repository_root=Path(args.repository_root), - model_root=Path(args.model_root), - cache_root=Path(args.cache_root), - ruler_root=Path(args.ruler_root), - execution_binding_artifacts={ - "repository_source_manifest_bytes": bytes(source_manifest_bytes), - "calibration_runtime_manifest_bytes": bytes(runtime_manifest_bytes), - "model_file_manifest_bytes": bytes(model_manifest_bytes), - "parquet_materialization_manifest_bytes": bytes(parquet_manifest_bytes), - }, - runtime_authentication_context={ - "base_runtime_root": runtime_context.base_runtime_root, - "git_executable": runtime_context.git_executable_path, - "staged_interpreter": Path(interpreter_path), - "package_runtime_roots": dict(runtime_context.package_roots), - "package_import_paths": dict(runtime_context.package_import_paths), - }, + context = _adapter_construction_context( + calibration_api=_AUTHENTICATED_CALIBRATION_API, + repository_root=args.repository_root, + model_root=args.model_root, + cache_root=args.cache_root, + ruler_root=args.ruler_root, + repository_source_manifest_bytes=source_manifest_bytes, + calibration_runtime_manifest_bytes=runtime_manifest_bytes, + model_file_manifest_bytes=model_manifest_bytes, + parquet_materialization_manifest_bytes=parquet_manifest_bytes, + runtime_context=runtime_context, + interpreter_path=interpreter_path, ) adapter = _load_adapter( args.adapter, diff --git a/src/recurquant/experiment013_calibration_api.py b/src/recurquant/experiment013_calibration_api.py index cfe1234..3694f09 100644 --- a/src/recurquant/experiment013_calibration_api.py +++ b/src/recurquant/experiment013_calibration_api.py @@ -26,6 +26,7 @@ RUNTIME_AUTHENTICATION_CONTEXT_KEYS = frozenset( { "base_runtime_root", + "git_executable", "package_import_paths", "package_runtime_roots", "staged_interpreter", @@ -70,6 +71,7 @@ def _normalize_runtime_authentication_context( if not isinstance(value, Mapping) or set(value) != RUNTIME_AUTHENTICATION_CONTEXT_KEYS: raise ValueError("runtime_authentication_context keys differ from the frozen API") base_root = _absolute_inert_path(value["base_runtime_root"], name="base_runtime_root") + git_executable = _absolute_inert_path(value["git_executable"], name="git_executable") interpreter = _absolute_inert_path( value["staged_interpreter"], name="staged_interpreter", @@ -96,6 +98,7 @@ def _normalize_runtime_authentication_context( return MappingProxyType( { "base_runtime_root": base_root, + "git_executable": git_executable, "package_import_paths": MappingProxyType(import_paths), "package_runtime_roots": MappingProxyType(package_roots), "staged_interpreter": interpreter, diff --git a/tests/test_experiment013_calibration_api.py b/tests/test_experiment013_calibration_api.py index 08bf8e7..547eada 100644 --- a/tests/test_experiment013_calibration_api.py +++ b/tests/test_experiment013_calibration_api.py @@ -7,6 +7,7 @@ import pytest from recurquant.experiment013_calibration_api import ( + RUNTIME_AUTHENTICATION_CONTEXT_KEYS, AdapterConstructionContext, AuthenticatedModelFiles, AuthenticatedSequence, @@ -32,6 +33,7 @@ def binding_artifacts() -> dict[str, bytes]: def runtime_context() -> dict[str, object]: return { "base_runtime_root": ROOT / "runtime" / "base", + "git_executable": ROOT / "tools" / "git.exe", "staged_interpreter": ROOT / "runtime" / "base" / "python.exe", "package_runtime_roots": {"calibration": ROOT / "runtime" / "packages"}, "package_import_paths": {"calibration": "Lib/site-packages"}, @@ -62,6 +64,10 @@ def test_context_copy_normalizes_exact_binding_bytes() -> None: def test_context_normalizes_and_freezes_runtime_authentication_paths() -> None: source = runtime_context() + package_roots = source["package_runtime_roots"] + package_import_paths = source["package_import_paths"] + assert isinstance(package_roots, dict) + assert isinstance(package_import_paths, dict) context = AdapterConstructionContext( ROOT, ROOT, @@ -70,18 +76,43 @@ def test_context_normalizes_and_freezes_runtime_authentication_paths() -> None: source, binding_artifacts(), ) + source["git_executable"] = ROOT / "changed-git.exe" source["package_runtime_roots"] = {} + package_roots["calibration"] = ROOT / "changed-packages" + package_import_paths["calibration"] = "changed/site-packages" + assert set(context.runtime_authentication_context) == RUNTIME_AUTHENTICATION_CONTEXT_KEYS assert context.runtime_authentication_context["base_runtime_root"] == ( ROOT / "runtime" / "base" ) + assert context.runtime_authentication_context["git_executable"] == (ROOT / "tools" / "git.exe") + assert context.runtime_authentication_context["package_runtime_roots"] == { + "calibration": ROOT / "runtime" / "packages" + } + assert context.runtime_authentication_context["package_import_paths"] == { + "calibration": "Lib/site-packages" + } with pytest.raises(TypeError): context.runtime_authentication_context["new"] = ROOT # type: ignore[index] + with pytest.raises(TypeError): + context.runtime_authentication_context["package_runtime_roots"]["new"] = ROOT # type: ignore[index] + with pytest.raises(TypeError): + context.runtime_authentication_context["package_import_paths"]["new"] = "Lib" # type: ignore[index] malformed = runtime_context() malformed["package_import_paths"] = {"calibration": "../site-packages"} with pytest.raises(ValueError, match="not canonical"): AdapterConstructionContext(ROOT, ROOT, ROOT, ROOT, malformed, binding_artifacts()) + relative_git = runtime_context() + relative_git["git_executable"] = Path("git.exe") + with pytest.raises(ValueError, match="git_executable must be an absolute normalized Path"): + AdapterConstructionContext(ROOT, ROOT, ROOT, ROOT, relative_git, binding_artifacts()) + + missing_git = runtime_context() + missing_git.pop("git_executable") + with pytest.raises(ValueError, match="keys differ"): + AdapterConstructionContext(ROOT, ROOT, ROOT, ROOT, missing_git, binding_artifacts()) + def test_adapter_facing_values_have_one_stable_importable_identity() -> None: sequence = AuthenticatedSequence((1, 2), "a" * 64, "b" * 64, None, "c" * 64) diff --git a/tests/test_experiment013_qwen35_adapter.py b/tests/test_experiment013_qwen35_adapter.py index 421c404..d674d6a 100644 --- a/tests/test_experiment013_qwen35_adapter.py +++ b/tests/test_experiment013_qwen35_adapter.py @@ -37,6 +37,7 @@ def _context(tmp_path: Path) -> AdapterConstructionContext: ruler_root=tmp_path / "ruler-does-not-need-to-exist-at-construction", runtime_authentication_context={ "base_runtime_root": tmp_path / "runtime" / "base", + "git_executable": tmp_path / "tools" / "git.exe", "staged_interpreter": tmp_path / "runtime" / "base" / "python.exe", "package_runtime_roots": {"calibration": tmp_path / "runtime" / "packages"}, "package_import_paths": {"calibration": "Lib/site-packages"}, @@ -244,6 +245,7 @@ def materialize( assert materialization_references[0]() is None assert adapter._execution_binding_artifacts is None assert adapter._runtime_authentication_context is None + assert runtime_calls[0]["git_executable"] == (tmp_path / "tools" / "git.exe") assert runtime_calls[0]["staged_interpreter"] == (tmp_path / "runtime" / "base" / "python.exe") assert set(adapter._materialized_sequences or {}) == {_sha(index + 1) for index in range(160)} assert all( diff --git a/tests/test_run_static_q468_calibration.py b/tests/test_run_static_q468_calibration.py index a3c3697..a5faf00 100644 --- a/tests/test_run_static_q468_calibration.py +++ b/tests/test_run_static_q468_calibration.py @@ -3492,6 +3492,238 @@ def test_official_loader_rejects_generic_and_preloaded_adapters() -> None: sys.modules.pop(runner.CANONICAL_ADAPTER_MODULE, None) +def test_runner_runtime_context_constructs_the_real_reviewed_adapter() -> None: + code = f""" +import hashlib +import importlib.util +import sys +from pathlib import Path + +repository_root = Path({str(SCRIPT.parents[1])!r}) +runner_path = repository_root / 'scripts' / 'run_static_q468_calibration.py' +spec = importlib.util.spec_from_file_location('isolated_runner_context_integration', runner_path) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +def source_entry(relative_path): + payload = (repository_root / relative_path).read_bytes() + return {{'raw_sha256': hashlib.sha256(payload).hexdigest()}} + +calibration_api = module._load_exact_source_module( + module.CALIBRATION_API_MODULE, + module.CALIBRATION_API_PATH, + repository_root=repository_root, + entry=source_entry(module.CALIBRATION_API_PATH), +) +git_executable = repository_root / 'authenticated-tools' / 'git.exe' +runtime_context = module.SealedRuntimeContext( + manifest_file_sha256='1' * 64, + base_runtime_root=repository_root / 'runtime' / 'base', + git_executable_path=git_executable, + package_roots={{ + 'calibration-packages': repository_root / 'runtime' / 'packages' + }}, + package_import_paths={{'calibration-packages': 'Lib/site-packages'}}, + pycache_prefix=repository_root / 'unopened-pycache', +) +context = module._adapter_construction_context( + calibration_api=calibration_api, + repository_root=repository_root, + model_root=repository_root / 'unopened-model', + cache_root=repository_root / 'unopened-cache', + ruler_root=repository_root / 'unopened-ruler', + repository_source_manifest_bytes=b'source', + calibration_runtime_manifest_bytes=b'runtime', + model_file_manifest_bytes=b'model', + parquet_materialization_manifest_bytes=b'parquet', + runtime_context=runtime_context, + interpreter_path=repository_root / 'runtime' / 'base' / 'python.exe', +) +assert set(context.runtime_authentication_context) == ( + calibration_api.RUNTIME_AUTHENTICATION_CONTEXT_KEYS +) +adapter = module._load_adapter( + module.CANONICAL_ADAPTER_SPEC, + repository_root=repository_root, + source_entry=source_entry(module.CANONICAL_ADAPTER_PATH), + calibration_api=calibration_api, + context=context, +) +assert adapter._runtime_authentication_context['git_executable'] == git_executable +assert adapter._runtime_authentication_context['package_runtime_roots'] == {{ + 'calibration-packages': repository_root / 'runtime' / 'packages' +}} +assert not (repository_root / 'unopened-model').exists() +assert not (repository_root / 'unopened-cache').exists() +assert not (repository_root / 'unopened-ruler').exists() +assert not (repository_root / 'unopened-pycache').exists() +""" + subprocess.run( + [sys.executable, "-I", "-c", code], + cwd=SCRIPT.parents[1], + check=True, + capture_output=True, + text=True, + timeout=120, + ) + + +def test_runner_context_reaches_real_capture_authentication_before_data_access() -> None: + code = f""" +import hashlib +import importlib.util +import json +import sys +from pathlib import Path + +repository_root = Path({str(SCRIPT.parents[1])!r}) +runner_path = repository_root / 'scripts' / 'run_static_q468_calibration.py' +spec = importlib.util.spec_from_file_location('isolated_runner_capture_integration', runner_path) +module = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = module +spec.loader.exec_module(module) + +def source_entry(relative_path): + payload = (repository_root / relative_path).read_bytes() + return {{'raw_sha256': hashlib.sha256(payload).hexdigest()}} + +calibration_api = module._load_exact_source_module( + module.CALIBRATION_API_MODULE, + module.CALIBRATION_API_PATH, + repository_root=repository_root, + entry=source_entry(module.CALIBRATION_API_PATH), +) +capture_source_path = 'scripts/capture_static_q468_identity_input.py' +capture_payload = (repository_root / capture_source_path).read_bytes() +source_manifest_bytes = json.dumps( + {{ + 'paths': [ + {{ + 'path': capture_source_path, + 'raw_sha256': hashlib.sha256(capture_payload).hexdigest(), + }} + ], + 'schema': 'recurquant.experiment013.source-manifest.v2', + }}, + sort_keys=True, + separators=(',', ':'), +).encode('utf-8') + +base_runtime_root = repository_root / 'unopened-boundary-runtime' +package_runtime_root = repository_root / 'unopened-boundary-packages' +git_executable = repository_root / 'unopened-boundary-tools' / 'git.exe' +model_root = repository_root / 'unopened-boundary-model' +cache_root = repository_root / 'unopened-boundary-cache' +ruler_root = repository_root / 'unopened-boundary-ruler' +pycache_root = repository_root / 'unopened-boundary-pycache' +runtime_context = module.SealedRuntimeContext( + manifest_file_sha256='1' * 64, + base_runtime_root=base_runtime_root, + git_executable_path=git_executable, + package_roots={{'calibration-packages': package_runtime_root}}, + package_import_paths={{'calibration-packages': 'Lib/site-packages'}}, + pycache_prefix=pycache_root, +) +context = module._adapter_construction_context( + calibration_api=calibration_api, + repository_root=repository_root, + model_root=model_root, + cache_root=cache_root, + ruler_root=ruler_root, + repository_source_manifest_bytes=source_manifest_bytes, + calibration_runtime_manifest_bytes=b'runtime-manifest', + model_file_manifest_bytes=b'model-manifest', + parquet_materialization_manifest_bytes=b'parquet-manifest', + runtime_context=runtime_context, + interpreter_path=base_runtime_root / 'python.exe', +) +adapter = module._load_adapter( + module.CANONICAL_ADAPTER_SPEC, + repository_root=repository_root, + source_entry=source_entry(module.CANONICAL_ADAPTER_PATH), + calibration_api=calibration_api, + context=context, +) +adapter_module = sys.modules[module.CANONICAL_ADAPTER_MODULE] +assert adapter_module.CAPTURE_SOURCE_PATH == capture_source_path +assert adapter_module.SOURCE_MANIFEST_SCHEMA == ( + 'recurquant.experiment013.source-manifest.v2' +) +capture_binding = adapter_module._load_capture_module( + repository_root, + source_manifest_bytes, +) + +observed = {{}} + +class CaptureAuthenticationBoundaryReached(RuntimeError): + pass + +def stop_before_artifact_hub_or_data_access(artifacts, *, runtime_context, **kwargs): + assert kwargs == {{}} + observed['artifacts'] = dict(artifacts) + observed['runtime_context_type'] = type(runtime_context).__name__ + observed['base_runtime_root'] = runtime_context.base_runtime_root + observed['git_executable'] = runtime_context.git_executable + observed['staged_interpreter'] = runtime_context.staged_interpreter + observed['package_runtime_roots'] = dict(runtime_context.package_runtime_roots) + observed['package_import_paths'] = dict(runtime_context.package_import_paths) + raise CaptureAuthenticationBoundaryReached + +capture_binding.module._authenticate_execution_binding_artifacts = ( + stop_before_artifact_hub_or_data_access +) +adapter_module._load_capture_module = lambda _root, _manifest: capture_binding + +try: + adapter.materialize_sequence({{'identity_record_sha256': '0' * 64}}) +except CaptureAuthenticationBoundaryReached: + pass +else: + raise AssertionError('real capture authentication boundary was not reached') + +assert observed == {{ + 'artifacts': {{ + 'calibration_runtime_manifest_file_sha256': b'runtime-manifest', + 'model_file_manifest_file_sha256': b'model-manifest', + 'parquet_materialization_manifest_file_sha256': b'parquet-manifest', + 'repository_source_manifest_file_sha256': source_manifest_bytes, + }}, + 'runtime_context_type': '_RuntimeAuthenticationContext', + 'base_runtime_root': base_runtime_root, + 'git_executable': git_executable, + 'staged_interpreter': base_runtime_root / 'python.exe', + 'package_runtime_roots': {{'calibration-packages': package_runtime_root}}, + 'package_import_paths': {{'calibration-packages': 'Lib/site-packages'}}, +}} +assert adapter._execution_binding_artifacts is None +assert adapter._runtime_authentication_context is None +assert capture_binding.module._CALIBRATION_RUNNER_MODULE_NAME not in sys.modules +assert {{'datasets', 'huggingface_hub', 'transformers'}}.isdisjoint(sys.modules) +assert not any( + path.exists() + for path in ( + base_runtime_root, + package_runtime_root, + git_executable.parent, + model_root, + cache_root, + ruler_root, + pycache_root, + ) +) +""" + subprocess.run( + [sys.executable, "-I", "-B", "-c", code], + cwd=SCRIPT.parents[1], + check=True, + capture_output=True, + text=True, + timeout=120, + ) + + def test_default_services_do_not_eagerly_import_static_calibration_modules() -> None: code = f""" import importlib.util From dae5587adc8f9a2b16335dfdc501e7f0a3f5e6ab Mon Sep 17 00:00:00 2001 From: HiddenTrojan Date: Sat, 15 Aug 2026 04:08:43 +0800 Subject: [PATCH 21/24] Record Experiment 013 runner-v4 calibration identity --- evidence/experiment013-calibration-identity-v5-h0-0f3ea5e.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 evidence/experiment013-calibration-identity-v5-h0-0f3ea5e.json diff --git a/evidence/experiment013-calibration-identity-v5-h0-0f3ea5e.json b/evidence/experiment013-calibration-identity-v5-h0-0f3ea5e.json new file mode 100644 index 0000000..82166b8 --- /dev/null +++ b/evidence/experiment013-calibration-identity-v5-h0-0f3ea5e.json @@ -0,0 +1 @@ +{"canonical_evidence_sha256":"2704625d96e6c7660652abcb664146110635bc686b2f2bc0b9034ecf9b8d9f18","evidence":{"artifact_kind":"recurquant_static_rht_q468_identity","calibration_binding":null,"calibration_split_half":{"assignment":[{"canonical_id":"872","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":0,"rank_sha256":"01e970ad8f49c615f8994f4fc293d373d517f753cf5cef3ee60e04b0f70e91de","ruler_category":null,"seed":null,"sequence_length":205},{"canonical_id":"907","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":1,"rank_sha256":"0302f20255cec68e850f67a965f94ac9c6e975fdbeb3654b213c03aaf97749f8","ruler_category":null,"seed":null,"sequence_length":195},{"canonical_id":"602","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":2,"rank_sha256":"0533693c5eafac0f2bffd8ecb5f531f37e3220e7347ef75eab142ed8a013bcf5","ruler_category":null,"seed":null,"sequence_length":131},{"canonical_id":"884","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":3,"rank_sha256":"08317423d6ed9f655783d9a55d4773b1ff31c0a8355dfd9186fb319920e3d84b","ruler_category":null,"seed":null,"sequence_length":180},{"canonical_id":"771","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":4,"rank_sha256":"0aa65ef4aee19c821f354944ea63d3a5573594cbf98328585a2bf06c2de91309","ruler_category":null,"seed":null,"sequence_length":210},{"canonical_id":"885","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":5,"rank_sha256":"0b8df4c9f6062430c6c4ccd1cb93ed972689c2181d647b3425c8ac35779c5f9c","ruler_category":null,"seed":null,"sequence_length":193},{"canonical_id":"672","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":6,"rank_sha256":"0d45859c2052e7483b7caae8e52246e30d4af1be980cbcb175164817f5ed511d","ruler_category":null,"seed":null,"sequence_length":164},{"canonical_id":"955","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":7,"rank_sha256":"15743574d0995e07e15392053b3cafd1cef3998b668e87041851ac95861883ca","ruler_category":null,"seed":null,"sequence_length":116},{"canonical_id":"794","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":8,"rank_sha256":"19ee9c8929dc155f23f27a50530b329f876367ad3aea0bf4ad48ce4621328517","ruler_category":null,"seed":null,"sequence_length":154},{"canonical_id":"641","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":9,"rank_sha256":"19fd16656926305e85d6a90a8abfc8688cde21fff9383ac96aa4a507279d79f9","ruler_category":null,"seed":null,"sequence_length":104},{"canonical_id":"744","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":10,"rank_sha256":"1afe31e9cfc41a6df7f6609d1e2119ad55693730c75b4bd46f7c9235522eafa0","ruler_category":null,"seed":null,"sequence_length":137},{"canonical_id":"647","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":11,"rank_sha256":"1b76da1efe10f0a39d6b61952eb3fee319e653390381bd4a8efb060e10161d77","ruler_category":null,"seed":null,"sequence_length":107},{"canonical_id":"763","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":12,"rank_sha256":"1c7722a133b7999ee3e6ff0c4c7d9286371137589ac7b013d5d9a1615c2eca43","ruler_category":null,"seed":null,"sequence_length":181},{"canonical_id":"835","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":13,"rank_sha256":"27e79049077f4f70e44f882154a065bb8fe957aa57c51a5f95d025e7f6b69e7e","ruler_category":null,"seed":null,"sequence_length":113},{"canonical_id":"720","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":14,"rank_sha256":"2c2ba3d77f6374f2c035569f8be3b970f76c25bc587f0f2bd165939b49af1c27","ruler_category":null,"seed":null,"sequence_length":292},{"canonical_id":"677","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":15,"rank_sha256":"3015295069fd5535022e22aa7d447fdfd3b893623d25fbe8b2279d94c7d99ee5","ruler_category":null,"seed":null,"sequence_length":120},{"canonical_id":"819","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":16,"rank_sha256":"3355f161793b19251d4281f96e19b37a970a709a1b417d52e6dfbb2a4812d792","ruler_category":null,"seed":null,"sequence_length":385},{"canonical_id":"950","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":17,"rank_sha256":"339900553b726e1b1256a75b102fbeed3f547bfae8d96f80e8fec0f8dec6c09b","ruler_category":null,"seed":null,"sequence_length":399},{"canonical_id":"622","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":18,"rank_sha256":"363cce01da9a0aa22049e9b3a8e04a2dc931468e02d0086bb7af971621cfc711","ruler_category":null,"seed":null,"sequence_length":373},{"canonical_id":"708","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":19,"rank_sha256":"372acd4106f75fd22c1fdb5d8e13b74c64a9364db2f490f50daa482c0962112d","ruler_category":null,"seed":null,"sequence_length":97},{"canonical_id":"878","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":20,"rank_sha256":"38728fc433a299c9ff5e0f49471dda51ee67c5a980c966c00c715bd76e7f1c88","ruler_category":null,"seed":null,"sequence_length":168},{"canonical_id":"648","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":21,"rank_sha256":"388ca5f79823a878cd3ce1e033b61e5cab8bf668fe57d751138244feef74a2f4","ruler_category":null,"seed":null,"sequence_length":225},{"canonical_id":"937","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":22,"rank_sha256":"3ad5eee8852be5065365af9c25ff78ec2974ee89b2c6fa6759e27b7de255e202","ruler_category":null,"seed":null,"sequence_length":110},{"canonical_id":"894","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":23,"rank_sha256":"3c7238b06352863f75aab36214380eedfa02c7c388385142283f0980d21c1f7a","ruler_category":null,"seed":null,"sequence_length":226},{"canonical_id":"653","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":24,"rank_sha256":"3dd1d2a846dba10b3ffa7e554091ac65fe9284c15f2d83dc44c705155130a763","ruler_category":null,"seed":null,"sequence_length":287},{"canonical_id":"679","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":25,"rank_sha256":"3e59d101b564d5e3344a2e764584a194a5f663f3b20f7e939d7f719f84dacba4","ruler_category":null,"seed":null,"sequence_length":136},{"canonical_id":"657","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":26,"rank_sha256":"3e6b96b2be683d5d193895bbc05beb5a13bfc3b1db01872788e0ae8133c72b36","ruler_category":null,"seed":null,"sequence_length":171},{"canonical_id":"924","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":27,"rank_sha256":"45104189ef24ab437b12b531013325cf0a41819cf135255a4ca993f29f74dfd2","ruler_category":null,"seed":null,"sequence_length":103},{"canonical_id":"686","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":28,"rank_sha256":"47010c76b0b792ee5452a5eb55e8ba40d386f4a17d895b443eb1a66556c90596","ruler_category":null,"seed":null,"sequence_length":285},{"canonical_id":"860","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":29,"rank_sha256":"471b89737df3331c676e99243030a77140d460cde893f819d2c7916e0a16f453","ruler_category":null,"seed":null,"sequence_length":141},{"canonical_id":"698","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":30,"rank_sha256":"4a9f3f47f58cfaabd9afda89457088ca35397d7acc80568d8d218a723451116e","ruler_category":null,"seed":null,"sequence_length":359},{"canonical_id":"827","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":31,"rank_sha256":"4b073dd1bc9f2a79d60985c8d50aae572f2030e62a55477c64702d735d849023","ruler_category":null,"seed":null,"sequence_length":174},{"canonical_id":"651","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":32,"rank_sha256":"4bf2b2787c95b7b8a2f8ff089c36ea2662b4987dbf230b02b81c424382853b7b","ruler_category":null,"seed":null,"sequence_length":155},{"canonical_id":"930","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":33,"rank_sha256":"4ffef951fb9708b54468eaa75c7b42b9f58c198c366cfacd6b4f665c9d77984f","ruler_category":null,"seed":null,"sequence_length":138},{"canonical_id":"962","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":34,"rank_sha256":"51a6c027909f096835086346110e63a131c1462156020bb129058b320f27516a","ruler_category":null,"seed":null,"sequence_length":149},{"canonical_id":"853","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":35,"rank_sha256":"525322d1199e29de4998af2c95211cd981e642e9d602fe16554a8a63fc02d729","ruler_category":null,"seed":null,"sequence_length":219},{"canonical_id":"722","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":36,"rank_sha256":"52acb8744119319aaa1a68b3e192415c937a6822ceb2963069d4ede3e9b75cc4","ruler_category":null,"seed":null,"sequence_length":388},{"canonical_id":"705","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":37,"rank_sha256":"547e7ebc86a8992205db2eda5a7117796f15c3042e90200d556582e58f881c17","ruler_category":null,"seed":null,"sequence_length":258},{"canonical_id":"944","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":38,"rank_sha256":"577f992e266f3ffe7684e9483310b96df9b338bd836e00af79e0116821fddaa5","ruler_category":null,"seed":null,"sequence_length":117},{"canonical_id":"849","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":39,"rank_sha256":"59660871dc9200a74f124b924d6e85f743e541a39ae93af501968a7ed5d0d751","ruler_category":null,"seed":null,"sequence_length":161},{"canonical_id":"919","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":40,"rank_sha256":"5a21a04a8534d8dff43c2d02da1415ffaabed4a09662059b05e916108da7665e","ruler_category":null,"seed":null,"sequence_length":113},{"canonical_id":"629","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":41,"rank_sha256":"5aeb994e6b892f1741b894b66750ea3690b71585304601a66a17d599e3e96bc5","ruler_category":null,"seed":null,"sequence_length":150},{"canonical_id":"820","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":42,"rank_sha256":"6063af147dd771453be5c51054092d7aa85f751eab9d697cc1b77be9259ed7ff","ruler_category":null,"seed":null,"sequence_length":105},{"canonical_id":"627","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":43,"rank_sha256":"608304ac3176b904f89e46ddf06a578afe6f80161ceb1adb2b5049c5b31a8c5a","ruler_category":null,"seed":null,"sequence_length":198},{"canonical_id":"755","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":44,"rank_sha256":"60b435df2cb81d92c2de2d1edc7b06a50a523b8d40047dd521cf241475d6b370","ruler_category":null,"seed":null,"sequence_length":209},{"canonical_id":"911","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":45,"rank_sha256":"614cef397364dc767d15426e06989a10147811417996eb4fa91721038ecac80c","ruler_category":null,"seed":null,"sequence_length":236},{"canonical_id":"874","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":46,"rank_sha256":"62fc38937c372269a01f02df68b088b1fdc004f7ae4590a50cb3673f385737d0","ruler_category":null,"seed":null,"sequence_length":153},{"canonical_id":"918","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":47,"rank_sha256":"6326299b9eaf024dfc7efa6b26bb898a5b869898127d546067800b05ba8c5932","ruler_category":null,"seed":null,"sequence_length":229},{"canonical_id":"803","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":48,"rank_sha256":"65df1000f86d2eacc5ecb7d9b91504d3c6074a9e9c5e6cc8abebfc45c0f60974","ruler_category":null,"seed":null,"sequence_length":142},{"canonical_id":"868","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":49,"rank_sha256":"69fda125d9dfbbac0a44eaf6db26953f3f644a70389add149dd1c47e3ca90bf4","ruler_category":null,"seed":null,"sequence_length":137},{"canonical_id":"666","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":50,"rank_sha256":"6c8674e41d1ec38f03b2d176e08aef538902ff0c35234be50164674f4e52a341","ruler_category":null,"seed":null,"sequence_length":108},{"canonical_id":"768","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":51,"rank_sha256":"71cb4040b63cf195e0e702fd08f6b6e9eeef03a3de4a3c29ca0a715a42e2fbf6","ruler_category":null,"seed":null,"sequence_length":144},{"canonical_id":"636","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":52,"rank_sha256":"7357df8332514f1e676d0104bd545961a71f432580a835738fb62ad7dbc89065","ruler_category":null,"seed":null,"sequence_length":127},{"canonical_id":"671","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":53,"rank_sha256":"7a7a9eda4414dd86e6dbd5ebf5e1ac84df7df6e12e3242cb2574817453c497b8","ruler_category":null,"seed":null,"sequence_length":193},{"canonical_id":"783","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":54,"rank_sha256":"7aabfc7187ec019772ef6f0f0ae5203e0193909146a86ecdc6a1f789f86ff6c3","ruler_category":null,"seed":null,"sequence_length":428},{"canonical_id":"695","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":55,"rank_sha256":"7ce9ea381ac2a57a8373c1253b0dfbedb3b2e838c4dfd92bb0730f83c972434c","ruler_category":null,"seed":null,"sequence_length":168},{"canonical_id":"851","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":56,"rank_sha256":"7df3d4757b59a5be15e6e4b9208bfbdbf189f4590d7b2e93ca0f84047f2a97b4","ruler_category":null,"seed":null,"sequence_length":131},{"canonical_id":"945","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":57,"rank_sha256":"7fdc834d39787ed9f97e73645f7df6692b711d910c52482f0bbab1d41512f720","ruler_category":null,"seed":null,"sequence_length":132},{"canonical_id":"687","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":58,"rank_sha256":"80a7b335ef9b11db152007e896afdd070710cbee60d4766ef829826eacbbf56b","ruler_category":null,"seed":null,"sequence_length":153},{"canonical_id":"908","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":59,"rank_sha256":"8267d3b36347390e9dbe928d4f2af728975e565b8c89232c8fe011b762fb9e75","ruler_category":null,"seed":null,"sequence_length":175},{"canonical_id":"973","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":60,"rank_sha256":"85f2e19defa2f2194e4bd627c43470d94dfce10775f216d0eee0f5b7c89a8113","ruler_category":null,"seed":null,"sequence_length":107},{"canonical_id":"795","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":61,"rank_sha256":"874c549c5d365887eba08a0eac638427d2825d7bedc3da4b4aa6c213de86162a","ruler_category":null,"seed":null,"sequence_length":325},{"canonical_id":"822","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":62,"rank_sha256":"87ee9e94899a058de1bc21331e5ffecb56f11ce924750fc73a6a928a2501d281","ruler_category":null,"seed":null,"sequence_length":203},{"canonical_id":"728","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":63,"rank_sha256":"884e088f56e16b57342c6b5d6b1021b0f7e756e9d8d1720ce65c243eb1c95acf","ruler_category":null,"seed":null,"sequence_length":165},{"canonical_id":"895","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":64,"rank_sha256":"888165087ffe22467a3cefc2c892a649a93905d8f22cd2389a8c7ba0be70f967","ruler_category":null,"seed":null,"sequence_length":293},{"canonical_id":"764","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":65,"rank_sha256":"8981728482ffcdb96388f812ddfe70ae76010f928900d65b3f4a8c8490ed266b","ruler_category":null,"seed":null,"sequence_length":125},{"canonical_id":"804","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":66,"rank_sha256":"8996abd03167db364eb3cf521940939970ca3b49310060c371c0e447dca9f4d4","ruler_category":null,"seed":null,"sequence_length":134},{"canonical_id":"732","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":67,"rank_sha256":"89d9cbbf3458bcc72f432dd40166313978822085bc657d2cc2ff5fe6919c27a8","ruler_category":null,"seed":null,"sequence_length":134},{"canonical_id":"839","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":68,"rank_sha256":"8c48b54163b5a2b5d91d303abe4d2f5990fd2296dd47e1960208c1d0286d9450","ruler_category":null,"seed":null,"sequence_length":365},{"canonical_id":"823","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":69,"rank_sha256":"8df1ac0f53587cc228a0d68d4959d98a79d5c3046a25094f36d1698d013a1a7a","ruler_category":null,"seed":null,"sequence_length":205},{"canonical_id":"713","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":70,"rank_sha256":"8eb162011c59bc92c4728ac7fe2fd64ac0f7f78f7aef1ba048088ed9743e16d2","ruler_category":null,"seed":null,"sequence_length":121},{"canonical_id":"772","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":71,"rank_sha256":"8f08bc003b61873c2e9f27eceb15bf7a3a451a8f4877c759cc73b5f437a170af","ruler_category":null,"seed":null,"sequence_length":166},{"canonical_id":"727","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":72,"rank_sha256":"925293e3739b4a2498fb5d6c9cbf9fbd57520f22f99a7b3468fad6290361cf36","ruler_category":null,"seed":null,"sequence_length":161},{"canonical_id":"858","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":73,"rank_sha256":"935132e4b17a56013a0312175973c9a6b2b092902d5bf1d474b03a34da4953f6","ruler_category":null,"seed":null,"sequence_length":177},{"canonical_id":"800","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":74,"rank_sha256":"9400093971582810aed9b43834286c1bbcdbe15e12fd1f749b9e3a7abbefa8f4","ruler_category":null,"seed":null,"sequence_length":104},{"canonical_id":"751","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":75,"rank_sha256":"96679ba0da36bb95058654f8fea49ff5305709d80cd95a1017d570f0d7997ddf","ruler_category":null,"seed":null,"sequence_length":243},{"canonical_id":"674","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":76,"rank_sha256":"96c806e69e6b65095699a2192e225cc22ad2b21211dbb8eae36893932ef358e1","ruler_category":null,"seed":null,"sequence_length":128},{"canonical_id":"960","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":77,"rank_sha256":"97692bbfd642b241043c3eb82fd110341e33df034bcc65a60b79b26933a15b3e","ruler_category":null,"seed":null,"sequence_length":124},{"canonical_id":"867","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":78,"rank_sha256":"979109f8d629072c0302399d65fa7263972fa9ed22cefff82f4c11028507238f","ruler_category":null,"seed":null,"sequence_length":170},{"canonical_id":"669","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":79,"rank_sha256":"98ddb8c36da9cd7adc3852209dcda61e9da6d3acd0d1fcf1211f25ae04c07cc5","ruler_category":null,"seed":null,"sequence_length":296},{"canonical_id":"656","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":80,"rank_sha256":"9979d6795117639a4d1f8afae4ddfa11dfeeefbf2ca8687a7e67b84643a1744f","ruler_category":null,"seed":null,"sequence_length":169},{"canonical_id":"862","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":81,"rank_sha256":"99e8fb364eec07a688b1d0dbc74ba9372c47719c0193b304c88bc2659ecd75c4","ruler_category":null,"seed":null,"sequence_length":177},{"canonical_id":"635","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":82,"rank_sha256":"9cf09b9e19663d0a700e97d8d627776519ef82d0d8608b62a5645b11309ec58f","ruler_category":null,"seed":null,"sequence_length":266},{"canonical_id":"756","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":83,"rank_sha256":"9d6ace56a87e526ec18af2d10e07c6dd1a3287a2d8271415ac4a0c3a3f642035","ruler_category":null,"seed":null,"sequence_length":139},{"canonical_id":"869","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":84,"rank_sha256":"9d975c8aee95cb7e1e54deeee4073d57a63712f67247e5fb61dcf791dbb16564","ruler_category":null,"seed":null,"sequence_length":352},{"canonical_id":"857","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":85,"rank_sha256":"9e00a569dd1098d4912c32f57e2351709bf3ffce9a75edf00179d5f023c41d41","ruler_category":null,"seed":null,"sequence_length":293},{"canonical_id":"607","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":86,"rank_sha256":"a18910fa12a223a0ecb7d6707500711e3b376af22c72b99625792f523b548d89","ruler_category":null,"seed":null,"sequence_length":207},{"canonical_id":"929","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":87,"rank_sha256":"a39d16127e92e6eb192ad63a45fa7db3125509f3a86fcc647d0a72ffdfc25483","ruler_category":null,"seed":null,"sequence_length":168},{"canonical_id":"606","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":88,"rank_sha256":"a4d325b0ce1088bd97fbaac9299e2d0dbb32b002026f2528825225d44f1e1e60","ruler_category":null,"seed":null,"sequence_length":144},{"canonical_id":"662","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":89,"rank_sha256":"a572f1763ed2dacfc5b6f8bafb04903bbc369519be95c8ecfd7d26a2792dc619","ruler_category":null,"seed":null,"sequence_length":350},{"canonical_id":"877","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":90,"rank_sha256":"a7576eaa33a3fb7087c29ae14949e5bfc2e1ef4449b997384e88a2d5dd631591","ruler_category":null,"seed":null,"sequence_length":93},{"canonical_id":"946","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":91,"rank_sha256":"a884dbdb72cb7d11613027848d08702a496c419bc5c987e9e963e2184c774bd8","ruler_category":null,"seed":null,"sequence_length":209},{"canonical_id":"785","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":92,"rank_sha256":"ab824ea47e78d00ae4aec6a45480e122e3703f1da3037053af0aa7a891b637ee","ruler_category":null,"seed":null,"sequence_length":152},{"canonical_id":"920","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":93,"rank_sha256":"ad357f573b8f19c5b89f84aa9097d23ace8d1488e9cb2c1322577137d0d53c2a","ruler_category":null,"seed":null,"sequence_length":251},{"canonical_id":"616","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":94,"rank_sha256":"b09c89c6a2366a5a51caa203e1f363d01e8340abcd1ca162839843b86eef1161","ruler_category":null,"seed":null,"sequence_length":221},{"canonical_id":"840","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":95,"rank_sha256":"b2be594c39afb592547a351b8235b8061e103c0c75d777ad1549d2e015a87989","ruler_category":null,"seed":null,"sequence_length":131},{"canonical_id":"644","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":96,"rank_sha256":"b6557529f8ec5764b47cd34b0b579eb20371c1c581831a55709bb7d1b27348af","ruler_category":null,"seed":null,"sequence_length":186},{"canonical_id":"704","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":97,"rank_sha256":"ba157c706c3d65835a4022b531b73e3f1b65c1660d0699fd7293821d2ebfe350","ruler_category":null,"seed":null,"sequence_length":153},{"canonical_id":"712","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":98,"rank_sha256":"bb6cd04968145b1904af1e96e70b74699cb7138a72c4c62e0b82ee87a3375fc0","ruler_category":null,"seed":null,"sequence_length":219},{"canonical_id":"882","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":99,"rank_sha256":"bf3e15adb4c326aab0f479085d1e9971db834476e9ecaf1a9c3d7764fec4378b","ruler_category":null,"seed":null,"sequence_length":112},{"canonical_id":"710","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":100,"rank_sha256":"bf5d27c709dca8c30c14453c4e20b7c75f1e93bd5952da6bbc741e43a68f0575","ruler_category":null,"seed":null,"sequence_length":163},{"canonical_id":"821","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":101,"rank_sha256":"c0ab0a4bb636c134ba2233ecc3d1925c81df2a89060c09a2908c9dddbc14b591","ruler_category":null,"seed":null,"sequence_length":282},{"canonical_id":"707","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":102,"rank_sha256":"c43bbdda480f2ebabe97769f2958f17a19febca5a53140948914ad1bdd0a6e3e","ruler_category":null,"seed":null,"sequence_length":200},{"canonical_id":"614","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":103,"rank_sha256":"c5450b1ef19a8eaf7e4710ad9115e76118a4634fb769d3924067bdec342e2e70","ruler_category":null,"seed":null,"sequence_length":165},{"canonical_id":"741","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":104,"rank_sha256":"c7c6fbada9b7a2d3d38750bc16cb99ea97efb71c0413563d3f399cc0722c6bcd","ruler_category":null,"seed":null,"sequence_length":124},{"canonical_id":"702","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":105,"rank_sha256":"c8a56538eebae04a29a21ac3c638e71a9ea53a1fdc12603034b61d9ef4359cc9","ruler_category":null,"seed":null,"sequence_length":332},{"canonical_id":"824","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":106,"rank_sha256":"c9106dec19a3fa31e33d340927464d15b6150cbc251a6aff64336706d0400b98","ruler_category":null,"seed":null,"sequence_length":129},{"canonical_id":"809","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":107,"rank_sha256":"ca59973992bc52f67689974ae41022e87d598c8c31de31955f30bfaf21889c3f","ruler_category":null,"seed":null,"sequence_length":173},{"canonical_id":"886","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":108,"rank_sha256":"cc1cdebfc08022bfe9fae59cf187db791a655ca5b48463b8c2725e74349fa708","ruler_category":null,"seed":null,"sequence_length":152},{"canonical_id":"789","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":109,"rank_sha256":"d71db894fab22f0f681f7e02a1ec80d922272c111bb9462b9d066866ab601c18","ruler_category":null,"seed":null,"sequence_length":98},{"canonical_id":"870","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":110,"rank_sha256":"dacc01a92b59eb20074bd1a8255fdbbc3298f37030e34e3d4e074c0bb66134ba","ruler_category":null,"seed":null,"sequence_length":208},{"canonical_id":"854","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":111,"rank_sha256":"dd5f39a3a28901c30d626c95276606c2a6ad00b0f64384c782406f9a13ef8d1e","ruler_category":null,"seed":null,"sequence_length":235},{"canonical_id":"646","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":112,"rank_sha256":"deb0492a1dc5a716095faeaf6d418cf2bf74040a83940a2690a6ea83442404de","ruler_category":null,"seed":null,"sequence_length":129},{"canonical_id":"605","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":113,"rank_sha256":"e161ff3b2e28c4f02462643895bc6e95b48655efc7ac0ace9eec5c4226576cb0","ruler_category":null,"seed":null,"sequence_length":125},{"canonical_id":"902","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":114,"rank_sha256":"e1ebf3f6386478a8625ad8c53858193b08d3e9a3ec267789cd98e85c340f314a","ruler_category":null,"seed":null,"sequence_length":310},{"canonical_id":"848","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":115,"rank_sha256":"e2bc9c9a9b23e33f1abf22dce38357e1e5c10acff55d2a208423270bc02a7289","ruler_category":null,"seed":null,"sequence_length":129},{"canonical_id":"847","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":116,"rank_sha256":"e351b9058c9e5c2e9e58444e1e4f2346c4886c6705652807fc2baab4f1806bd0","ruler_category":null,"seed":null,"sequence_length":143},{"canonical_id":"630","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":117,"rank_sha256":"e553afed5e6fb0d6843dfe2f22bdb6aa65d8f483b406e1b83ee5a2bf2e87fba2","ruler_category":null,"seed":null,"sequence_length":327},{"canonical_id":"793","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":118,"rank_sha256":"e76c7363df6aa1bca2a659469266d4228a66b4a2e10bf91c06346d0eaf0da89e","ruler_category":null,"seed":null,"sequence_length":209},{"canonical_id":"725","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":119,"rank_sha256":"ebb93bbe303910a142ceeda734226a9701884eaee00b68852b557cfc23fc231b","ruler_category":null,"seed":null,"sequence_length":164},{"canonical_id":"903","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":120,"rank_sha256":"ed33b084d9852e0df6989759582108080423613a5014d3f1150c3cbbc0d60573","ruler_category":null,"seed":null,"sequence_length":161},{"canonical_id":"797","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":121,"rank_sha256":"edb3a9f5d11a94af9db154ef9fee83d36402a77de27eb0ba830a3902dca5536c","ruler_category":null,"seed":null,"sequence_length":147},{"canonical_id":"792","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":122,"rank_sha256":"ee24c9054290bd8ccfeccee4993ac8dc6e0e20c72cdf4fbe5d28593c947a2246","ruler_category":null,"seed":null,"sequence_length":130},{"canonical_id":"615","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":123,"rank_sha256":"eed8e28b1d920cec5525bff2f0f861ca6e43429ed75ae657e5754ce34b81d71a","ruler_category":null,"seed":null,"sequence_length":338},{"canonical_id":"640","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":124,"rank_sha256":"f08156d84d873d2ae9184c8db61828a6c9a15f94b5a43b4ef8fe5c1867c2f16d","ruler_category":null,"seed":null,"sequence_length":118},{"canonical_id":"709","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":125,"rank_sha256":"f94bcd46daf0c1bd69bc60844bac9acd167ddc7981f10db96385b6d755574ec2","ruler_category":null,"seed":null,"sequence_length":390},{"canonical_id":"846","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":126,"rank_sha256":"fb81297eb57060d653be5ab5aa61ab169ac4bb236168fbda838bc333f61d4bc7","ruler_category":null,"seed":null,"sequence_length":305},{"canonical_id":"936","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":127,"rank_sha256":"fefd4316fc5fd98157422b169451622b064b3709b67d87d78e00278992813ea2","ruler_category":null,"seed":null,"sequence_length":297},{"canonical_id":"http://www.gutenberg.org/ebooks/1219","config":"default","configured_length":null,"group":"pg19","half":"a","rank":0,"rank_sha256":"061889c1d506cd1d5fb9d49ee4d63712d7e12e6056ffcc25a15ed400766bb5b8","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/38160","config":"default","configured_length":null,"group":"pg19","half":"b","rank":1,"rank_sha256":"10e82e9edc65238326568f332ff5556646b86707e652e24b051769eb8ad7fb02","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/39447","config":"default","configured_length":null,"group":"pg19","half":"a","rank":2,"rank_sha256":"14037bdad952830661d418776aaf4616905c877bfdb291977c357381de5c229c","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/29413","config":"default","configured_length":null,"group":"pg19","half":"b","rank":3,"rank_sha256":"217a13c587e9ae2e779aae14a77fa4a49dbbcbea34bfd717de53c2ac37ea9675","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/1253","config":"default","configured_length":null,"group":"pg19","half":"a","rank":4,"rank_sha256":"2a24d338d8456351047da3a9c12b7feb6c9e39c040ea62275c9284d4bc3b02f6","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/35786","config":"default","configured_length":null,"group":"pg19","half":"b","rank":5,"rank_sha256":"5b4d0ebce948cb01a07e9166f2db8c4d3163c3ce8780be96639120dc13fb26e2","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/17774","config":"default","configured_length":null,"group":"pg19","half":"a","rank":6,"rank_sha256":"5cfbeaf96d6b8bf5cbe8754a74cd22b63225e5c1775594c88f7ff24fc3be1d90","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/10609","config":"default","configured_length":null,"group":"pg19","half":"b","rank":7,"rank_sha256":"73a499a3165431d2fcf4fe5184ef91447b8184d5bf8f0683adb49a4bf058e0fb","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/20279","config":"default","configured_length":null,"group":"pg19","half":"a","rank":8,"rank_sha256":"7da541ed6b56ea9ddd7749aef82c4f66b36beb7ca9ebd5b3ab2623c60996cac9","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/28065","config":"default","configured_length":null,"group":"pg19","half":"b","rank":9,"rank_sha256":"beee6f7ab98cda2f3595fe22f5221f16fd4ce8717a7fe77344ea7e851c85287f","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/24903","config":"default","configured_length":null,"group":"pg19","half":"a","rank":10,"rank_sha256":"c4b007d21ff91cbeb1959618d1f954e864dc7b874ae3986598c860a30b425cc0","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/33622","config":"default","configured_length":null,"group":"pg19","half":"b","rank":11,"rank_sha256":"dcca53b6159ae6063a9e62f780ecd4130f39bf20f2342fe194d39248032cb43d","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/26008","config":"default","configured_length":null,"group":"pg19","half":"a","rank":12,"rank_sha256":"e5734f2587660eccf1f1cae256502a6d7f22fa8d8898f4dee591a3e3a931de27","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/36808","config":"default","configured_length":null,"group":"pg19","half":"b","rank":13,"rank_sha256":"e9713427a3a331bb20abf5840a4a40e37e72569131f89b7ab0691653bc679901","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/2773","config":"default","configured_length":null,"group":"pg19","half":"a","rank":14,"rank_sha256":"f0e4622ba87c3913d530c0f8f7f20c419b2375ff1ca073e6062914dbabe1a4f5","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/32024","config":"default","configured_length":null,"group":"pg19","half":"b","rank":15,"rank_sha256":"f8ecebbe43befef18b00a5780b7383b2a5e3971aca73ed9caf82dd87b7be9b58","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:fwe:length=2048:seed=12339:sample=0","config":"fwe","configured_length":2048,"group":"ruler:aggregation","half":"a","rank":0,"rank_sha256":"407456144cdf7e7f879eeb8c590fc8039a5de9c5d2a99e094f58c354dc7a2e14","ruler_category":"aggregation","seed":12339,"sequence_length":1951},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:cwe:length=4096:seed=12340:sample=0","config":"cwe","configured_length":4096,"group":"ruler:aggregation","half":"b","rank":1,"rank_sha256":"6eed2f87558224dfd68001e835c96568c05e5f827c533e157752f375f4500e19","ruler_category":"aggregation","seed":12340,"sequence_length":3576},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:fwe:length=4096:seed=12339:sample=0","config":"fwe","configured_length":4096,"group":"ruler:aggregation","half":"a","rank":2,"rank_sha256":"9ba9392d1dc6140c8a464c6ab01eae04fd883d8520b997db8d8e7a81ec69412a","ruler_category":"aggregation","seed":12339,"sequence_length":3987},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:cwe:length=2048:seed=12340:sample=0","config":"cwe","configured_length":2048,"group":"ruler:aggregation","half":"b","rank":3,"rank_sha256":"acfea4ad2d1ba514b26bd0835657efe402f76ae3d519c669004fe8f3398c41ba","ruler_category":"aggregation","seed":12340,"sequence_length":1862},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=2048:seed=12339:sample=0","config":"vt","configured_length":2048,"group":"ruler:multi_hop_tracing","half":"a","rank":0,"rank_sha256":"089158a73f7f141394a585c20b3642a57e30b83534a971bffe8d8cc58d1d88a0","ruler_category":"multi_hop_tracing","seed":12339,"sequence_length":2011},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=4096:seed=12339:sample=0","config":"vt","configured_length":4096,"group":"ruler:multi_hop_tracing","half":"b","rank":1,"rank_sha256":"0c5867a34d4c0109a91512496cb821eeda3eacfff858dc77cce30e9fdd0dec20","ruler_category":"multi_hop_tracing","seed":12339,"sequence_length":4056},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=2048:seed=12340:sample=0","config":"vt","configured_length":2048,"group":"ruler:multi_hop_tracing","half":"a","rank":2,"rank_sha256":"47daa45868dba636bc69249fc76653ed16cc4674127f6229ffb7b5c6e0dc816b","ruler_category":"multi_hop_tracing","seed":12340,"sequence_length":2009},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=4096:seed=12340:sample=0","config":"vt","configured_length":4096,"group":"ruler:multi_hop_tracing","half":"b","rank":3,"rank_sha256":"cea1514b80ab90b346af0afd72d4e596f64be107a08c70781a292c2824193d97","ruler_category":"multi_hop_tracing","seed":12340,"sequence_length":4057},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_2:length=2048:seed=12340:sample=0","config":"qa_2","configured_length":2048,"group":"ruler:question_answering","half":"a","rank":0,"rank_sha256":"362ee5e975e3602b3afddead8d433435ec23dbad4defa23337e192b538091cbc","ruler_category":"question_answering","seed":12340,"sequence_length":1893},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_2:length=4096:seed=12340:sample=0","config":"qa_2","configured_length":4096,"group":"ruler:question_answering","half":"b","rank":1,"rank_sha256":"4e8e5919e5dec4b8c1e7aed66e68abf8414441353d6669352f7bb5eb6f3560ba","ruler_category":"question_answering","seed":12340,"sequence_length":2281},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_1:length=2048:seed=12339:sample=0","config":"qa_1","configured_length":2048,"group":"ruler:question_answering","half":"a","rank":2,"rank_sha256":"743c29a00e4cbe2d6bb62a01300b447fc3a78ded5878e9c03e7d80686d62e6c3","ruler_category":"question_answering","seed":12339,"sequence_length":1368},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_1:length=4096:seed=12339:sample=0","config":"qa_1","configured_length":4096,"group":"ruler:question_answering","half":"b","rank":3,"rank_sha256":"ec18351546c72bc1cad5f950ced4793e70ee2fa618a341924278809a62774186","ruler_category":"question_answering","seed":12339,"sequence_length":4040},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_single_1:length=4096:seed=12339:sample=0","config":"niah_single_1","configured_length":4096,"group":"ruler:retrieval","half":"a","rank":0,"rank_sha256":"4a756a6ea055639254af12c8568b7afc07bb626029548efa51a12d72a0109efb","ruler_category":"retrieval","seed":12339,"sequence_length":3955},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multiquery:length=2048:seed=12339:sample=0","config":"niah_multiquery","configured_length":2048,"group":"ruler:retrieval","half":"b","rank":1,"rank_sha256":"5ac13727e634d2432f98b2670f23761bdef39b61ebfadcecff6aa6e1dc1966a9","ruler_category":"retrieval","seed":12339,"sequence_length":1289},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multikey_2:length=2048:seed=12340:sample=0","config":"niah_multikey_2","configured_length":2048,"group":"ruler:retrieval","half":"a","rank":2,"rank_sha256":"8deff28bb1380b3aa1be3b678ea1793381004620e9b4a67fea888e2a859cb53c","ruler_category":"retrieval","seed":12340,"sequence_length":1828},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multivalue:length=4096:seed=12340:sample=0","config":"niah_multivalue","configured_length":4096,"group":"ruler:retrieval","half":"b","rank":3,"rank_sha256":"b13ed20ccd6d9f0502b4d16468a25828441ad0e8b8e8063b1dd3f5d674a1815f","ruler_category":"retrieval","seed":12340,"sequence_length":3964}],"assignment_sha256":"a42cf4b332cc8cf58b27709d7d261fc03a356b27ec1c9ccd56914d99e60c1797","namespace":"recurquant.experiment013.calibration-split.v1\u0000"},"claim_boundary":"This artifact freezes Experiment 013 data, tokenizer, source, runtime, and metadata-only model-file identity. It is not quality, latency, novelty, state-of-the-art, deployment, or breakthrough evidence.","content_manifest_sha256":"ee72483a8f8b4370c9e667e4287747e5bc358aeb0265a58167140f4e780a7b29","datasets":[{"canonical_id_field":"task_id","canonical_id_manifest_sha256":"f46a3c772f1416c75bdd8668c69f4886a32a243c1737b13bbddfdc23b9605c70","config":"full","dataset_id":"google-research-datasets/mbpp","formatter_id":"recurquant.mbpp-prompt-code.v1","formatter_sha256":"882e20ec9f5cbcb7e6f1310cbf46d19153721beac41ecc7ee308c39be17532ff","key":"mbpp","revision":"4bb6404fdc6cacfda99d4ac4205087b89d32030c","split":"train"},{"canonical_id_field":"url","canonical_id_manifest_sha256":"8d7ed934210f5e78f13559cdf48518b922d3c203df9258d0de3c4d62a18bdaa7","config":"default","dataset_id":"emozilla/pg19","formatter_id":"recurquant.pg19-token-slice.v1","formatter_sha256":"faea2480bf85adcd34339cae88a0e9b631b705eb547020572db74402bf525730","key":"pg19","revision":"c021754c8e01c5b1cc83a1f549c1f97fbbb756b8","split":"train"},{"canonical_id_field":"configuration_id","canonical_id_manifest_sha256":"d7fec5f2a57409fb842dcb967de5ed1efce00253723f2d891b28681b8d57952d","config":"official-generator","dataset_id":"NVIDIA/RULER","formatter_id":"recurquant.ruler-official-generated-record.v1","formatter_sha256":"4f797cbca4dfbe802c47dc5b238111b629f1497e46c84ea116d06319012016b6","key":"ruler","revision":"c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a","split":"generated"},{"canonical_id_field":"task_id","canonical_id_manifest_sha256":"c53bc1db6c7965cb13d4238321ef09c0c7b3dbfeb4f4736bfd6e46e2914449da","config":"default","dataset_id":"evalplus/humanevalplus","formatter_id":"recurquant.humaneval-plus-prompt-solution.v1","formatter_sha256":"12204389715ddc210e5a8b1b291f4fbcaf2b64fde94c22699873de80d682204c","key":"humaneval_plus","revision":"d32357cf319e50e9c8d8dab5ea876c72b0fd321b","split":"test"}],"execution_bindings":{"calibration_runtime_manifest_file_sha256":"80ca233a29af4facbb334fd4fb51a4f6e9a3d6815465cb79b9f3db63ef668d6a","model_file_manifest_file_sha256":"586d9c7e520f3bbd99ecef30663bf07d283eb14622475c58891becd8e033b05c","parquet_materialization_manifest_file_sha256":"ee5628e50e5d3516fd79077542d355fd915455ac0e53128d372f4177ad63d39c","repository_source_manifest_file_sha256":"d948e71423534f97eef77215e260cd549122625d1ca424584b995a09fbabf96b"},"identity_only":true,"identity_schema":"recurquant.experiment013.identity-frozen.v5","model_contracts":{"conditional_scale_check":{"cold_start_peak_hbm_limit_bytes":8053063680,"id":"Qwen/Qwen3.5-2B-Base","revision":"b1485b2fa6dfa1287294f269f5fb618e03d52d7c"},"primary":{"id":"Qwen/Qwen3.5-0.8B-Base","revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68"},"weights_loaded":false},"phase":"calibration","promotion":{"candidate_canonical_evidence_sha256":"c77f520a2266b9aa7776591e12fe029beec60101aa97b7c7b594640d0d277d0c","candidate_file_sha256":"33243873c1e9f28509618cf29e15437b87e7d50dd74bc0e35f6ce432bf46cde0","explicit":true},"promotion_required":false,"protected_identity":{"ordinary_tests_may_read_protected_content":false,"stage_b_read":false,"stage_c_read":false},"record_count":160,"records":[{"anchor_manifest_sha256":"9727dad06649abe3aad4a15772fd49b9a0ae5c0ef55e61bba6ccfd0445ab8f73","anchor_positions":[7,15,23,32,40,48,56,65,73,81,89,98,106,114,122,131],"anchor_positions_sha256":"24769466c0c30738992bfaea31b322a83686427b8c8c2f42d5977ce98da1e86f","canonical_id":"945","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,31,39,47,55,64,72,80,88,96,104,112,120,129],"fisher_boundary_sha256":"98d4c6b32acad28c83342cab67b79f665cc5144d363b8e6653d190aedaddc45f","horizon":1,"input_positions":[8,16,24,32,40,48,56,65,73,81,89,97,105,113,121,130],"input_token_ids_sha256":"5d42f41e2436cb702ed1f8eb971b20537d123d7e1f41409244fd8dd8129656fc","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,33,41,49,57,66,74,82,90,98,106,114,122,131],"target_token_ids_sha256":"9a4cf735c9ab0560b97058fccd7a2d08e1a7cc3d126701d11007e641f736719b"},"formatted_content_sha256":"aa6feb35c931bc49027c89402eb6fe154ef150b92030a5f3e730756026205a4c","generator_receipt_sha256":null,"identity_record_sha256":"3c49ec4e6136f525892d68d4cf7267b90d1836a9011dc55e73ece42bc882546e","prompt_token_ids_sha256":"5766712b5c83365901b89092414948a16c2a894641c0580978a6720d74c7bb4b","ruler_category":null,"seed":null,"selection_rank":0,"selection_sha256":"006481602de32e6484f069e30640315b4902f198bc2b977318b0a90084e2503d","sequence_length":132,"sequence_token_ids_sha256":"91c8172e893e28a3e7c965916084be5fd76619bd9dc7881f155bfceacf710738","source_content_sha256":"dbd241865f0bf98d18ba09369665294382ef86e1be0440c9b18c083d5cb8529d","target_token_ids_sha256":"16acd83d1b28de14a0fcfcbad7f9a2538e0df11843ba645c1a55bcfeb285cecb","token_span":{"cache_exposed_start":132,"cache_exposed_stop":132,"prefill_start":0,"prefill_stop":112,"scored_start":112,"scored_stop":132},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2efa60ce513767dd377aebb493099ff3ebdc8a0362f30593887e54997aaa3ffc","anchor_positions":[8,18,27,37,47,56,66,76,85,95,104,114,124,133,143,153],"anchor_positions_sha256":"e227da918c51f69b24ec0ca4e8d85817a5330fb6e593d88068dce4e00d5d6d1c","canonical_id":"794","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,18,27,37,46,56,65,75,84,94,103,113,122,132,141,151],"fisher_boundary_sha256":"f690f6f75e34bb40e62a6b7fc6b3bb68573fc9ae30bacc4e60a641b17ec90391","horizon":1,"input_positions":[9,19,28,38,47,57,66,76,85,95,104,114,123,133,142,152],"input_token_ids_sha256":"bf4865a5ba31ed871b7bd8e3db64daa50d06ca08cbecab0fe5005d6b1a8347ad","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,20,29,39,48,58,67,77,86,96,105,115,124,134,143,153],"target_token_ids_sha256":"68567590e4faf2baa82a41a27758544c2001b1f8b881cbca709b0b85b1bb0ad4"},"formatted_content_sha256":"d8535b9c9deb650f87372133fd68f93792a2190a2ef9fe56ca55a611be6956e0","generator_receipt_sha256":null,"identity_record_sha256":"19a48bcd8ab87c826d9cf5e9160f127b2f1f1be8e45a068c8e44863d1648ee1e","prompt_token_ids_sha256":"b21de4f99224333b5f102dbb403f23e8367304e199e27380c91e192867b7cc1b","ruler_category":null,"seed":null,"selection_rank":1,"selection_sha256":"0172341ebe7bf74bb15297781b17837005ed9a9efbcb89d7bac34d2b25000aba","sequence_length":154,"sequence_token_ids_sha256":"b4da2809a0dead622eddd5200a89a8d4ffcc43b2c108c805e70f4449f813d222","source_content_sha256":"27e38eda1acb40e360eceaac200a88d7b2850555fa05a1efc123ec8c4a915f21","target_token_ids_sha256":"348ffb18662c2da5694709279e9caf22da5687654401cf9aa365e6382bba0ba8","token_span":{"cache_exposed_start":154,"cache_exposed_stop":154,"prefill_start":0,"prefill_stop":101,"scored_start":101,"scored_stop":154},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e0ef0af8afe4b6c4a81f3cabbd81a8ee7f213579fe41ab99f6103d4b0219b7e6","anchor_positions":[9,20,31,41,52,63,73,84,95,105,116,127,137,148,159,170],"anchor_positions_sha256":"fa5efe05ea2fd4bb11defcbff4d3edc28a1c8dc11911a2450816825e972052d2","canonical_id":"657","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,30,41,51,62,72,83,94,104,115,125,136,146,157,168],"fisher_boundary_sha256":"68596f206280b567de8e4d47c206f7bf68a9bb74b4435cf78f400f9f1ab7d29d","horizon":1,"input_positions":[10,21,31,42,52,63,73,84,95,105,116,126,137,147,158,169],"input_token_ids_sha256":"e3067d1464515f4ac7220f562d34c1c90289997d9fb5fefc653bd53f5f81f18e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,32,43,53,64,74,85,96,106,117,127,138,148,159,170],"target_token_ids_sha256":"a85ac1c61455daf925b47063d1ff1f66b29773ceec176e8263f5e6f25a1b3ea1"},"formatted_content_sha256":"058b7f8c5600efa023e5b2daf39dfaf3c3ed7a333e538e1af40e7b0b66c74221","generator_receipt_sha256":null,"identity_record_sha256":"53d315ac0c1c29a162fde0f0fcec7febb3700ec10fef28e6ed4749c118c383e8","prompt_token_ids_sha256":"d229c54edd42db7f254ae246f41c7e3e4207ee238c734a8f628e2b15ef6d2ea8","ruler_category":null,"seed":null,"selection_rank":2,"selection_sha256":"01c4ac54c87a838a046990fc691e469914fa8fffc3bd5230e6a5954f963ce09b","sequence_length":171,"sequence_token_ids_sha256":"2cc570399bb13417211fedf4c35131117f7d22b47aa4bd94e603f6303bbecf3d","source_content_sha256":"87238f15d4968d72a1caf68b3a7d6d09014ca1cd79cd8a42053ba1d0df51fe15","target_token_ids_sha256":"470c291e9349adfcd9620fddf232ec6756cdb6300d975612d2a94ad69e7deca4","token_span":{"cache_exposed_start":171,"cache_exposed_stop":171,"prefill_start":0,"prefill_stop":76,"scored_start":76,"scored_stop":171},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1a950a1c62dc0a9cadc655d3670aa54a1e6cb647d9a1892093d06959656b3aa9","anchor_positions":[19,40,61,82,102,123,144,165,185,206,227,248,268,289,310,331],"anchor_positions_sha256":"42314f586870267235ff8a21de13271c7bc12dbde0efe06732f5f12b3dd32240","canonical_id":"702","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[19,40,60,81,102,122,143,164,184,205,225,246,267,287,308,329],"fisher_boundary_sha256":"1c176e9d1164044fc7633185803af174aed6b6dde3205b6a93344da5dde70a67","horizon":1,"input_positions":[20,41,61,82,103,123,144,165,185,206,226,247,268,288,309,330],"input_token_ids_sha256":"6439978ab1e3336355d0e247c88f80b5fe217a10e6f08dbf7d49232dc4526902","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[21,42,62,83,104,124,145,166,186,207,227,248,269,289,310,331],"target_token_ids_sha256":"1e85d0fddb9002f2699f987057a43eb9c49e9fa5ef1f342f1e03060dd826b7df"},"formatted_content_sha256":"154dfb8b73bb69a8b67592d334c059c4ce3fa2a8dfe9716538e93896567d0906","generator_receipt_sha256":null,"identity_record_sha256":"08e19720136cb0b9e54d61f609f904ecacefee69429d33d7a9794d8b46085bf5","prompt_token_ids_sha256":"bddadf4e931724e7fd559f3f6b3d7128e8ca27b120efc2124cf05a63e7b2f6c4","ruler_category":null,"seed":null,"selection_rank":3,"selection_sha256":"01d2ae83b6bab88a4805a43141ed140fa1eb40a0d24f5579edffc9db44efe0e9","sequence_length":332,"sequence_token_ids_sha256":"91c517b2851f293a1c925d44556ecdd49cf6fc1874d095521f4dba829c5d5237","source_content_sha256":"97404d0bd18d3e30402ad3b46bee420986959afd67ef272b725a56c88634eec4","target_token_ids_sha256":"c94cc439fcffac9c155039ed067b4cf6d4dccca08e579f728aaf004b72be2d8e","token_span":{"cache_exposed_start":332,"cache_exposed_stop":332,"prefill_start":0,"prefill_stop":151,"scored_start":151,"scored_stop":332},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3ed130f8f695b8d6eb298a7742bd1582483350b3fd739da368175309b21ef0b1","anchor_positions":[8,18,28,37,47,57,66,76,86,95,105,115,124,134,144,154],"anchor_positions_sha256":"dda22fb55d9a1168e0db21fb1e43033ab66ab6d1c06af8fc27c118e4671afeeb","canonical_id":"651","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,18,27,37,46,56,65,75,85,94,104,113,123,132,142,152],"fisher_boundary_sha256":"9efbc09f1ef4234ec391fd7fee3a2a47a03dcf28666c0aa550bfc04ea6bcde0c","horizon":1,"input_positions":[9,19,28,38,47,57,66,76,86,95,105,114,124,133,143,153],"input_token_ids_sha256":"b5b770374e871c6af58aaed89d7fdf049c3423bc742e960e5f9a6f4e991faf81","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,20,29,39,48,58,67,77,87,96,106,115,125,134,144,154],"target_token_ids_sha256":"8271372b97b8f4205c2218769701468573205610c8d62b3474ad0d357b160840"},"formatted_content_sha256":"1ccb4c308ca379db3003d12f2a08486a59d19d9a104a1208f6f91af5104409dd","generator_receipt_sha256":null,"identity_record_sha256":"aa519005f371474709ebcac572071735625e58adcc2ea5484d987a610bd03f36","prompt_token_ids_sha256":"d3dc05d28fe582fdfab921a7ad3bac6d80d1d9368804e5955561df1f18f29e1d","ruler_category":null,"seed":null,"selection_rank":4,"selection_sha256":"030870e9654f434f272f49b61535aad38a9c0eacc093ccedb4b1328a163d00f7","sequence_length":155,"sequence_token_ids_sha256":"b4fc5ea0199d736cffaba0bfb4e5db058abd8417d844063350138206ca085d62","source_content_sha256":"c093103b7e2b0fc0cff523616e4dd14264349d2add37f8cc2e248dbd630be4f3","target_token_ids_sha256":"39e1f51bdd6ad42a474c0dde935c2ac6a6696b58b1ccc288e77f6bb849d587ca","token_span":{"cache_exposed_start":155,"cache_exposed_stop":155,"prefill_start":0,"prefill_stop":117,"scored_start":117,"scored_stop":155},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"30f57c28b1d21a916283d0e281b04bbb0087858a98279771835d257c5187478a","anchor_positions":[17,35,53,72,90,108,126,145,163,181,199,218,236,254,272,291],"anchor_positions_sha256":"8d9aa0aedd74a7302174328c9e98110efb9351fb25a7e9f001ca338898298fc3","canonical_id":"720","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,35,53,71,89,107,125,144,162,180,198,216,234,252,270,289],"fisher_boundary_sha256":"c47ca1eef3f1024a8ccaaa6a3156efd81373c0981d1b7794d0e74a6a4596fa0e","horizon":1,"input_positions":[18,36,54,72,90,108,126,145,163,181,199,217,235,253,271,290],"input_token_ids_sha256":"699513d81a38caf1a3166032551a6daa9d12868c8369d97465ec8d642a321141","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,37,55,73,91,109,127,146,164,182,200,218,236,254,272,291],"target_token_ids_sha256":"12845035b9a29a7b08b584abc1f28701bd58dc87b1360013b830d1863deed34d"},"formatted_content_sha256":"08569657d1fd1af97815b4671f162ad3cb646b9db6f4ade1b53b237267f3360c","generator_receipt_sha256":null,"identity_record_sha256":"4c68c0c7fd3b4f5ef68188ca20cfd016414d07212968996c98d64d097d36f013","prompt_token_ids_sha256":"41dcd0cbf8f176b6eabaa2189a2cc9dad19e1d0e1b7466a57fcee4fcd8695223","ruler_category":null,"seed":null,"selection_rank":5,"selection_sha256":"0331ac5ad147dee4ef983f58ff124f388c082a848eb8f7a1e2df5c58e3096b83","sequence_length":292,"sequence_token_ids_sha256":"f0361fd70f4b1f9d2a2cbb95c296333e1e9f146ad837ece7cb669c2993a27bab","source_content_sha256":"e916cd640a7665ee439bd983ca0769129de74af589e7e5eba3b559f7ea146c60","target_token_ids_sha256":"bf0037e9004b38848b545334c18c95d9234edfda314a7fc61ac5ba9fd36b25af","token_span":{"cache_exposed_start":292,"cache_exposed_stop":292,"prefill_start":0,"prefill_stop":240,"scored_start":240,"scored_stop":292},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"076f7a7d27bd0edfad63c949645fb3eea8f9d8d32c238b4af19053c5407a8b2b","anchor_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,160],"anchor_positions_sha256":"cea919870992245499fb8d8a858d848e1e76f02a4e2230ada935442533c03198","canonical_id":"903","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,18,28,38,48,58,68,78,88,98,108,118,128,138,148,158],"fisher_boundary_sha256":"dfb2af41b7aacbbc1cb03c5a460d70edf42a188290ce3b01ee741cbb663a7aef","horizon":1,"input_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,159],"input_token_ids_sha256":"8e0cb14d6334e3493c75e96c63848e991fa05b6bc027f04b99b09dd2cbddaf7b","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,160],"target_token_ids_sha256":"6800fb22eb04b0dcd2b08692779155cb065925a066e2f97f52600c51c38ed71b"},"formatted_content_sha256":"f296f132da28a9fa5eeea0a099d9ad97955044e13b9fe30f6d9300ff9e041c78","generator_receipt_sha256":null,"identity_record_sha256":"a64089c9832bde6f4f730dea47c3c93a3ac71dd538c45b2169607180571708d9","prompt_token_ids_sha256":"a454237da9066ead36df97766389f9b28ac1b5df54bd0d9073baa9459aae696d","ruler_category":null,"seed":null,"selection_rank":6,"selection_sha256":"03a537026beb3ae778851b82f172083dc21eb07f0ce7d1683756019ad86e6d98","sequence_length":161,"sequence_token_ids_sha256":"86cdcf0d01cc99b913a8495e4074583c23687d2a77e68c6b0cda5700932a8937","source_content_sha256":"edfb4574ba6b7cb41c0f0d1dc6452e8ac7fea922b120c3ad057a2e77b15fb36e","target_token_ids_sha256":"5fb56e9b4dfe8c41e4ff0043461c0b44248cf165cac01b9f42beb4f0e756f516","token_span":{"cache_exposed_start":161,"cache_exposed_stop":161,"prefill_start":0,"prefill_stop":83,"scored_start":83,"scored_stop":161},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"6ed1f32d878fa277d86681ed3b0a9e7b9b172c3cf90d4213020217eb3296cf68","anchor_positions":[13,27,41,56,70,84,99,113,127,142,156,170,185,199,213,228],"anchor_positions_sha256":"e3fc0847eae8451ba50c7fae586b6479de5eec6dd1bcdf44504db22f0d700d62","canonical_id":"918","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[13,27,41,55,69,84,98,112,126,140,155,169,183,197,211,226],"fisher_boundary_sha256":"53c76b32efa019cb22ef480e27bc5b9dc7c12c0683b0d1b8ad250ac6571380a9","horizon":1,"input_positions":[14,28,42,56,70,85,99,113,127,141,156,170,184,198,212,227],"input_token_ids_sha256":"14e214e41d049733a9ea7e8c02fce42e7aba76edb16178e6d3405bad4c23055c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[15,29,43,57,71,86,100,114,128,142,157,171,185,199,213,228],"target_token_ids_sha256":"24028e01155b1e01eca8ccfc7c2080a6a56b4a3e7c15839043e13b8a00a8ee8b"},"formatted_content_sha256":"f4ed4188dc61ab3e90526a061246e459b1d5fe2fc68d01226a124501eec0be2b","generator_receipt_sha256":null,"identity_record_sha256":"4f00e2535ec2aa52070ed18120472b9d45845790809ac1fb5a7a26a8f6b51d20","prompt_token_ids_sha256":"d8f70964c5c840cc4c74b02a5c432adb08de3a38f706cca7ff9f50f513f616e7","ruler_category":null,"seed":null,"selection_rank":7,"selection_sha256":"03f2b70cb41aa2df2ca06599a3740907c1bd89976be4f7d79613cd3428eb75d7","sequence_length":229,"sequence_token_ids_sha256":"b42ae62c8481ac868fb8f833b03b35ab7f84490d87329022e4548df9b8a49299","source_content_sha256":"1c74c6e9767910ee576993d40b0be7e4f8a3653223ce943b2b48da881f043551","target_token_ids_sha256":"c0130989066437b78efb8eb68c747a3645ecb631cb508775b1f7b001deb4e1e0","token_span":{"cache_exposed_start":229,"cache_exposed_stop":229,"prefill_start":0,"prefill_stop":96,"scored_start":96,"scored_stop":229},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"32d8375f975ffd5157bb123bebf58d926b2fb6e715d79b1c29711866e8ae1d7a","anchor_positions":[11,24,37,50,63,75,88,101,114,127,139,152,165,178,191,204],"anchor_positions_sha256":"48082390de232d2e911509462169a4b3ebe8a03329ea579a66fe38623c52b3bd","canonical_id":"823","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,49,62,75,87,100,113,125,138,151,163,176,189,202],"fisher_boundary_sha256":"ca339ac21962b12f69ea2e43cd5efc5fc9f045e43e311f236cff0f3fc7a9532c","horizon":1,"input_positions":[12,25,38,50,63,76,88,101,114,126,139,152,164,177,190,203],"input_token_ids_sha256":"9060d0dc9e82be8da8cc47571caa8e03245ea38a15689377bf15f3403304608c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,51,64,77,89,102,115,127,140,153,165,178,191,204],"target_token_ids_sha256":"9c64639807a2aba57e44b252f7c9204c7994ff505668345c293c352b795673ab"},"formatted_content_sha256":"72e3934a85cf2d4cb4b035ef066f9a69edbe304668167f633c705ac5a7c3ee74","generator_receipt_sha256":null,"identity_record_sha256":"6775751ca115709c8dd0187f2d2309b90237ea599613ee56420cc077c5a86d67","prompt_token_ids_sha256":"a7322a6a594060b8983da2adc969063889ad65c9524cfc9d9ace5e4856362612","ruler_category":null,"seed":null,"selection_rank":8,"selection_sha256":"04318f2e0a32de485412bc0aa95414984aabdb5973974b77667909ca919923cf","sequence_length":205,"sequence_token_ids_sha256":"c39424dfee203bd58c46a3bbaafc51b57eb8c76c9f8ad175fd4355d227510f19","source_content_sha256":"7be1174fc06994281445e57b76099586c2214066f9358928e22281d793715d3f","target_token_ids_sha256":"c0ca3637b8b1da8af549e87df7bf8e67da2eb444227889016f2678ec5d8b420b","token_span":{"cache_exposed_start":205,"cache_exposed_stop":205,"prefill_start":0,"prefill_stop":120,"scored_start":120,"scored_stop":205},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"bb690c1557693c9a1151a55a056b413abe207a0ede6130a1c4eee76cd7832910","anchor_positions":[8,17,26,36,45,54,64,73,82,92,101,110,120,129,138,148],"anchor_positions_sha256":"34299c685b72af2a076ee41a9cc49b613c0e3c8039202f3412e50762385cd13f","canonical_id":"962","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,26,35,44,54,63,72,81,90,100,109,118,127,136,146],"fisher_boundary_sha256":"f483b9a7b54d8e98c4fb81ac8e8c508a85638477f08243915cc53d3ee584a92e","horizon":1,"input_positions":[9,18,27,36,45,55,64,73,82,91,101,110,119,128,137,147],"input_token_ids_sha256":"562a3beb47bbf20a72e84b88e739d67bf0a6fd83ea3ec4a06ce7d8527bc32c9d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,28,37,46,56,65,74,83,92,102,111,120,129,138,148],"target_token_ids_sha256":"8d91bcd0f3417c6e6ff0e30b8ef2ad4a5200ab497081407b0de689fac826fe32"},"formatted_content_sha256":"35f9c6bf72383681061847ec15dc2a4d69a179546f872d72042b1dbeb7eb3c94","generator_receipt_sha256":null,"identity_record_sha256":"7f528a9f6b6f25735216e2ddd0cf3bc970efee42f4ddc18607d7615c585b5da5","prompt_token_ids_sha256":"ad2690a82eb727187859d0918f78622936f9220218fa708cc305b79f9e3129b2","ruler_category":null,"seed":null,"selection_rank":9,"selection_sha256":"053a8089119cb6c0a1699960c729b5d0bbf904347c2611f2bdbc20d68ac98896","sequence_length":149,"sequence_token_ids_sha256":"39192da46d67ba8a6a1aecc11d2517cb72d4adbd21944f31d854f4aebde4f950","source_content_sha256":"6d510f0a092393c876a9d3827db9cccd4b422742e08dfee9a9094f3786b67fbf","target_token_ids_sha256":"24d4ddf4dc36fb254790754f2826967503025402abfe4dc1c81a57b608e358c1","token_span":{"cache_exposed_start":149,"cache_exposed_stop":149,"prefill_start":0,"prefill_stop":87,"scored_start":87,"scored_stop":149},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"510bbf0acc641074d0ed15a7484a726b97a35b663fe88ff076ca0808ea574c8f","anchor_positions":[5,12,18,25,31,38,44,51,57,64,70,77,83,90,96,103],"anchor_positions_sha256":"307c6a1944f2d94c59465c6af96e251aaf0ea665f8f1a3e0e55e3eb0568d8551","canonical_id":"641","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,11,18,24,30,37,43,50,56,62,69,75,81,88,94,101],"fisher_boundary_sha256":"74d1a793166b47319560030b1ea4243c84cca2ef770c4bd539a7069d57a6516c","horizon":1,"input_positions":[6,12,19,25,31,38,44,51,57,63,70,76,82,89,95,102],"input_token_ids_sha256":"693ad260a9464069c62f03bdc9f97044efaadaf73fba426e3993f726d373516e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,13,20,26,32,39,45,52,58,64,71,77,83,90,96,103],"target_token_ids_sha256":"24fc6cf6c761d9d2d39f6a0e2d1988fd8c0568a28c602b83a298a2efe14f5b75"},"formatted_content_sha256":"0d5d8389772326b9a23b37c5d64023b87ff76167c738f0f04a73f350d75146a1","generator_receipt_sha256":null,"identity_record_sha256":"d257e6e664de7884c47c6c8efd30d9049f908be0c1f0f15604c5b1e680278db7","prompt_token_ids_sha256":"8b73365de43af24e2b250e80fbb6c87c821ce915961cb60bb5f92dd492aa4e25","ruler_category":null,"seed":null,"selection_rank":10,"selection_sha256":"0561b3053aedb5bf75595d41c4c1d56dffc9d7d22d4f594c77acd661612fbbda","sequence_length":104,"sequence_token_ids_sha256":"edaafb16efe7861b6641b4284c1b03cabf4553b33c20400911590227e2717873","source_content_sha256":"b4af7687b1fdeeba11f23fbb43c11d07ffdb4918405f0574dba17f6932087d51","target_token_ids_sha256":"10c4c8e1ab1664e434fe26b8093fe1929571437707a4a4c5082c71bb6879f108","token_span":{"cache_exposed_start":104,"cache_exposed_stop":104,"prefill_start":0,"prefill_stop":80,"scored_start":80,"scored_stop":104},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"77e62d656a470f85ca90c98dfc91b52de62fec0d714ab54d9d78b6111588449a","anchor_positions":[16,34,51,69,87,104,122,140,157,175,192,210,228,245,263,281],"anchor_positions_sha256":"d3c1ef41c3fc59649a4da8be56fc51bf8a16621374c8218a7236e1c502a9ed29","canonical_id":"821","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[16,34,51,69,86,104,121,139,156,174,191,209,226,244,261,279],"fisher_boundary_sha256":"6ce814085aa258c14254daa52b6c87e5c35bb95bb9ce43819236089ed670a938","horizon":1,"input_positions":[17,35,52,70,87,105,122,140,157,175,192,210,227,245,262,280],"input_token_ids_sha256":"52e2a2814003d5a0c9558d2807c349140c1daee27300a41a386f98243a887244","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[18,36,53,71,88,106,123,141,158,176,193,211,228,246,263,281],"target_token_ids_sha256":"a841d33aff758b3642820a6e5c0b714add3840501814ab31d90f6dc5b41336dd"},"formatted_content_sha256":"9b4c39a7f47e7c5485594bfc1f0bc255be70f74e8e73be2bfab32e77bd11cb37","generator_receipt_sha256":null,"identity_record_sha256":"bbe20a7f23dfdd53afdd97893a491105ab3d0d770f28da4cafd4d0f760a10e07","prompt_token_ids_sha256":"8438715e3ee27d1c3e5e158afeb22e85f9543635b4c24c7151828b62262082b1","ruler_category":null,"seed":null,"selection_rank":11,"selection_sha256":"05ca9ce71ec3ccbff2dc9051e6519ff621aeaf051f13006ef2a903db53cfa322","sequence_length":282,"sequence_token_ids_sha256":"d4a708e94f01dc74ebdbaa4a0850da22f19f8c7616df125de0fa93a03cd941d1","source_content_sha256":"6cdab710688743670db0377d5ff8806298470ee86aa94a2f4fd430f2542e26b4","target_token_ids_sha256":"52fee9ad2633a9725f62006b3a2a598f494e32fd6e12dcf43893a911e7032a91","token_span":{"cache_exposed_start":282,"cache_exposed_stop":282,"prefill_start":0,"prefill_stop":246,"scored_start":246,"scored_stop":282},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"88e7f4c26333024cf786b4c1d640f43c25aadabb8055c4006984b0b773283a63","anchor_positions":[6,13,20,27,34,41,48,55,62,69,76,83,90,97,104,111],"anchor_positions_sha256":"70a9c6d084ac7d3e411adc69adf277b706895b48989c28a96164561595a44b13","canonical_id":"882","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,19,26,33,40,47,54,60,67,74,81,88,95,102,109],"fisher_boundary_sha256":"af33587dbbed535e25e0f9ff381dc426652e98f76482f907c3a1b91a43afe65b","horizon":1,"input_positions":[6,13,20,27,34,41,48,55,61,68,75,82,89,96,103,110],"input_token_ids_sha256":"2e82b13f62352679ed5dae9b03694e1c2eeb56e00f762a0373672ed48440f408","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,21,28,35,42,49,56,62,69,76,83,90,97,104,111],"target_token_ids_sha256":"d9c7534eb6dbff16502f2748b6ab6576ab5d76645faba1e8bc35a67323be8c39"},"formatted_content_sha256":"8164f8fb3728db418776886f3233778e01a1c5b7e54b4f350ff9c98d6f4c3018","generator_receipt_sha256":null,"identity_record_sha256":"79d822ab45c87b51b8aa2c0ecb29bfbbd85453f82f6a3e764bec8805d2f9708b","prompt_token_ids_sha256":"4c61f175c320b5f738697b92438e1bb158f00bd6a061e49a529ad3158d4f02d5","ruler_category":null,"seed":null,"selection_rank":12,"selection_sha256":"06182bf7b5db58b26f203c4f99a7fb97d3252f5d92dd5f4333d47e49043b0f9d","sequence_length":112,"sequence_token_ids_sha256":"a69b8ee713bb9e4fdf0cdec61d49a10f148243a2cf8fe92a4420823b82bb3839","source_content_sha256":"f9edf8d912223ba1c68389438b5a930e3ff8116a79ef7e9e4ab5b7f48cad2e1e","target_token_ids_sha256":"b9330c3608f52265a4d54352f0d4d3821513a3adbea5b89bf9f5146143a012fd","token_span":{"cache_exposed_start":112,"cache_exposed_stop":112,"prefill_start":0,"prefill_stop":90,"scored_start":90,"scored_stop":112},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"026a11f123c6c02b349d154973681d1484316586c77571635cb75396928e8801","anchor_positions":[8,18,27,37,46,56,65,75,85,94,104,113,123,132,142,152],"anchor_positions_sha256":"75c1fd0cbacbbfa75bfedc1647ca1a15436ad446c2021b146a8d7af4567321f4","canonical_id":"874","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,27,36,46,55,65,74,83,93,102,112,121,131,140,150],"fisher_boundary_sha256":"69de8bf124376d38cfd76f8801bc74e0e352d8cd33f8dc6b34cb10e4c6ff73d3","horizon":1,"input_positions":[9,18,28,37,47,56,66,75,84,94,103,113,122,132,141,151],"input_token_ids_sha256":"75967a14501363b155570c3d4d66959cde5cce50c53c8fa1fadf71f39e4218de","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,29,38,48,57,67,76,85,95,104,114,123,133,142,152],"target_token_ids_sha256":"38400510781c2bec0042957302068be57d0438f8b57f3a4a775204a7ae26b03b"},"formatted_content_sha256":"8ee29791043cd7726bdd815083a307f0d693443f77d0fc9af6c9249422f82a60","generator_receipt_sha256":null,"identity_record_sha256":"5f505f1a07e986bb3a69f547553e68edda687e89817230981e3d65a20f3b5d61","prompt_token_ids_sha256":"af780c9a24c687de5d7bb86c8de2e3a05d07e1e769e2bed9aef9bc82e0c2b3e0","ruler_category":null,"seed":null,"selection_rank":13,"selection_sha256":"06e6c099b79a31351c8aa115e8c35681452320d265ba3e7c9deb647189b5aedf","sequence_length":153,"sequence_token_ids_sha256":"059517b1e6703c7276b532d66d5dc53b137178142a0fbcebbf7a85ed806bb2df","source_content_sha256":"7be81d34f54ea71fa53a94438dc656b1e14cf83c1f0cb8b85a03aeec675a95cb","target_token_ids_sha256":"a00664d6a993b427df9b77218f9f6ab241e98e255c2754127a6aeb30818bf82b","token_span":{"cache_exposed_start":153,"cache_exposed_stop":153,"prefill_start":0,"prefill_stop":82,"scored_start":82,"scored_stop":153},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7e8e992a57a2c58864e789d43316ec4ca5557685022402a123eb7bec86e0a31d","anchor_positions":[8,17,26,35,44,53,62,71,80,89,98,107,116,125,134,143],"anchor_positions_sha256":"486e7f47d72e2b3d4a862156a2b41e9dbb05a70d78349fdcac0b01e09667750a","canonical_id":"768","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,25,34,43,52,61,70,78,87,96,105,114,123,132,141],"fisher_boundary_sha256":"2f57f4c3f66cd0d31d7220f66c2563c2f42ed9d860ccd032cab629a4a62b2d7d","horizon":1,"input_positions":[8,17,26,35,44,53,62,71,79,88,97,106,115,124,133,142],"input_token_ids_sha256":"951070a8ec9f2842c988081eae148ff3e650f6e041bad423eb8cf80577aa09ca","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,27,36,45,54,63,72,80,89,98,107,116,125,134,143],"target_token_ids_sha256":"be7a411acd1f8c53e98dbbe661a8f0e08925a5287a3a082d567b4348f5daea4a"},"formatted_content_sha256":"19fb2a15b1650f22847079c098721ed0b4ae36db1faae49cb10e7f8b7f8a951c","generator_receipt_sha256":null,"identity_record_sha256":"4f37dd32d3554ba3f0e3842355c0812d1532bf5eeb4d09edf28e778ec744907b","prompt_token_ids_sha256":"5abd62e58720f9b71da73a2f319d99d1dd3f0fe89c148c1e335674d33e0db6bc","ruler_category":null,"seed":null,"selection_rank":14,"selection_sha256":"08d405d25f1c1656f2532ca7d15875b63c46516c4e46a564326b3696f5943347","sequence_length":144,"sequence_token_ids_sha256":"d924aa1344bedd7c5de7beb28914b5f02d01c9242e93916c3ebf5e13a1b777e2","source_content_sha256":"96a4ff323c23dc1eba72d4b800a65cd77f46bad4c7b017e7716f69b066cf251e","target_token_ids_sha256":"2bb90e5895fb397d1b3239e2f50aab932ff08813656890b44cefc5327ce9f0fc","token_span":{"cache_exposed_start":144,"cache_exposed_stop":144,"prefill_start":0,"prefill_stop":79,"scored_start":79,"scored_stop":144},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2a32d45497a46d8765c2bef47acddb45c96e41bdb29282abb99a946dd86db58b","anchor_positions":[15,32,48,65,82,98,115,132,148,165,181,198,215,231,248,265],"anchor_positions_sha256":"0348f2e9e936ebff365da2919e86c0d7b23f030deacd2557e2975ff2c1bb6238","canonical_id":"635","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[15,32,48,65,81,98,114,131,147,164,180,197,213,230,246,263],"fisher_boundary_sha256":"bbb2193a791665a372d6f0871c8b972270ed33e5afe4d627b3f13c69caa5172e","horizon":1,"input_positions":[16,33,49,66,82,99,115,132,148,165,181,198,214,231,247,264],"input_token_ids_sha256":"54ae9eda3529af89c8491dfa14c002e2bbbddacc5f7fc9bd8f98dfd841106b17","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[17,34,50,67,83,100,116,133,149,166,182,199,215,232,248,265],"target_token_ids_sha256":"8e9dff8e5415e27d61633f2fbae7b6f5a404922b80ae0dc2d994004e0c00ff77"},"formatted_content_sha256":"252812434def51f2af7fe0f324a5922cddb2523bd81d2dfc65720c67ba9d9871","generator_receipt_sha256":null,"identity_record_sha256":"e2908b95f86b0b763a5d77e597f5d8cc0e9ff3b96e4129645e8ee178160dc9e8","prompt_token_ids_sha256":"99adc1c15eb492246a3ad45eb9cd6dc0a5f949c95b99aa332c2c8fe742dac882","ruler_category":null,"seed":null,"selection_rank":15,"selection_sha256":"09bdeb42fd2df571b9c13e3aecc461eec43355320ee267a53066bcbb23769161","sequence_length":266,"sequence_token_ids_sha256":"2ca3fac260d1b58347c86dbb21f794efef0df7c42a0de4d942d549481c583b2d","source_content_sha256":"b3bbd3ae7ff06e057b52c17af8b75ac83d20e8b436ca1c82f6f7ccf94d0a9214","target_token_ids_sha256":"b44b18d625486bf3a519bc0a87d4dbfbfb4851798e7e50bacb07a66cab69939c","token_span":{"cache_exposed_start":266,"cache_exposed_stop":266,"prefill_start":0,"prefill_stop":214,"scored_start":214,"scored_stop":266},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e6416dbb61ce7ce16cf3fff7ce25a1bacfecbb3dbf2fde6f06d8fc6699998cd6","anchor_positions":[5,12,19,26,32,39,46,53,59,66,73,80,86,93,100,107],"anchor_positions_sha256":"b9647c4baef8fee686d8a882e4d530418b6f7cc8044a0a83370fa394c3bc5ef2","canonical_id":"666","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,18,25,32,38,45,52,58,65,71,78,85,91,98,105],"fisher_boundary_sha256":"1a644132ebeecd04edf2c98217977f7fcbdedbec84e64b6b50a88a6151232772","horizon":1,"input_positions":[6,13,19,26,33,39,46,53,59,66,72,79,86,92,99,106],"input_token_ids_sha256":"32f5ac39d606f1ddf6a4e4346735931cd6faad7feec35e950f17b8edbcb045a2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,20,27,34,40,47,54,60,67,73,80,87,93,100,107],"target_token_ids_sha256":"eed792902f0c9e240b65aaacd0467e2ff2f4a7eba83a333214fd67d942feac55"},"formatted_content_sha256":"e3a3c1e1ed1b292d4877ff78da3ca92b8112f6b28b41a2eb63f18c8d72057088","generator_receipt_sha256":null,"identity_record_sha256":"ec5fb87c296ab7d9fc8671afd53e5bc11fd05c7ee0bf20f88d758453cb95da2d","prompt_token_ids_sha256":"92348f208743eebc810c3eeffd27e797bfbebe693b75b29192878c2766c17cb5","ruler_category":null,"seed":null,"selection_rank":16,"selection_sha256":"0ad2bb04e64cf7bafe43963d9c975c7597c00f97a065f172225e36b8de1be401","sequence_length":108,"sequence_token_ids_sha256":"05aa7077f2114bfc39c337134a54171570e1bf7f5e40e875dfd4ccc76cd3c5bd","source_content_sha256":"c5657a0fc079e76b566db9dad6a041aa51a885e91a3ce60f4b56e268d4f2283e","target_token_ids_sha256":"34eea4ea001c4d78cc33cc3368d1d71873c37b7e910507eceb9597f35255e3ef","token_span":{"cache_exposed_start":108,"cache_exposed_stop":108,"prefill_start":0,"prefill_stop":69,"scored_start":69,"scored_stop":108},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b641a12066174c147da77baa5576307cddc4326d490da447977a9880c63d2be5","anchor_positions":[19,39,59,80,100,120,141,161,181,202,222,242,263,283,303,324],"anchor_positions_sha256":"229481c1cae947d75547887104b565fe100ef01406aaa1ae33b6680fa4942d03","canonical_id":"795","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[19,39,59,79,99,120,140,160,180,200,221,241,261,281,301,322],"fisher_boundary_sha256":"075b4606ad628cb7bea3693f59ccdb0efa6c6599cce58a6552534c47baf39ab1","horizon":1,"input_positions":[20,40,60,80,100,121,141,161,181,201,222,242,262,282,302,323],"input_token_ids_sha256":"23b5ebefaddbeeafb86642c66656119a5122ac4590adfc291caa30b36d9aa0f0","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[21,41,61,81,101,122,142,162,182,202,223,243,263,283,303,324],"target_token_ids_sha256":"67fe7ce1a7b687bd4b008457e44ac3e68602c818d9241e2c029c21d035fb039c"},"formatted_content_sha256":"ba441afe1323472e606fbc265ed2df392c0910aa7b69c72034eb237f6f1eac12","generator_receipt_sha256":null,"identity_record_sha256":"462faa7c85aef3e98a366b53fff20c26ce10eaccece6d0cf5e990a957d47bab0","prompt_token_ids_sha256":"276d4bcbbf513e902b3e32cea17826db3f7c89340cf3d4a5944f77e4fe398378","ruler_category":null,"seed":null,"selection_rank":17,"selection_sha256":"0b64b061a95be11408788bd7d48178a22fe67b04451aacb2824273de604613c6","sequence_length":325,"sequence_token_ids_sha256":"3bed81ccd0aff452ebe2792b7538c8dab5d74a0ddf5fc9de5e67201d559dd9f4","source_content_sha256":"0c82e760835310eb8ee865c643c6bbd2fe9a92e58f43e818f9fe857fa8bb2b34","target_token_ids_sha256":"07898ad18dc9917a62d1d7d2c673264e506ffa3cd0296f61f5355d73ff74e2a1","token_span":{"cache_exposed_start":325,"cache_exposed_stop":325,"prefill_start":0,"prefill_stop":290,"scored_start":290,"scored_stop":325},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"73863421843464e12bf37b9bdf19118b2a0edae3ea6b28b7bb83ae34025fdb99","anchor_positions":[6,13,20,28,35,42,50,57,64,72,79,86,94,101,108,116],"anchor_positions_sha256":"65c4371e0db2357f72608fb79361ad62ba950063eae1f1df95567c3178cfceb4","canonical_id":"944","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,13,20,27,34,42,49,56,63,70,78,85,92,99,106,114],"fisher_boundary_sha256":"8f181fb4592d87100a01b9b5022fe28dec60a6c1a630f0253a95ef609f751477","horizon":1,"input_positions":[7,14,21,28,35,43,50,57,64,71,79,86,93,100,107,115],"input_token_ids_sha256":"bb992854b746a2d0322b0c73747a5b6e481b4ac55013af85dda8fc8c0bc4e324","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,15,22,29,36,44,51,58,65,72,80,87,94,101,108,116],"target_token_ids_sha256":"efc4327e8bc745b94eb74df47cb9d34c605ce69f88ab01258f192e2678c5c5c8"},"formatted_content_sha256":"588cb6ded28fd05719867c9a3c2ef078440822880f6fbe41587f8ab604fc41cf","generator_receipt_sha256":null,"identity_record_sha256":"a019ad696f9c4fe577d6efe7e08f14e983f53e6a023062d550dc4ffd16fef99d","prompt_token_ids_sha256":"6430dca518b59b80172b9552c64bfa32b6116fc5855d53c075e496d174d33f72","ruler_category":null,"seed":null,"selection_rank":18,"selection_sha256":"0b9a44c3022b9f3f3616c768c1535452329656aeea799dcb45a719751b3126e4","sequence_length":117,"sequence_token_ids_sha256":"a3b8d96a49a5559eb5401f9e9f9b40c0785ee84f73cf189a884618ba0cc77865","source_content_sha256":"2099f30aac30b18f4a8fa27a7204c64734c587cf50057d835eae933211f12c7c","target_token_ids_sha256":"73123ac6d971eeb25450ba23964d4d95870b3421781c2b721b8e3f5217fe1ffd","token_span":{"cache_exposed_start":117,"cache_exposed_stop":117,"prefill_start":0,"prefill_stop":91,"scored_start":91,"scored_stop":117},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"a3fb6da730653e3034c80d7d7da56cd1a27244994198f932d4b653175e07528d","anchor_positions":[16,34,52,70,88,106,124,142,160,178,196,214,232,250,268,286],"anchor_positions_sha256":"88ca5e0c73dcbbad356df2a5c320d1e65852f5459c97030e41b86ba39ef13c38","canonical_id":"653","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[16,34,52,70,88,105,123,141,159,177,194,212,230,248,266,284],"fisher_boundary_sha256":"f3b0c45ebf98940a7a54cb33ffec71487e06a0482e824289ffd032de00327ea1","horizon":1,"input_positions":[17,35,53,71,89,106,124,142,160,178,195,213,231,249,267,285],"input_token_ids_sha256":"d7dccbe6c436987756ad6166a8d563fa686494edff0e090f19a94996077cdfc1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[18,36,54,72,90,107,125,143,161,179,196,214,232,250,268,286],"target_token_ids_sha256":"8493cc8e2ef026b2ca43108aa6e7db63ec761ce211452609a1bb2760a0f66f2b"},"formatted_content_sha256":"157b49e7f32965e28ef9bb626baf1d0356b6d5f38d74f4c673a15e0198ce6751","generator_receipt_sha256":null,"identity_record_sha256":"31193a44e28b9b97aab66d0eff836463790dd908e352239fd1a3dee6e1ad0a3a","prompt_token_ids_sha256":"a3e2cc82d1e964fda655ec40dc4f4058a3f6f4adaa8b6def330531e94950c8e6","ruler_category":null,"seed":null,"selection_rank":19,"selection_sha256":"0c32b016c1fcca024f19127827b8ece87776b1df963f55614f9a15b354218018","sequence_length":287,"sequence_token_ids_sha256":"1d7f3ec55af99be19b85291e5b80c9d9555e9360cd4af99ded037154beaaad61","source_content_sha256":"e8146f7d1066326e2120f2f49e89fe13318dde45bc338d1b6a350b475840face","target_token_ids_sha256":"f4ea61620cf4a9d1cd26e8136680d5f09ad7d79a197bd08433ab22b67e683560","token_span":{"cache_exposed_start":287,"cache_exposed_stop":287,"prefill_start":0,"prefill_stop":249,"scored_start":249,"scored_stop":287},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"adeda0b376d342263631c70ee440fab7010e921ef7e9bc2997f75798d31a644d","anchor_positions":[17,35,53,72,90,108,127,145,163,182,200,218,237,255,273,292],"anchor_positions_sha256":"c72256a78f816fa7bf8e540cc3aaf3fc1795369a3859efe48a9c4a8cce0d2489","canonical_id":"857","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,35,53,71,89,108,126,144,162,180,199,217,235,253,271,290],"fisher_boundary_sha256":"f5b74e46bd51ccc196ee08a6967a1597fb09f1c1b2ba66309ed0bcb627f70190","horizon":1,"input_positions":[18,36,54,72,90,109,127,145,163,181,200,218,236,254,272,291],"input_token_ids_sha256":"555cbfa7f8498b7200297ecda4d0d95381d1e0e366a25c75a729a308b819e751","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,37,55,73,91,110,128,146,164,182,201,219,237,255,273,292],"target_token_ids_sha256":"ded8ec9b50bd3a7f2b508c6b34745e277cf32245748a23e733afeb28ab5be811"},"formatted_content_sha256":"4156119aeb73684201921a09e3d2fef7e8426ebd10babb52788c0f7e8e9813be","generator_receipt_sha256":null,"identity_record_sha256":"670b8008445ce0955e14e6aa12268d4f91c343a57d01f2618609843306761743","prompt_token_ids_sha256":"5e5ccf81697344dfc507d09f593651afc2e0d360fb85f31936aa419b069701de","ruler_category":null,"seed":null,"selection_rank":20,"selection_sha256":"0f3520a5dd05d707070cfc5723f57c992db4df4d8d98965890fa13a516a29c32","sequence_length":293,"sequence_token_ids_sha256":"ddd8c7e579f53e5c52e946378d772b93a5ed813a063da195e831d38abb0cc958","source_content_sha256":"22096f4ec39e2e444074f7f2cc3873ad56b3e27bb407587437fb05693324c674","target_token_ids_sha256":"66d83ee4ec91a16b9bd4d5c77a1b45e1850fd3594f689ead1e55c781506cfb6f","token_span":{"cache_exposed_start":293,"cache_exposed_stop":293,"prefill_start":0,"prefill_stop":271,"scored_start":271,"scored_stop":293},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"fa5b57e6457aed64b16e39b74ac6aa26eccd3fc3019570266249d9c77b045ed2","anchor_positions":[10,21,32,44,55,66,77,89,100,111,122,134,145,156,167,179],"anchor_positions_sha256":"fe99fb9f0105e4d1413d20b0bbf51aa70554cf263494d9944696f0b9ca84c89e","canonical_id":"884","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[10,21,32,43,54,65,76,88,99,110,121,132,143,154,165,177],"fisher_boundary_sha256":"7a8012c30b1d696314102eab8bd4402dcf733de71d67674552a2c672d81c2cc8","horizon":1,"input_positions":[11,22,33,44,55,66,77,89,100,111,122,133,144,155,166,178],"input_token_ids_sha256":"391312b446f027fbfa057f16da7c25100505a8356e160bc745be4b022c43ebba","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[12,23,34,45,56,67,78,90,101,112,123,134,145,156,167,179],"target_token_ids_sha256":"4f09e35ae3b70a36f49caa3032145ff047fcc34b2e0c9615bdfa4ec5effcacf4"},"formatted_content_sha256":"e2917eaf992f7bb16e2280d5b40f59553c293a9c2ff814a21cdbf4193246972d","generator_receipt_sha256":null,"identity_record_sha256":"170955cbe700df9f79cb32d7fd38d8902e1a304a3a7a3844b606bfcee8b8366b","prompt_token_ids_sha256":"676841a61aa3135e7a3ec64d03888de021cc34698ee1b25aebfff640fc502cf1","ruler_category":null,"seed":null,"selection_rank":21,"selection_sha256":"0f7355686d92107703dcf05c4240667b8c62bf7fc1f0c45fe1704ffc1686d685","sequence_length":180,"sequence_token_ids_sha256":"fc621b262cdee5eeb2686915a8d202aba1e489894daef49f381679515598da1f","source_content_sha256":"9cea206d7dc3308fcc17678f64598526990d06b799c3a6060837e47d5e0ac2ea","target_token_ids_sha256":"18f70329692e7510f6bc5c35dd7288681cb2d7fc3a45398cb46fe8c4064fc5b4","token_span":{"cache_exposed_start":180,"cache_exposed_stop":180,"prefill_start":0,"prefill_stop":112,"scored_start":112,"scored_stop":180},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"c56fd94d27ac497a874a2872a9378539bce23e6f40c0c88da8548530343fcee1","anchor_positions":[9,20,30,41,51,62,72,83,93,104,114,125,135,146,156,167],"anchor_positions_sha256":"eaf3a0e814f88177633695c30e9a05b6b492aa64c90bf98da98cb9be3917c812","canonical_id":"878","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,30,40,50,61,71,82,92,102,113,123,133,144,154,165],"fisher_boundary_sha256":"45cdd8b5f6368af3ea5312cd27a1db7a47955c6f64da0b88e27faf49b3c816e1","horizon":1,"input_positions":[10,20,31,41,51,62,72,83,93,103,114,124,134,145,155,166],"input_token_ids_sha256":"800e65a128ae4172d15f3bf9d35062d514957f43f113ef8390515f7bad4eb648","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,32,42,52,63,73,84,94,104,115,125,135,146,156,167],"target_token_ids_sha256":"cf7c2293e05be3baaf208adda2a202c2dac0599b886a00e133b951f71671b17f"},"formatted_content_sha256":"9fa1e6f58327152b9fd6caf18d856e0920f7b962af65f17c0b516a39a15e889c","generator_receipt_sha256":null,"identity_record_sha256":"9c45e96b568ec62ab8d97f8c4b13adc325eb92bf34bc8cb045cbf49deb8d226a","prompt_token_ids_sha256":"280373fa3a2da39e89a082829b753432d7f22d6ef6366e83598df36823d26b55","ruler_category":null,"seed":null,"selection_rank":22,"selection_sha256":"10b149086df38d9067b558f5886d640b68f4204287a8886f65495246268313d4","sequence_length":168,"sequence_token_ids_sha256":"0bd5f59a58f0f806de221fe3c637774e5c9e47f72c11f0b19b9f2cb06d977dfe","source_content_sha256":"44ecb08d42ef8de88bcb0e909c2e9befe4a195daaebf6c8c10e1baaf0438944c","target_token_ids_sha256":"5b3eced18f97c0b17b291bb1e4d2e732f1ef7b1b304ac11beb67409c8f2d3bc6","token_span":{"cache_exposed_start":168,"cache_exposed_stop":168,"prefill_start":0,"prefill_stop":139,"scored_start":139,"scored_stop":168},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2d6e6a2df8f80c87c48f19e05e8d274114304b958532988537c2cfd6c7ba447a","anchor_positions":[11,24,37,49,62,75,87,100,113,125,138,151,163,176,189,202],"anchor_positions_sha256":"0e1a219b40ec9118f3f3cab251874074a9782d1da2d399b19337c5a52ff6788a","canonical_id":"822","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,36,49,61,74,86,99,112,124,137,149,162,174,187,200],"fisher_boundary_sha256":"6667326d144985716907297f9fcad0d67d899e5951cfd594d7c70e6f0b3e7ec4","horizon":1,"input_positions":[12,25,37,50,62,75,87,100,113,125,138,150,163,175,188,201],"input_token_ids_sha256":"70f7035d5ef6a263a7316e7d5fe9a1d64361df51aca8f8b97408bc66942765a2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,38,51,63,76,88,101,114,126,139,151,164,176,189,202],"target_token_ids_sha256":"b72dca6074d7b5b12b92fb70cfe49900c4d0b2869f86c323e978902c8635fa80"},"formatted_content_sha256":"e207600ca200fa35202626097b7ce35aabbc9a909f74a7f4a9634823421b3f85","generator_receipt_sha256":null,"identity_record_sha256":"6f1cb4fcebbe64bdece660cf8e72d4b0d933e1d5e76363e0fc65bc84df091ccf","prompt_token_ids_sha256":"a45819695adc36f87315b6277308921c06ea8a973dd4acd584c641b42a489685","ruler_category":null,"seed":null,"selection_rank":23,"selection_sha256":"10e329cfe9c44c390c0748888b664bc73f711559a095943ffb716a5c49a43bfe","sequence_length":203,"sequence_token_ids_sha256":"7ca67bbfaf0cf4d58cd3fbf7999f0bdea010a29908aa1277c433c3640ccc93fb","source_content_sha256":"c34011770653c990605bfd0878f3caa642227458bbd94408174d49073634373c","target_token_ids_sha256":"d355095fa08c0462073247a017dce85a1e1739895f047802a3f3fba9380d16ac","token_span":{"cache_exposed_start":203,"cache_exposed_stop":203,"prefill_start":0,"prefill_stop":71,"scored_start":71,"scored_stop":203},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"89cd198269ee29a7bf3a758e494f86cdd7b4c8eb634e87418ec24fa6977ffb90","anchor_positions":[8,18,27,37,46,56,65,75,85,94,104,113,123,132,142,152],"anchor_positions_sha256":"75c1fd0cbacbbfa75bfedc1647ca1a15436ad446c2021b146a8d7af4567321f4","canonical_id":"687","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,27,36,46,55,65,74,83,93,102,112,121,131,140,150],"fisher_boundary_sha256":"ce22804019d01ac4a4b1ecc051fc015d0fc2edfdc083fe9b46238115a22d2681","horizon":1,"input_positions":[9,18,28,37,47,56,66,75,84,94,103,113,122,132,141,151],"input_token_ids_sha256":"fc68589c14fc3c7f27c207419457fdea2a3ca0ea5556ab837c7e77422872e50d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,29,38,48,57,67,76,85,95,104,114,123,133,142,152],"target_token_ids_sha256":"2aafef74e6bdf1473faf98f2e2f51258d9876ed6aa14fd3c8d901e57a6a67670"},"formatted_content_sha256":"34caea8f7845c77c8c7c5cb04e37b6b1594df9805f377e4ad243d7b6bd787df4","generator_receipt_sha256":null,"identity_record_sha256":"5ead1d0afb62e76c6d447e766cb0e770b7b3122bc85e14984b945f91559084eb","prompt_token_ids_sha256":"7231469d89e6746cffb3134204ab6225bd06db47e1a21a1b29978270a92818f3","ruler_category":null,"seed":null,"selection_rank":24,"selection_sha256":"1241cc732d851bc4d6e31425c413658747a68ac5bf1df371701b6bac3339de38","sequence_length":153,"sequence_token_ids_sha256":"0ff8fac034540d5fae83061b024846460ae750e94227132be74a1f8b13d0c5f7","source_content_sha256":"570e823f5d18678380cee81c7f8171a7dccbbe1956f576e5bef831ef0417fbfc","target_token_ids_sha256":"9dd750e2b89888cecb138a1c442f682c640b1ce4caf2a92c920fc630aff9b969","token_span":{"cache_exposed_start":153,"cache_exposed_stop":153,"prefill_start":0,"prefill_stop":89,"scored_start":89,"scored_stop":153},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4c2c007c3a069505a8984ea48151216aceab5ff41919fef5caa62a9c95724294","anchor_positions":[5,12,18,25,31,38,44,51,58,64,71,77,84,90,97,104],"anchor_positions_sha256":"3d14c7e60b70361fe3f48971fff7c78a0e3a89500325f79e8590e96f876c6d84","canonical_id":"820","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,11,18,24,31,37,44,50,56,63,69,76,82,89,95,102],"fisher_boundary_sha256":"a5d52b2d880ccd9e8c57da79f29d3e8871447c9521ac6f1afce731259780a7b4","horizon":1,"input_positions":[6,12,19,25,32,38,45,51,57,64,70,77,83,90,96,103],"input_token_ids_sha256":"f6d753d6a779f0b43d8eb24d7213ecc4ab0ee16118bcbc17c3cb053d5730670b","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,13,20,26,33,39,46,52,58,65,71,78,84,91,97,104],"target_token_ids_sha256":"75597ca42c1c682f0e8f425a875b8483f8c45067ec1d096e0fd49b964402322c"},"formatted_content_sha256":"c5473e872de3d1d10c5f923d4246d5e6e41987ce6e60afff50ad39c665b75d2b","generator_receipt_sha256":null,"identity_record_sha256":"bfa5a8c5d9a1e111c3b629cdc0f09fc8a0dada73721cc93beb5ce531328d1f35","prompt_token_ids_sha256":"9edc0e91d3b033f1f21804597df8021ebb7cede1211331d0898ea29ce2062107","ruler_category":null,"seed":null,"selection_rank":25,"selection_sha256":"1269c5ac64a35cc30e3aab0138aa3e2e5cf40fa1a583bd687f3718d0b78632b5","sequence_length":105,"sequence_token_ids_sha256":"d9f20cb4b5b15d6acba099a4e472ad67b130db359488d35bb838bd17655f28b3","source_content_sha256":"1cda14e8c3edcbccfd8ccbee369285b27f5c9203af7f1d0171339a6fc431ddbd","target_token_ids_sha256":"845201cf3c1564028c1f00ea5c73aa8c447fdf960863b76d2f859ed548bcd8a1","token_span":{"cache_exposed_start":105,"cache_exposed_stop":105,"prefill_start":0,"prefill_stop":74,"scored_start":74,"scored_stop":105},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"f689b6853781e9f787183224c1fd3cee86b9e7cc33bd38b263db4768f634cf37","anchor_positions":[14,30,46,61,77,93,108,124,140,155,171,187,202,218,234,250],"anchor_positions_sha256":"ee1858c2e927a82713536b65cb17e66e4c3e2dba47ef6753229f8cbe7def8d65","canonical_id":"920","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[14,30,45,61,76,92,107,123,139,154,170,185,201,216,232,248],"fisher_boundary_sha256":"0d18b72689a6cc158337087ae0bf96353aba3ba691e2c3ad3383f821d707d7a0","horizon":1,"input_positions":[15,31,46,62,77,93,108,124,140,155,171,186,202,217,233,249],"input_token_ids_sha256":"af45fdc8e583f1a48a51520447e6ae51e48ff97bf7106924892afcd5b445f380","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[16,32,47,63,78,94,109,125,141,156,172,187,203,218,234,250],"target_token_ids_sha256":"9fb5b7afdc77178b1ad18dc9c00bf26c57c1c7f7283fef7c2915df9d37953a04"},"formatted_content_sha256":"ca28cff2764b8d05657413d50f09faa64113177e180abfde5843c1dd2e08ff63","generator_receipt_sha256":null,"identity_record_sha256":"af618c65e0f65410c7799e66f057af21d4ef73c2941c226d581f4f95c21ec761","prompt_token_ids_sha256":"5b15d5261297dc4ac22a17ffede7ef05db15e3f11f0c33e02fb3a3b0e0716a02","ruler_category":null,"seed":null,"selection_rank":26,"selection_sha256":"131778dc6e39fd27c980789805b135c6f9d71c73da87e2cf5d8f360898a2dfd2","sequence_length":251,"sequence_token_ids_sha256":"c20ed19761b4f9b066a1329b36e80f89c62a064c75eaf0dc86e28e961252022c","source_content_sha256":"a36a2af1cadc864f5b43b1a1232f93e477ab83df4d2faa13fb766e215ba061ce","target_token_ids_sha256":"3f83098fcd8caa7e8285f24d6317b53da289468363aa33dd7c3ad415fa2f3ee7","token_span":{"cache_exposed_start":251,"cache_exposed_stop":251,"prefill_start":0,"prefill_stop":215,"scored_start":215,"scored_stop":251},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1df227dd0db9ee9876c5933450c5e9d7dad0c0b569bfe3c3c9999cb603466a66","anchor_positions":[12,25,38,51,64,77,90,104,117,130,143,156,169,182,195,209],"anchor_positions_sha256":"33a74989d1780ad7a110110bf9c662c1607a2b0974227d3d4808405cc10473b3","canonical_id":"771","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,207],"fisher_boundary_sha256":"ca40d8ca4200a96f42fdd96325a52572dfd4c03985d05bc5122da59ebb5411f5","horizon":1,"input_positions":[13,26,39,52,65,78,91,104,117,130,143,156,169,182,195,208],"input_token_ids_sha256":"4a16859fc5dbcd936377e8ff10cf7ee24881b00e341b233f5d9eb6bcea0473b2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[14,27,40,53,66,79,92,105,118,131,144,157,170,183,196,209],"target_token_ids_sha256":"5ca4ff0a723a1b4d28096b092591f8877fbcca9586d49c5ca5cec329031ee210"},"formatted_content_sha256":"12a4cc60de825c6e0b4030d980e6efa58ef1883c0dc4e19c9247b74a5cf96589","generator_receipt_sha256":null,"identity_record_sha256":"b0f0620e81f02da6395b8984ee9a7ea9030dc636bfe539b7bdc6cada58f330c8","prompt_token_ids_sha256":"f7dad551ce5d1c884e8df0a56f12ba6a9086a14bf88ab24c0585392f67253369","ruler_category":null,"seed":null,"selection_rank":27,"selection_sha256":"13342bc1fc998ac6effb2746ad6dd189bd6e08647608c26dfca8ba6e4aa9bc7d","sequence_length":210,"sequence_token_ids_sha256":"880adca8dd7b7e2a5ad8761db452ab4f71a3a2ddb89a2fa3d3f5a58203aefb0b","source_content_sha256":"f9bc388a30026b9c50d24c57c12c983ad57c610e3b1805c0f31da6113e7f0655","target_token_ids_sha256":"946d1f65de1ab807e8d9cdfe1ab226186db882b34df45f0fb02c5dc1c917ec65","token_span":{"cache_exposed_start":210,"cache_exposed_stop":210,"prefill_start":0,"prefill_stop":75,"scored_start":75,"scored_stop":210},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"ba5d2d0908e27069abed969317f8767e700f0653a11ffa619097bf013257832e","anchor_positions":[21,43,65,87,109,131,153,175,197,219,241,263,285,307,329,351],"anchor_positions_sha256":"cac63358bd3a7b31df1079c37c0d5ebcf2a9d198f8fbdbfabcbd20682cead52a","canonical_id":"869","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[20,42,64,86,108,130,152,174,195,217,239,261,283,305,327,349],"fisher_boundary_sha256":"2f61fd72d21168bcf628f22bfa5c23cc9ae7adad9c324ddc9436931a5fc1078a","horizon":1,"input_positions":[21,43,65,87,109,131,153,175,196,218,240,262,284,306,328,350],"input_token_ids_sha256":"4681d4fb7732b42586111bbc62eecd3d429bea617e31837702eba8e398a2cdec","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[22,44,66,88,110,132,154,176,197,219,241,263,285,307,329,351],"target_token_ids_sha256":"4703a2598c12a6054f3c45ce108e141b4a06b5c95d21417d8999f4cbef5fb5b9"},"formatted_content_sha256":"a0eb7c677531c1708849fe663f5664ca25a2bb7202ed10517fbd94dbde91daa8","generator_receipt_sha256":null,"identity_record_sha256":"f65800d83e860fc9520ba9f8b4f750dbd13615d735ec53de0c7592e72d90220a","prompt_token_ids_sha256":"b39968cf98321965c22a9c256c9561e9cc6e27d9a79af006cc89227ec7120efd","ruler_category":null,"seed":null,"selection_rank":28,"selection_sha256":"146f84886fd8ea499b01ff6f18e9470af408d245c5e61abbaa90c362cb26c801","sequence_length":352,"sequence_token_ids_sha256":"1b54a34c4d7ffc13d96aefb4e29514f90f431dde94601897a2dbb235edc52720","source_content_sha256":"aaffc31c45370a56d2068d7903cea3700521215fda95a64f3df629df4b8eeb38","target_token_ids_sha256":"604ca3456eb59e105b48ea010ab6e7f9f2dcfb395b4632901098473fb57640a5","token_span":{"cache_exposed_start":352,"cache_exposed_stop":352,"prefill_start":0,"prefill_stop":305,"scored_start":305,"scored_stop":352},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2bb8e9815739f4c21c430d24915060fe6bd9dba2fc037408c960a316957378f8","anchor_positions":[7,15,23,31,39,48,56,64,72,80,89,97,105,113,121,130],"anchor_positions_sha256":"4b59ffbfdc6171d05db7e4ce13fd81c13fb1c2704ef54cbfaede26eafe2b1382","canonical_id":"851","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"fisher_boundary_sha256":"0b7b6d1d65846d9e17cf0ca2a441e8df727713097fccdd92a3cd8bdb20be9263","horizon":1,"input_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,129],"input_token_ids_sha256":"08877abaf9a7a17c7f7f67ebc2737c37d6453391c6d7b8ec1fbcd14c32c4e1fa","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,33,41,49,57,65,73,81,89,97,105,113,121,130],"target_token_ids_sha256":"f8b2c9f022f43056408bb17bc5c2c95096dde7d6dfeeade0d28c5bf51c33888e"},"formatted_content_sha256":"571149c507816476043bbda2c1fdd724f3ecb9950bd16943ff2ad7eb4ea3f436","generator_receipt_sha256":null,"identity_record_sha256":"7f27dea0fefa17b65ffacb46cfdb9b6d59002d7ca0b9361a0895371d7110d026","prompt_token_ids_sha256":"286d768c4aa541e076ba087112772071c0ed8c5cf796ee399892f394bcbf99e9","ruler_category":null,"seed":null,"selection_rank":29,"selection_sha256":"148757e7e40834cbdd797fc8415b8ff9af1ad4582e23345b11d6f4df20524e54","sequence_length":131,"sequence_token_ids_sha256":"8d6714078aaa6f0fbd4d5dffc75dbd163c2b9867713c0dffd9d5fd8f7017a68b","source_content_sha256":"a2daebc00878723580b55d11ecc7c733a3b30dc6ac7d2162e0a5ddb14516f888","target_token_ids_sha256":"2007499fc1bb16a5e15e13b3ebe6e9c56b039a7ed23a50d2c653934a2788085c","token_span":{"cache_exposed_start":131,"cache_exposed_stop":131,"prefill_start":0,"prefill_stop":95,"scored_start":95,"scored_stop":131},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"348a971192939b69d748687327be2a9dfe6f51af36995ecaa6f6c112cad31b71","anchor_positions":[9,19,29,40,50,60,71,81,91,102,112,122,133,143,153,164],"anchor_positions_sha256":"e70afdac88bd21ff5ffd057462f0868a1aaa408a52296b58ecd4900f7f13759c","canonical_id":"728","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,39,49,60,70,80,90,100,111,121,131,141,151,162],"fisher_boundary_sha256":"de5b5407dcf2dbe46dcd273dbcb9112fd1e97d1b3786ce35fa8124c57a372da9","horizon":1,"input_positions":[10,20,30,40,50,61,71,81,91,101,112,122,132,142,152,163],"input_token_ids_sha256":"164cace43db8303a6bad35155786734ad14df0e29d82329399b4eb77c2ab2a51","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,41,51,62,72,82,92,102,113,123,133,143,153,164],"target_token_ids_sha256":"cf736f31488418c09e969cb79604c6e24ee27824d7b99f2cf3615788cd99df11"},"formatted_content_sha256":"876263584044b675e092dfa08f39f07f3de5267b9c2cc8ebc0f0dcd3f7e69bed","generator_receipt_sha256":null,"identity_record_sha256":"07b155ad0595cc23e7b6f3ba638e79b47f8790d682a99a574040b9b316eb1770","prompt_token_ids_sha256":"7673bf381655d73528d4ffb834dd0461fec79f874ac5706819d772a4fd92f274","ruler_category":null,"seed":null,"selection_rank":30,"selection_sha256":"1488999c83a1784fe27b3d4c152a98b782b1ce9c84a2375a4dba8bc8a3845802","sequence_length":165,"sequence_token_ids_sha256":"cbf68229cee850133e8ccd5fc251f1c822ecfb6aa1c9466194360609b2644aa8","source_content_sha256":"1e63d65d4f23efcd1ebde304ca66d531e9fdfa44f73cf04c72637826af3e7c94","target_token_ids_sha256":"519538cb0a590f8242c78a833368585a5650cdbbaffd4159acb21311b25be8bb","token_span":{"cache_exposed_start":165,"cache_exposed_stop":165,"prefill_start":0,"prefill_stop":128,"scored_start":128,"scored_stop":165},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"fd0a190e30ef1d0dac28f65cd40bb5a6bcf3b1813b383cf8c3595fc399c3cf68","anchor_positions":[8,18,27,37,46,56,65,75,85,94,104,113,123,132,142,152],"anchor_positions_sha256":"75c1fd0cbacbbfa75bfedc1647ca1a15436ad446c2021b146a8d7af4567321f4","canonical_id":"704","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,27,36,46,55,65,74,83,93,102,112,121,131,140,150],"fisher_boundary_sha256":"3f642dc0ec35708247c2582e0525688edfba25c8e20f052b50fddeb5a552f23a","horizon":1,"input_positions":[9,18,28,37,47,56,66,75,84,94,103,113,122,132,141,151],"input_token_ids_sha256":"c7f609f9659eee383626ea3792b82efa4d88eed93f90d975eb30c098b4b461a1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,29,38,48,57,67,76,85,95,104,114,123,133,142,152],"target_token_ids_sha256":"04c201a63e6b0f0929e297c5714da1234f09af3ca5cb45bc1b660e160389b369"},"formatted_content_sha256":"cae02a47f70b35282c8bb71d3b57044d50c57479d6d860ab99959bd7d6a92fc2","generator_receipt_sha256":null,"identity_record_sha256":"5ee66dd333ec6248d85a5f68ed91d44953b5193938257c58e4a4d12487288565","prompt_token_ids_sha256":"049fdd2c0e15e77a6301829c690e8a3c665ce6861ab91933d27aa964bb51de01","ruler_category":null,"seed":null,"selection_rank":31,"selection_sha256":"149396b0bf9de8c384e257ecabf9d186fe87150d089c3ff0d04e2adce108c249","sequence_length":153,"sequence_token_ids_sha256":"d06f1e44e281b3cfff0ab509061ec63103257ce65b9382636808622420a3847b","source_content_sha256":"9acfae6a9b6bf5138640d082e7cd35c1728c6aef929c1163bdc349fd4539d647","target_token_ids_sha256":"413994788ec19a2c46be1007e6767b52c27d228534ace24859c56b31248a539d","token_span":{"cache_exposed_start":153,"cache_exposed_stop":153,"prefill_start":0,"prefill_stop":114,"scored_start":114,"scored_stop":153},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"30b1518b021112b74a55061d787eb229a1e0d9413d6dacc4728a4dab232a67b7","anchor_positions":[9,19,29,40,50,60,70,81,91,101,111,122,132,142,152,163],"anchor_positions_sha256":"868d4ebf721b69ebc32ed80fb23287a93509d3cda6e8df7dfacdd2b56b0be622","canonical_id":"725","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,39,49,59,69,80,90,100,110,120,130,140,150,161],"fisher_boundary_sha256":"9143ea3e85262814307335f186e336430d15da7ab0b7b3d21349bf2995db95b8","horizon":1,"input_positions":[10,20,30,40,50,60,70,81,91,101,111,121,131,141,151,162],"input_token_ids_sha256":"596886915afefd5186de7498848bbc83ca47716f8e4d7009a44f03810ca08ca5","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,41,51,61,71,82,92,102,112,122,132,142,152,163],"target_token_ids_sha256":"3039436dc9e159ba1ad775ff006ed4075d724a7118b474bb9fb8a1f2e8adf4fa"},"formatted_content_sha256":"ba9dc9a284681815182d925a5298e2f7da2b4d824b73ef346adbda8f64016575","generator_receipt_sha256":null,"identity_record_sha256":"e596da18d1c37026ef17f3e29e6e8ff1d0a69c227e1cbbad9c536f121a0a3fa1","prompt_token_ids_sha256":"9d320289cf51463daa5b252b98ca89089d036d231b6c861dc2405bc54707a135","ruler_category":null,"seed":null,"selection_rank":32,"selection_sha256":"14ed1df91ab46cf4aa753625f81c4925778f6febc4e611062de8ad6126487c18","sequence_length":164,"sequence_token_ids_sha256":"b6e36e161450f26908c1d62afda5bc88c353a091bbe87f9de4dd698d1df5afe4","source_content_sha256":"d222c883a2e980af7460497236d6865d722ce738a2d6303dcb39cefe0ea11658","target_token_ids_sha256":"da7b35aeba5cd5e4fc9de97bd08c7c8e930056923fea9d072dfd25b57ef54703","token_span":{"cache_exposed_start":164,"cache_exposed_stop":164,"prefill_start":0,"prefill_stop":138,"scored_start":138,"scored_stop":164},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"33a12c7dff5a4e3f2b0e78cb96a2e15ffca6d15f728512f48ad3d2a521547859","anchor_positions":[12,26,40,54,68,81,95,109,123,137,150,164,178,192,206,220],"anchor_positions_sha256":"9cd7cad8fc6a3e6b59742658c86bba1f77992157cd1a23eb295add08927945d5","canonical_id":"616","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[12,26,40,53,67,81,94,108,122,135,149,163,176,190,204,218],"fisher_boundary_sha256":"7fb746d0e7bad7bf9d3e8daeb30cb4fb5e6d8058e6e2f60f208e45d9d3074ef8","horizon":1,"input_positions":[13,27,41,54,68,82,95,109,123,136,150,164,177,191,205,219],"input_token_ids_sha256":"2984e5c39ef7c7a9017c275ad13366ddae1e4a944113a0d7634a8743fe7d838e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[14,28,42,55,69,83,96,110,124,137,151,165,178,192,206,220],"target_token_ids_sha256":"f8ac64591035ce55623fdb95289f20ce8ce2512016905cd8c55cdbd1eaf0892e"},"formatted_content_sha256":"4c31755567c8f3a5c363166b6b45afe0e9756596cab355453c6d1bede973f1cb","generator_receipt_sha256":null,"identity_record_sha256":"2d8025f16c114b643173084fd6a55004c4cc8d89c19ba719eaf77c614f3de5bb","prompt_token_ids_sha256":"e3e1a17d69f2fe4102ab51a9bb887707430d5bd517f752ff79cb5ba3bc19372e","ruler_category":null,"seed":null,"selection_rank":33,"selection_sha256":"15619d70a9d1e84b42b7fe6f67fb25251ed4e033a22a1f6b44afdb5190e0d9e4","sequence_length":221,"sequence_token_ids_sha256":"c14c8b8afb98ee4e0db1a7d15b4a5ad8f4ceed0533d70f5c7f3e3069aa019030","source_content_sha256":"1754fb38b7c558a69da8e9273fbe27d53c7428292d713cdcc926533238ea0dd2","target_token_ids_sha256":"67f9de9099caf2a8542295769dccf90b9868362889179ae8905fbdfffbfa259d","token_span":{"cache_exposed_start":221,"cache_exposed_stop":221,"prefill_start":0,"prefill_stop":172,"scored_start":172,"scored_stop":221},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"df20597eac46fe01ffe3b097acab98b2815becb962637bf63848ef96325ccb51","anchor_positions":[16,34,52,70,88,105,123,141,159,177,194,212,230,248,266,284],"anchor_positions_sha256":"875344e57ef3cc2aeb5a93339807a07f348a76ff8e531ac8b017af57bdfad6d8","canonical_id":"686","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[16,34,52,69,87,105,122,140,158,175,193,211,228,246,264,282],"fisher_boundary_sha256":"a3d4c1cea37694ff9e48138187338acaf151bbb01faf6beacef99dbb88b7a3f4","horizon":1,"input_positions":[17,35,53,70,88,106,123,141,159,176,194,212,229,247,265,283],"input_token_ids_sha256":"e11b1d7a44256148ceeca485fc09b3a0bbcb82bb1fc8f781be0549ae79fe1f2a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[18,36,54,71,89,107,124,142,160,177,195,213,230,248,266,284],"target_token_ids_sha256":"89eebf9531df2b055714e83202f980d1b550ad965a5daf4c7bdb06671f9842cf"},"formatted_content_sha256":"7351330811ac6c42a440a1ec8bc6d19bdf453494f7fd6204e144c59693722149","generator_receipt_sha256":null,"identity_record_sha256":"66b94f7added65074e9aa08de692376e4e61c77477886d2f1f02bd7a5f752106","prompt_token_ids_sha256":"c9ee635d4f345c508119836f02286e8a883aaafc0a5504f5149f7ce65b3e8a58","ruler_category":null,"seed":null,"selection_rank":34,"selection_sha256":"15a952922cfb59e1c69a11112c4ca386841fc6ef24bc7d4b5506686777d97a8b","sequence_length":285,"sequence_token_ids_sha256":"fcc6b59259d07edcb03ac21e620699ab241c7ce75c923d03ee63d3146e5b591b","source_content_sha256":"84f9793e00ac40962b5d2488ed7ea603278d041e9ce4d1fdb4c80a3cb038287b","target_token_ids_sha256":"f0132a38a41b59f822f698b45717035a5147aeda26ea2ca039b6d45f96067ef7","token_span":{"cache_exposed_start":285,"cache_exposed_stop":285,"prefill_start":0,"prefill_stop":239,"scored_start":239,"scored_stop":285},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"969aa08fc8cf0fcef08b212a868ffd0ef6cba7360b8cdfd4c7e057bdeb42b613","anchor_positions":[23,47,71,96,120,144,168,193,217,241,265,290,314,338,362,387],"anchor_positions_sha256":"5de510290d52f91a9e5aaaf002df55793d821f7c4325536087b3fe975a00c5db","canonical_id":"722","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[23,47,71,95,119,143,167,192,216,240,264,288,312,336,360,385],"fisher_boundary_sha256":"5bcdece273089c73eeb060133d26006686f4ba55e2f679475c817201f0f06663","horizon":1,"input_positions":[24,48,72,96,120,144,168,193,217,241,265,289,313,337,361,386],"input_token_ids_sha256":"387e8436656479c468480e0d7ab6fb8a04966a25430d2cf7ba6b946e933a5988","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[25,49,73,97,121,145,169,194,218,242,266,290,314,338,362,387],"target_token_ids_sha256":"d4c1910bee8959bc2af95badfdaa811dfbfff9ce3708801ed5e9602140453d3f"},"formatted_content_sha256":"66886a05110a3b2882fef0d7f3d69fe9e4813e5a230a25f32ee61406ab9d5da2","generator_receipt_sha256":null,"identity_record_sha256":"dae3872dc358589c8a222ed2b8f364150c472c88b23243e89cbb751bec3bf394","prompt_token_ids_sha256":"e0dd20f7abb2fd8622ac76878d7d8dba366a9267d17eb23cde37996d51815bad","ruler_category":null,"seed":null,"selection_rank":35,"selection_sha256":"165983c8c5992c88338b5a83c3e955a3b379e2938838aa804f752b6af24364d5","sequence_length":388,"sequence_token_ids_sha256":"be18de19017b46d5b7be59183a8ecda4c60a214ae50f068dd8ce1b4ea6848323","source_content_sha256":"20253504a8b3a9c5fa26e7d29c98f783efb48d1960c35ff104acfdc24f41bd57","target_token_ids_sha256":"5e021e0c05f5c5bc55c907080f16d71f9b17ea980d1c4a1c35db2a4193541b91","token_span":{"cache_exposed_start":388,"cache_exposed_stop":388,"prefill_start":0,"prefill_stop":344,"scored_start":344,"scored_stop":388},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"da5073651bb69cf285a41628d42d9f0b3d2b37c1bf90aa7aab1b73c4d9096d21","anchor_positions":[6,13,20,27,34,41,48,55,62,69,76,83,90,97,104,112],"anchor_positions_sha256":"311e7702507c904e70997e0b842193b8c519ddd45b6861c3e52fdd12fcbe662f","canonical_id":"919","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,19,26,33,40,47,54,61,68,75,82,89,96,103,110],"fisher_boundary_sha256":"46adb9e95c41a48fc3a2299f313d710d273451aec915fcc44050066d7e62fbe7","horizon":1,"input_positions":[6,13,20,27,34,41,48,55,62,69,76,83,90,97,104,111],"input_token_ids_sha256":"27444c193cd674bda4df496bd61a9e9daaf7ab3670462bf76e183b0cb9f16771","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112],"target_token_ids_sha256":"0c19883626f27df1a14cd3ef14afef3bc348c0cd1bda790d1e08185bca73a595"},"formatted_content_sha256":"032c4c4ef8d61477f89f627c7da791c2a394cc3d48b63ae5e73ee2fc12217e59","generator_receipt_sha256":null,"identity_record_sha256":"3ff17e45b728b8142b6678e3a33eac324c6262961cd2c9f7c19d1310c440aeb7","prompt_token_ids_sha256":"23b1b41e6716b12d13650e7aa105d3a76602e537cb02e4accc5f592ffd9c138a","ruler_category":null,"seed":null,"selection_rank":36,"selection_sha256":"165a7fd7caa5808dac419a6b8728f561f43cf78710437e2638e5c6a82bb7617e","sequence_length":113,"sequence_token_ids_sha256":"4582364354dd4ea26e506fc7c1845a9a6a878d03218ac93a0725170aef66c313","source_content_sha256":"30238651c7d4eb4dd16bbcf4c54ef3c91884f1d06eeb02ff38b97cb5e03a6225","target_token_ids_sha256":"9f6a8727f6485bb7bce051a35976a9ccfdc83c3af9309cb39f3a93789a51f928","token_span":{"cache_exposed_start":113,"cache_exposed_stop":113,"prefill_start":0,"prefill_stop":86,"scored_start":86,"scored_stop":113},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0a798f70d770e513f2e911c70a33835f666289b6e23ff31551a773ef75b38aec","anchor_positions":[6,14,22,30,38,46,54,62,70,78,86,94,102,110,118,126],"anchor_positions_sha256":"5dd8de14849d09f9037c328d85f9f1208f24e656ecfb3354483ee5aa62c57bab","canonical_id":"636","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,30,38,45,53,61,69,77,84,92,100,108,116,124],"fisher_boundary_sha256":"5f2e24eb47dfbebce18f2c2f64fd45917172c9eab2bd9cec54bb4fd111eb2142","horizon":1,"input_positions":[7,15,23,31,39,46,54,62,70,78,85,93,101,109,117,125],"input_token_ids_sha256":"4dc0a187eaaa7554b02100bc49019061f57a4f625ac418d8d6b7708a4c85f659","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,32,40,47,55,63,71,79,86,94,102,110,118,126],"target_token_ids_sha256":"3904fbb873aaac279d16ea071f1eecd284ddfaf982e8b4b118e527a7c5cd1663"},"formatted_content_sha256":"38120716e8579745533e38a1b2e5c3ab60964ce9826f4692be5d38a23625c7d5","generator_receipt_sha256":null,"identity_record_sha256":"6b00dd2ed569a9e5852db2dae0aa92821b9dbc00778205f14f3b67a886091087","prompt_token_ids_sha256":"c53edd29ef5c93c57c169235bf2b32cb118e20df561bc94d9b39faebc2a587c0","ruler_category":null,"seed":null,"selection_rank":37,"selection_sha256":"198ca758f04e266276eba3d3b469b52d783bc89308214f93d72af2c6d4e11dc0","sequence_length":127,"sequence_token_ids_sha256":"60e0f64c1dcb89512f1401f29f0d940ea152343ebc4764e52809ea4c5588f847","source_content_sha256":"054f097d046f21661577c0ed913168b649add5d35032de9efef4938f8a1a24ee","target_token_ids_sha256":"d1bf3a7c3a1a467f3f8e1eac6c4887bdfbdc387af6b7c29fcb19d45df9277f3b","token_span":{"cache_exposed_start":127,"cache_exposed_stop":127,"prefill_start":0,"prefill_stop":94,"scored_start":94,"scored_stop":127},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"d11ddad979bbc50d6db14cf77f968c46cb786a0ce2dc77ba5f5b28d500fc3596","anchor_positions":[11,24,37,50,63,76,89,102,115,128,141,154,167,180,193,206],"anchor_positions_sha256":"c25b8f380f538bd903f3e5b66a99e3d50700aeecbba5fa891f2350659ebff705","canonical_id":"607","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,50,63,75,88,101,114,127,139,152,165,178,191,204],"fisher_boundary_sha256":"a15bb1e5b2d2a3301ccdb2e0cb8998f04674305617ecc9881b7d20c208dd21c1","horizon":1,"input_positions":[12,25,38,51,64,76,89,102,115,128,140,153,166,179,192,205],"input_token_ids_sha256":"7ffaff1442181b24e662e144cb14110cb590e37477796426bec7dca4fc4d39ae","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,52,65,77,90,103,116,129,141,154,167,180,193,206],"target_token_ids_sha256":"05b028901595cb91b630adba80dd4555b614c059c122db592c5aa980143feafb"},"formatted_content_sha256":"ef35e3704df515714d76b547483c483743548e7cc069f31e95b66dec72a510e5","generator_receipt_sha256":null,"identity_record_sha256":"80bb8f3d83b1925af377e081ac063f63989368883ef4e04dc49b8fd8f00c500b","prompt_token_ids_sha256":"5bd1a90f46714aa81aed8226efa90f47442e2892377804149115218b9fd3b0c7","ruler_category":null,"seed":null,"selection_rank":38,"selection_sha256":"19faecf7ffafd480a6fa766dc937ba0709e42a5d5263c93130eb1037763f8871","sequence_length":207,"sequence_token_ids_sha256":"020df0554b8bcb9a4b7f491a4513578dc9b60d47ffc0d439886b3997c36eb986","source_content_sha256":"c541b88b979a7ec60640a49c7a9db7db502b97f5fed4d1970da91470204697c0","target_token_ids_sha256":"28dd01dbcdd5f0d19a9bef9f9fbc6491786783fdb97ad1fb07f945a93ffc7cc9","token_span":{"cache_exposed_start":207,"cache_exposed_stop":207,"prefill_start":0,"prefill_stop":141,"scored_start":141,"scored_stop":207},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"46371c5d2e1a407eb39f03172e7b9a21c7b932f5d655758a3c0ce64ba8c669a2","anchor_positions":[23,48,73,98,123,148,173,198,223,248,273,298,323,348,373,398],"anchor_positions_sha256":"8d9eba8a281ab33b27dd14ed9d345bfd773fffcabf08f24d986bb5ecd2f5015a","canonical_id":"950","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[23,48,73,98,123,147,172,197,222,247,271,296,321,346,371,396],"fisher_boundary_sha256":"843c418f1af24b45838fcae84492105e4320ec3725733998cacab7b6853efde8","horizon":1,"input_positions":[24,49,74,99,124,148,173,198,223,248,272,297,322,347,372,397],"input_token_ids_sha256":"d9a45e7a4afdd75464b73c4c6185b6888c636708adbfe2ece93d9586f2e48950","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[25,50,75,100,125,149,174,199,224,249,273,298,323,348,373,398],"target_token_ids_sha256":"b25ab7ecce47ea372a3f59f87969ac12e02c7358ce537fa47921278d5bbace65"},"formatted_content_sha256":"99720600b7a4e17df2bcc2762c83cdf70cc53626e5ec631c9bb32d7398d93e45","generator_receipt_sha256":null,"identity_record_sha256":"6d22eef549de3a81fc0a232affc18a615e9f0fce2de79d390b05e11ebdbf763d","prompt_token_ids_sha256":"8801aebf9d0be65bf1d4cac5f7ba35ceab845d7b0e20d5f6bfe717cf70fcac04","ruler_category":null,"seed":null,"selection_rank":39,"selection_sha256":"1b30f64a04ff98347d4a23bd49f8964fc6d3976e5d73b89babb36d891b544775","sequence_length":399,"sequence_token_ids_sha256":"0c20245c48d7a840676ca88457d31004701b407e4992f56ed7e4aca800bd3cee","source_content_sha256":"67c27b9d1db42865751e90ac9eb8dce913fa954d522bc50f1a124ed869dcc3b6","target_token_ids_sha256":"80ad07987d1074046d11f85abd02309f81753af95f7b2510f15b49ce074774dc","token_span":{"cache_exposed_start":399,"cache_exposed_stop":399,"prefill_start":0,"prefill_stop":85,"scored_start":85,"scored_stop":399},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"88045b346b8da1ff4c8b2efa7461ce5138643eb8b76bb78bed0cb8b0de1efbf4","anchor_positions":[11,23,35,47,59,71,83,95,107,119,131,143,155,167,179,192],"anchor_positions_sha256":"97c036645b9b6a78e53e848879a7e3ee08a70ff938ed6e98aa7098b21ab4319c","canonical_id":"885","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[10,22,34,46,58,70,82,94,106,118,130,142,154,166,178,190],"fisher_boundary_sha256":"810182ed20f2eb26e385debb7eceaceabe95cdd144936b72eb2f3945a5610256","horizon":1,"input_positions":[11,23,35,47,59,71,83,95,107,119,131,143,155,167,179,191],"input_token_ids_sha256":"76887b11d917341d902fd087a797d1b8148960fe5a5c3bd03223991175a7d00d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[12,24,36,48,60,72,84,96,108,120,132,144,156,168,180,192],"target_token_ids_sha256":"6d849b678d0b30547bb1a54e0ea0ac4ce468ce61603c7c10ffdab4c4e48459ab"},"formatted_content_sha256":"a16af1d88bf778a540f042c4c4f117a8a55e110b844fac976861bf833b58928c","generator_receipt_sha256":null,"identity_record_sha256":"210e74e96d878fcdcc906fc9925e4f204d9363fe486a768730fb41f707b3047a","prompt_token_ids_sha256":"731bddd1572f8ac6e10c56476d8d41ce0f5b35ad3da79217ad289e53697fc68f","ruler_category":null,"seed":null,"selection_rank":40,"selection_sha256":"1bf4005ab722efdcf542cb1b134158d2a2ec80c4bb99a978bf37320204ea651d","sequence_length":193,"sequence_token_ids_sha256":"221d7396fd6ba0730b59b2d163b37447727eeada7ea48a5bc944748d4c03df26","source_content_sha256":"c09a41add9e81f949a02fef6dde8a12287741692b045d26b3459d9cbf51a329b","target_token_ids_sha256":"a28fddf10e4bd937c83e1787639bbae521e60b6a3d5d421086e7e8d04ced9afc","token_span":{"cache_exposed_start":193,"cache_exposed_stop":193,"prefill_start":0,"prefill_stop":82,"scored_start":82,"scored_stop":193},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"5631703787387bea51be9c3d8508c71583e012079113a9f10bbec4245f396cf1","anchor_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"anchor_positions_sha256":"debc2f860d51bf30c9ccfe7850d3428800e80a005f976ab2943a8642c33fb808","canonical_id":"646","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,30,38,46,54,62,70,78,86,94,102,110,118,126],"fisher_boundary_sha256":"ac073c216400b3bc77300dbc3f3bd0735b111cf717a637bf602d1117e5d70dd5","horizon":1,"input_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,127],"input_token_ids_sha256":"06dec0ee7d5561ebf4a6924837605f275d606eb9ead89de4bbe6b7ec86a0c69d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128],"target_token_ids_sha256":"819be8fbf775e438d651a64a9a89d31a86d639d85622f2e6a979e72bf5469879"},"formatted_content_sha256":"42d2696d713ac8654462eed80e51a0f4a279ae3bd0b2fac3a803e706e31df249","generator_receipt_sha256":null,"identity_record_sha256":"a62e891eaa4a9d3d4ea47d185ffa6c03a9b1d28dcc7dec3429aece7413ef0936","prompt_token_ids_sha256":"e5220dd3a6c5ee6f26bffcedb6d05bd93a890b9b7d74832a9669975115e37150","ruler_category":null,"seed":null,"selection_rank":41,"selection_sha256":"1de0cdbbd1b50e7684dc3deead2b93036bacf525241fea00a2c68aeb5d1e9cf0","sequence_length":129,"sequence_token_ids_sha256":"eba6b177f82a24193e70a824138922ac8eee52933ebb80cb81a777d6e0f85943","source_content_sha256":"14c86e5b0221ac25a5d693c8996c478c0b60503bd396981a5ba2629d0c6b6f99","target_token_ids_sha256":"6c75f6536a4d7c5a2942fd540f9268e2e1961c4c3dd4523408da058253f543b3","token_span":{"cache_exposed_start":129,"cache_exposed_stop":129,"prefill_start":0,"prefill_stop":88,"scored_start":88,"scored_stop":129},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7105aa2d8387cb83f89b844f1f0a3176a54c1be40233dec8811a37cb7bee09d4","anchor_positions":[5,11,17,23,29,35,41,47,53,59,65,71,77,83,89,96],"anchor_positions_sha256":"bbeb8538883fad44ee21dd0032311ae447587513a5cfd0b0e896721a67ec397d","canonical_id":"708","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[4,10,16,22,28,34,40,46,52,58,64,70,76,82,88,94],"fisher_boundary_sha256":"977350ea6b433e2f68043b28b121e4816ce8fb2f7358ecf47291b22c9adc1129","horizon":1,"input_positions":[5,11,17,23,29,35,41,47,53,59,65,71,77,83,89,95],"input_token_ids_sha256":"a2ea5635b427055cb1e2e99736b9dd1cada7580fac25e57e539b72697c2a8ea9","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96],"target_token_ids_sha256":"81ee8d37b3f38a8c027e847f395edfdc8428e67a0cfd5bb3a5d12bfdac304ab3"},"formatted_content_sha256":"c810e44140744e432b3b9ff12b48167b78e7a00f4f078b3d9b5a3f539a0438dd","generator_receipt_sha256":null,"identity_record_sha256":"ab8e48a41997cface254c352c24a699bef6c43ae1858c93baddd4ba95bbef8e1","prompt_token_ids_sha256":"5a9bc94048f86f543dd1e7c3e8040a400a75b1f99b6ef637101d4590969f22bd","ruler_category":null,"seed":null,"selection_rank":42,"selection_sha256":"1e4c5d9ae6b7070238249263248d1c8593c90a70aeeb70af89325a5b3be72629","sequence_length":97,"sequence_token_ids_sha256":"23e232e07e81e613d2b01d8540dfd4e20039938471bf3dbebd3ba8c57449e8b0","source_content_sha256":"a670341f8d1f903c87ab584beb9dd6355bf23c707ac087606d85d5e1094169ba","target_token_ids_sha256":"bc70ef552d90fd9d0e6a01a2dad09bad247647bd98b2af5d9afba83f63d97b8a","token_span":{"cache_exposed_start":97,"cache_exposed_stop":97,"prefill_start":0,"prefill_stop":79,"scored_start":79,"scored_stop":97},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"21fdfa6c62576c444b2ede715e8c0f394692bdae740be5657a09fb9ac5465380","anchor_positions":[7,15,23,31,39,48,56,64,72,80,89,97,105,113,121,130],"anchor_positions_sha256":"4b59ffbfdc6171d05db7e4ce13fd81c13fb1c2704ef54cbfaede26eafe2b1382","canonical_id":"840","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"fisher_boundary_sha256":"f1a95ea699018f29238a0e61c6067bc03c5622df1f9f38ee7310c244a4f3bc65","horizon":1,"input_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,129],"input_token_ids_sha256":"db9ca78375767d3dc1fc43cd6274a248a1200a4e579725f08ae34f2d1594030c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,33,41,49,57,65,73,81,89,97,105,113,121,130],"target_token_ids_sha256":"d4ee1e1651e25e76151be8dfefda3d59a9153f9a6418389ce485081fde471280"},"formatted_content_sha256":"9aced07cb336136d232b88c651523d949c3ed1b58608f3e91ab8eb1e0f92a7d3","generator_receipt_sha256":null,"identity_record_sha256":"baa3355a5d2917d2ff4b713073abc3c98d3168719b1d9a669dfc8d2745e62db8","prompt_token_ids_sha256":"9b97f8bc9144022005c6d732aa6daf7482cce1d0462c9f41e4e54f455d770a9b","ruler_category":null,"seed":null,"selection_rank":43,"selection_sha256":"1e5afc59d9964484ca6906bae912fa0a9c9871e246e9a5aa6f4e2d2cd74a61e4","sequence_length":131,"sequence_token_ids_sha256":"ac96beecd6d569ac72fd89a9d15d46adb2c7e2ff3ab53a38a47e2ebac4b5c19a","source_content_sha256":"144d4c36a2f15b6079c87286abc57c5c33d11947809eec3b55d2c56b0abdca35","target_token_ids_sha256":"60cc8ab098d798eaf3a7d1177d7c65092515a6756c2fb167019eb1588a7a5f57","token_span":{"cache_exposed_start":131,"cache_exposed_stop":131,"prefill_start":0,"prefill_stop":98,"scored_start":98,"scored_stop":131},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"92d974a85b650173fa2852040052d155e429f141353a391c32eb9c8b22bf9582","anchor_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,208],"anchor_positions_sha256":"e3c9aaa391385675ab3475417e405da92d531052d937e046ddc609185dc2c7d4","canonical_id":"793","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,50,63,76,89,102,115,128,141,154,167,180,193,206],"fisher_boundary_sha256":"bcee19793289e8c6540ea881a8f492bb16b99a21c7f08133aae2e4a62163e585","horizon":1,"input_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,207],"input_token_ids_sha256":"ea265790ccf2b7ad1bc5812d8c7ad1653088a1b938adf116bbe32e00c4b3161c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,52,65,78,91,104,117,130,143,156,169,182,195,208],"target_token_ids_sha256":"03166cef7fd0abd95fc728344274b68579c18cdff9751955cb55d37472750e8e"},"formatted_content_sha256":"5990b44a405865af219b0ebfe72ca5f4807c7fa9ea8d17a998d181a60e7e6e2a","generator_receipt_sha256":null,"identity_record_sha256":"7694eeb6f1cf845cfd47b3f8c037b4453af3e6f1af57cbc1b271567d78747feb","prompt_token_ids_sha256":"a215965eef2fc3b7d4240f33fb78e20e16353f44c552a86876020a731cfb23ca","ruler_category":null,"seed":null,"selection_rank":44,"selection_sha256":"1efdf31f8359ff37ddbd6972a5b1f70a89aad0090992f1439e95a68b17f5805e","sequence_length":209,"sequence_token_ids_sha256":"b0ada4ec9a33d0cbd5317f8965026c54cce944770d6db49dedf1045e3e120210","source_content_sha256":"f51880de3d4f3f8df181ab757d4737250300a2719eb925110ed1a7025baefea4","target_token_ids_sha256":"1bb894993d9298db11a3f618f8c67b31b484d0fa664e5d9a74950bae08de8292","token_span":{"cache_exposed_start":209,"cache_exposed_stop":209,"prefill_start":0,"prefill_stop":108,"scored_start":108,"scored_stop":209},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"467b274cf4fd1ebd3fc1f65e47b4ebcef7ec6daac053d9c806d526d02b1aa9d2","anchor_positions":[7,16,25,34,43,51,60,69,78,87,95,104,113,122,131,140],"anchor_positions_sha256":"adf29a4dd3126019eaa451ccdb6bbdf039b38d8dc8e8cbe224d799517db47f03","canonical_id":"860","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,25,33,42,51,59,68,77,85,94,103,111,120,129,138],"fisher_boundary_sha256":"4298d52c37b3378977ffe8fcdbbb8de8a2001e201f91fa007b5bc8caf344f153","horizon":1,"input_positions":[8,17,26,34,43,52,60,69,78,86,95,104,112,121,130,139],"input_token_ids_sha256":"ff88adf412ffcfaf2eb5e6577b0406f784344c338d07181aab928b4e45c4aedb","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,27,35,44,53,61,70,79,87,96,105,113,122,131,140],"target_token_ids_sha256":"cb42fbfadbf84d40f31a090f455333c32b8d82097fa30bbe81c411778c79d549"},"formatted_content_sha256":"9fa46af05168957f4bcef1fe5cc161d5fd1f8aec927a8f95b00c13ce334a2346","generator_receipt_sha256":null,"identity_record_sha256":"f03d6c0de51871dd6a4c550ca9bb84df67fdd8703da60597ef107f6b4b5998cc","prompt_token_ids_sha256":"a9df32740687672e9bcea5fc074876046b128fd5263c7a057d9f57e081343d25","ruler_category":null,"seed":null,"selection_rank":45,"selection_sha256":"1f2c607266f18f654c50ab67f529127bf81f03531b6d79c46ac97f5081f09a65","sequence_length":141,"sequence_token_ids_sha256":"13d911b52ef7a2c203854b8ab41f9083733e9bda1ca8ed67607907aacf4bb0f8","source_content_sha256":"6948e55f2c0c12a3d6c826e33977a3b358f503e8cfccd743ee937a4f53cbcd15","target_token_ids_sha256":"37c5ab0e675f04a104215395c9a2a8503e4871823234c8abbb133ac6babb0449","token_span":{"cache_exposed_start":141,"cache_exposed_stop":141,"prefill_start":0,"prefill_stop":95,"scored_start":95,"scored_stop":141},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"37d7574d8d2b723c82778a768a2a566b7f49e8ebb69db0fb5d1e6794a930a2a8","anchor_positions":[5,12,18,25,31,38,44,51,57,64,70,77,83,90,96,103],"anchor_positions_sha256":"307c6a1944f2d94c59465c6af96e251aaf0ea665f8f1a3e0e55e3eb0568d8551","canonical_id":"800","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,11,18,24,30,37,43,50,56,62,69,75,81,88,94,101],"fisher_boundary_sha256":"54b8850d26dd8a6a43bc711ad584ae0be7143cd933d1768dbf78ab9d2e4721bf","horizon":1,"input_positions":[6,12,19,25,31,38,44,51,57,63,70,76,82,89,95,102],"input_token_ids_sha256":"5182574bad58e65b680691603b61908c731b487efc2d522fac61ffb07e8601e2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,13,20,26,32,39,45,52,58,64,71,77,83,90,96,103],"target_token_ids_sha256":"8836df71d734324c70c31d4de1e0ff0cdde8edd0e57b420ca0ed65319cddd528"},"formatted_content_sha256":"b47d463d8300a4f8eb63bc1a7f94408c9b894a1853d6b4b63d05eed61f97e078","generator_receipt_sha256":null,"identity_record_sha256":"83d65e18df6abcd124b640d68f38f75296045dc30136a8095de98e9a56ff6059","prompt_token_ids_sha256":"f32d5f100044bbea939023be6610b696d665c2e49ac55c3b2478fd990e92c75d","ruler_category":null,"seed":null,"selection_rank":46,"selection_sha256":"1fda23ecb8320de65f93f8ffeca9bc66d1d6ba82fa80bd9e2d70cb080ea19be8","sequence_length":104,"sequence_token_ids_sha256":"2095ea60e58d7a748f80d99738c7213d620ebe1064bc23e828b8298509455223","source_content_sha256":"251053bc7b023ed45edfd2a618f866b0c06fd77cb43c5c6c4b6ad81c515873fc","target_token_ids_sha256":"8ddb65e178ceb2fa2d463b1981c1b2a0d308bd079411c8156ce1cba86f28173f","token_span":{"cache_exposed_start":104,"cache_exposed_stop":104,"prefill_start":0,"prefill_stop":83,"scored_start":83,"scored_stop":104},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e3228f24ef2326d02eac4d51adfc4dac5f98a046b3f1fa2c9d460bc590e7ad00","anchor_positions":[9,20,30,41,52,62,73,84,94,105,115,126,137,147,158,169],"anchor_positions_sha256":"1de8f9c2ad33c2ca63d619d4e737066294b86b501bd566dd6f84cd9fa731efd0","canonical_id":"867","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,30,41,51,62,72,83,93,104,114,125,135,146,156,167],"fisher_boundary_sha256":"6cdf085217e9a92a862d08a98b72af716319adcb02639b733b87d792f63b1261","horizon":1,"input_positions":[10,21,31,42,52,63,73,84,94,105,115,126,136,147,157,168],"input_token_ids_sha256":"b3f41b7d18076730b37f241371d12f38791b7734d302d899fcfb2ae98bfa7e20","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,32,43,53,64,74,85,95,106,116,127,137,148,158,169],"target_token_ids_sha256":"f68aa42c7401aa2aec59c231aba070689444ab604d38e31d8644873caf22a672"},"formatted_content_sha256":"c5d4a00fec65b64c0d75bc557b515c301fadade8597055f2ebb3d094051ad750","generator_receipt_sha256":null,"identity_record_sha256":"ac31836b26155996736a3cd0eebcf92e2d25df1fbba42f351c07926a34d557cc","prompt_token_ids_sha256":"3965eb8ca467ffc97d00622d372a5e88bf51d885d8f90daaad663ac6a3125362","ruler_category":null,"seed":null,"selection_rank":47,"selection_sha256":"207c6e9619ca7dfd647996b0ddc78892694e1df2e515a63deabfce4784b7300c","sequence_length":170,"sequence_token_ids_sha256":"a45517724ce7682620b190d1954ae73f1c66fd716f768ae8411b3ebd337b8523","source_content_sha256":"2da1fe1d0cc596300fd25d9a0f0e96da35c84b424c329057298f8e3540157f26","target_token_ids_sha256":"c7498f64ba251e766cd9dfe1464d55b2e2348ec90765cbe0ec26f908cbac2a9c","token_span":{"cache_exposed_start":170,"cache_exposed_stop":170,"prefill_start":0,"prefill_stop":114,"scored_start":114,"scored_stop":170},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2f3dfc43992ed341ed7978295644b024d0c191c9ec9c0ab7120ce608b8e69aa6","anchor_positions":[22,45,68,92,115,138,162,185,208,232,255,278,302,325,348,372],"anchor_positions_sha256":"3ce5659702a26cd1b59b1d16461526035d0580f1b9adb943021fa501179be80c","canonical_id":"622","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[22,45,68,91,114,138,161,184,207,230,254,277,300,323,346,370],"fisher_boundary_sha256":"3dd21721af5c559d0bf97f5a0d8335eac58669690242a9a65414cb07123ceab2","horizon":1,"input_positions":[23,46,69,92,115,139,162,185,208,231,255,278,301,324,347,371],"input_token_ids_sha256":"b7ce16f1b70f8a32ac421274ec67536496f84420741a5dccf20534b7d1c9f94d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[24,47,70,93,116,140,163,186,209,232,256,279,302,325,348,372],"target_token_ids_sha256":"d9f650368349431e7cbd102c1fa8783d9dc57c0d60cd0a5c4c784738af1f4c19"},"formatted_content_sha256":"216e5e12034d35e11932dcbdc1b82573653565c8944e50d962328af76cf74782","generator_receipt_sha256":null,"identity_record_sha256":"6ac868c67f1d02fb4ef0c5a4472eb0b28625461b9b81108f54d3aa3aed1b7d70","prompt_token_ids_sha256":"521456ee3e26ec55cb42c2589084173f76e5275561b51201d380d4eed502f265","ruler_category":null,"seed":null,"selection_rank":48,"selection_sha256":"210000c62df8127b95ae009f6518f2d6201b9506f6c8506835dca4eb8bbf293f","sequence_length":373,"sequence_token_ids_sha256":"1407f58699c66ad8f21f74fcf9606d047865d22dea2021a94e18700ba9dec3a8","source_content_sha256":"2000263b26b6240378d05d98725621f6c3e2bdd39265a3d356bf34ba4967d9ee","target_token_ids_sha256":"4676880dc99f1babf2689d76705c312671bc2ff81019432ec05b98c6156e2587","token_span":{"cache_exposed_start":373,"cache_exposed_stop":373,"prefill_start":0,"prefill_stop":189,"scored_start":189,"scored_stop":373},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"38589b6a6ebd3baba9bd23ed78bdceef6846e6529843ad4232525292bdec2cbe","anchor_positions":[6,14,21,29,36,44,51,59,66,74,81,89,96,104,111,119],"anchor_positions_sha256":"e83bd2e331c896fb6e9946121cbf13d77b4819161e7591880bc3055c1dbc069a","canonical_id":"677","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,13,21,28,35,43,50,58,65,72,80,87,94,102,109,117],"fisher_boundary_sha256":"683f76b65c4ebceff74d9ba1f3da44c4048119e0043947c4b3fe743eb29acf47","horizon":1,"input_positions":[7,14,22,29,36,44,51,59,66,73,81,88,95,103,110,118],"input_token_ids_sha256":"5605175529bcf8df342d6991176c6dcd594b6850a1f276f1e516f05daaf07e33","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,15,23,30,37,45,52,60,67,74,82,89,96,104,111,119],"target_token_ids_sha256":"25df50b9075a9d7a5a4d1e2bac00e89268616661a6d17f51d339ca2bb9a7ad99"},"formatted_content_sha256":"441853ff923cc424ab0aa7fec31c4aaa73bd11a24cff64d1daad0da0b63dda69","generator_receipt_sha256":null,"identity_record_sha256":"0d876619f10e2042398c541c7464cf9b347fb64e1ac8107841be3973e213cc36","prompt_token_ids_sha256":"0d2e23bc0f9b51a958475c21ec2c44735aa0f223221380fb0cbc53e877eea9e9","ruler_category":null,"seed":null,"selection_rank":49,"selection_sha256":"213beddb6f6d7179f5bc0d037d084c135da38be5e6cebc224e9ee64574990d1b","sequence_length":120,"sequence_token_ids_sha256":"ef85c6f458a4c28417a4ebf89c6986376b20129c419177fae9b89453d5ee6bad","source_content_sha256":"33110b4565d527f05bb143f7b956a5df85324d2da6b82595171ebb52b1c28695","target_token_ids_sha256":"cb45a27d57424658ef3029fa23e359436aec54c0636af1e913e0b7b0b1a63c83","token_span":{"cache_exposed_start":120,"cache_exposed_stop":120,"prefill_start":0,"prefill_stop":85,"scored_start":85,"scored_stop":120},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0af2d86cf1695b8e2f191abffee4365a9731009779912cf6c2993138f08a66fc","anchor_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,160],"anchor_positions_sha256":"cea919870992245499fb8d8a858d848e1e76f02a4e2230ada935442533c03198","canonical_id":"849","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,18,28,38,48,58,68,78,88,98,108,118,128,138,148,158],"fisher_boundary_sha256":"dacbce55717fb2ad9e2c1c565b4cdc586f652e71481489ba9f2fe0fbdb01d223","horizon":1,"input_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,159],"input_token_ids_sha256":"abbb2fdf32082b7edb9cffcb691959ba71fa426d6242d133388daf5b12207e54","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,160],"target_token_ids_sha256":"b6dd6b170c917647555dae8b2934c8757a2075a9fd62bdcc5c741d21a155b8c1"},"formatted_content_sha256":"b97e246c5fb6006e691f8dcc86a87679113f7a78d64aeac8c4bdcded6e3527a5","generator_receipt_sha256":null,"identity_record_sha256":"4e41b786817605fe72e6c9fdd1fee25d51c2700b517525bce5f2ed34d6095542","prompt_token_ids_sha256":"57f92b2bb27768c978d8e3c41a9efc9e0ebbc4d569c63d7d84dd70b17515d47c","ruler_category":null,"seed":null,"selection_rank":50,"selection_sha256":"21f0779cb4b05ae80d3df3e594cf70e52fde9b8adc7fd7457a46be4c7a8aaf45","sequence_length":161,"sequence_token_ids_sha256":"aec4c452b53bb10d8fe6bcd4d1f7770db728b6d1f1928e376b6757e469976806","source_content_sha256":"fedc2450a7af5f522a92d0e1052fa8033d9fb19a48f2d667a7f31093a26f405c","target_token_ids_sha256":"45ed26c261c710a2cc312afbf071387265818704baa2885344b040c5f1d5b70f","token_span":{"cache_exposed_start":161,"cache_exposed_stop":161,"prefill_start":0,"prefill_stop":75,"scored_start":75,"scored_stop":161},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"f8a20553c70e4c3efaf0458db6c7c1d06d0e2dfc9ab4f2783f010af84e82721d","anchor_positions":[7,16,25,33,42,51,59,68,77,85,94,103,111,120,129,138],"anchor_positions_sha256":"08b85b98703c7ea55928e16b78a87011e7254eb0d0516881c1cef449b5ad76ae","canonical_id":"756","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,24,33,41,50,58,67,76,84,93,101,110,118,127,136],"fisher_boundary_sha256":"859315451573e13275eaa90fef292f4abcb53e2d7979744ca1a559cd552531ab","horizon":1,"input_positions":[8,17,25,34,42,51,59,68,77,85,94,102,111,119,128,137],"input_token_ids_sha256":"c70dd0ee0947d887043e4ef090390d9b55dcb4940348ebb271a5299668f94c6a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,26,35,43,52,60,69,78,86,95,103,112,120,129,138],"target_token_ids_sha256":"844ca5f05e19572138e424833fe71cb51a25987fd79b470130b42fa975c1a39b"},"formatted_content_sha256":"79bb829fa41f1b56a259ec1d6f01ea3454814c377128ca4380d61741eac44e55","generator_receipt_sha256":null,"identity_record_sha256":"1eb2616b62b6f8e94a42917d3cba2e04e70983835585b47745bb90e8e2aba08c","prompt_token_ids_sha256":"f39e99e8861e0fba4e540dc710f8fcf78133eb7be50e166ae431c650dfe3c4c9","ruler_category":null,"seed":null,"selection_rank":51,"selection_sha256":"21f1cc6a98b0063838189643fcd01e4b5a84fbc7b4cc8effe18cc4d33580ac9c","sequence_length":139,"sequence_token_ids_sha256":"c8504b3fc7419f64b9e0f081c9f129180471f53a90800918c11449b486ba7987","source_content_sha256":"cb27d1dd0a93a0f5b25d8f2486a5017c8d78e16aaed3c07e71a04cc440d1aa5a","target_token_ids_sha256":"66b6c358b326ed60b26f60ece3aa3a72bae96813ca059d87e44cc7dfcdb72c3b","token_span":{"cache_exposed_start":139,"cache_exposed_stop":139,"prefill_start":0,"prefill_stop":91,"scored_start":91,"scored_stop":139},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e32814316554b0f79cc5669981964cddf032347b4da8365b25ddadb67a9a91c9","anchor_positions":[6,14,22,30,37,45,53,61,68,76,84,92,99,107,115,123],"anchor_positions_sha256":"4161f0e0da9e01e9fca23a34e70979683578e6b7ae825c0791088249cd49ed8e","canonical_id":"960","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,21,29,37,44,52,60,67,75,82,90,98,105,113,121],"fisher_boundary_sha256":"1e87180a8c7bf1f65c301b64b88ff3c34dd59beba4dc53810695b834c0c5e9de","horizon":1,"input_positions":[7,15,22,30,38,45,53,61,68,76,83,91,99,106,114,122],"input_token_ids_sha256":"3f2fcef30c7d427d14cbcac1db88f5e74dec170dbc068333bce564584133cb71","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,23,31,39,46,54,62,69,77,84,92,100,107,115,123],"target_token_ids_sha256":"cbc1bf9d8cb5df620060a192384df8638cb094155f8978e9bcebfa5f6b4e3276"},"formatted_content_sha256":"3e4da97d0bddb12409d97ddcd69beb695eb66abac525bc59990220b07aa13b42","generator_receipt_sha256":null,"identity_record_sha256":"f15b0404946c938be5d272c212d67ae55ec18c9f4abddd2bf888443f303e871c","prompt_token_ids_sha256":"d4bcbfe95a1ebf023872fe53289683ee41d253e100dab52d598c34e9239b284c","ruler_category":null,"seed":null,"selection_rank":52,"selection_sha256":"2296dea570e1139150ccd317f7f4064da5f91c662c46dc92bacd2cafbd8693bc","sequence_length":124,"sequence_token_ids_sha256":"9c259585007e9ef7b2dde2fe8a3ecce5850274464b4c662d95e215932a3c753e","source_content_sha256":"b9eab9a91cd92a96f9a16006fb04105309ec0c2a0fe6dcbb8b8d6fd5c5ad7e81","target_token_ids_sha256":"ff3e6cdf6ef65136f4d5fc6ee8370dd4b879f47033e6e5ea58283168cdb8eae9","token_span":{"cache_exposed_start":124,"cache_exposed_stop":124,"prefill_start":0,"prefill_stop":65,"scored_start":65,"scored_stop":124},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0dad65eaaf28f71b8151a7dd5bfd62b3f371591729ce9a325cd6fdb1ee5d99d2","anchor_positions":[6,14,22,30,37,45,53,61,68,76,84,92,99,107,115,123],"anchor_positions_sha256":"4161f0e0da9e01e9fca23a34e70979683578e6b7ae825c0791088249cd49ed8e","canonical_id":"741","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,21,29,37,44,52,60,67,75,82,90,98,105,113,121],"fisher_boundary_sha256":"b7544cc2c06d73eb766987d95229a15c4109c2c9f31ee7a8109e1411b1f01f8f","horizon":1,"input_positions":[7,15,22,30,38,45,53,61,68,76,83,91,99,106,114,122],"input_token_ids_sha256":"523dd1079553b7be63d71e5a3bbe2ebd41dd50b1105590b40352ba6d4ac94529","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,23,31,39,46,54,62,69,77,84,92,100,107,115,123],"target_token_ids_sha256":"2ee4f36ffbf325d3c0f22cbdeb5f755c29cd1766c4b85bc98367f2e4e3a615fd"},"formatted_content_sha256":"d9e493674ac0bc6520c04b25bf6f2a33ab4010024516b62c9111a1e85fd4b7ed","generator_receipt_sha256":null,"identity_record_sha256":"09fd441b743d4db344555f2991adfeebeff8cbe76ceaaaa25dd4bfc5c8a9c2cf","prompt_token_ids_sha256":"a05bc2e81b72ead55f4ab23081aa3740bd5e94b12c6910ff66b4d5cd8eadc104","ruler_category":null,"seed":null,"selection_rank":53,"selection_sha256":"23920e4407f636372457bfab6052016160b52662fc2489114a897651a77f02a1","sequence_length":124,"sequence_token_ids_sha256":"a1e0d450709d8ec3da71149a480fb07c80668b98425c657471e339ba4b919bc2","source_content_sha256":"1b782afb3ea77d326167c0cdc75cf3bafd8c54f38e6656373399c6745f9a6cda","target_token_ids_sha256":"97d18e68c69d67cc25a2770126b11ec9a4009b8103ec5fbc12cf9840d69f3f7f","token_span":{"cache_exposed_start":124,"cache_exposed_stop":124,"prefill_start":0,"prefill_stop":77,"scored_start":77,"scored_stop":124},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"5220a6e57a98f18ce0fc3020e6cb3e579946006f270473a46fff4ee28d307f6e","anchor_positions":[10,21,32,44,55,66,78,89,100,112,123,134,146,157,168,180],"anchor_positions_sha256":"ecfeff7f0aa8026e3a3e67c7fccf117f33e2cac59220b1af92163b27f13f2667","canonical_id":"763","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[10,21,32,43,54,66,77,88,99,110,122,133,144,155,166,178],"fisher_boundary_sha256":"10832530119cc9dc7e8f231ec358a5a567e0064379fc6c4d752e727059021017","horizon":1,"input_positions":[11,22,33,44,55,67,78,89,100,111,123,134,145,156,167,179],"input_token_ids_sha256":"d8e6511cf865ccef1847855dd93e81b5f9a377da5712f1e2b5e6b7a1d615ed7b","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[12,23,34,45,56,68,79,90,101,112,124,135,146,157,168,180],"target_token_ids_sha256":"98ea0caf56819312967e777bc6cb39044d229924079654c58f5313c625d125a9"},"formatted_content_sha256":"1b9193d68d5fbee2ffb750c200030e38b128c512a5dd53901cd0fbc6e9110bf6","generator_receipt_sha256":null,"identity_record_sha256":"a6f87c07fe76e3512de8f93c16a19c9ddf32b2cb8f73284e08fa27a983b4e54a","prompt_token_ids_sha256":"e163147f03192bd26bc2e593039155967245812898896a5b3ccb488a19a3c2d5","ruler_category":null,"seed":null,"selection_rank":54,"selection_sha256":"253e3c374826f96651ce10f0c8bf234e0019e15677351f0a5fa4ac234821073d","sequence_length":181,"sequence_token_ids_sha256":"b058735b952391450e5ec12b9e1a916c6996ae263f8a33dfff1ecab2519a6f62","source_content_sha256":"c7874b2b046cc08bdc1d3485818b5e8aeab82684fbbfe9b46528e3f93a8c34c0","target_token_ids_sha256":"6c2dfbee49b175c1cadd33dbec419143d5d228a7875f0d9be04cef929c3fa52f","token_span":{"cache_exposed_start":181,"cache_exposed_stop":181,"prefill_start":0,"prefill_stop":113,"scored_start":113,"scored_stop":181},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1f12a81f4bcfa4113d746de8d83b09b12d5bf22d5365a3e007cbd5533bc387bf","anchor_positions":[5,11,18,24,31,37,44,50,56,63,69,76,82,89,95,102],"anchor_positions_sha256":"09c822b94eb395a26f0742b1f48efa1c07ea626dba571c077685b9f6cf7f997a","canonical_id":"924","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,11,17,24,30,36,43,49,55,62,68,74,81,87,93,100],"fisher_boundary_sha256":"9076cfcc1469ae3fc7502f41192086d69ec6e14f6981282dccd0613cf5f2b721","horizon":1,"input_positions":[6,12,18,25,31,37,44,50,56,63,69,75,82,88,94,101],"input_token_ids_sha256":"9c80dd3459e67298ce46335c51e9430667ca9c812c445dd5bda165182570e1d0","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,13,19,26,32,38,45,51,57,64,70,76,83,89,95,102],"target_token_ids_sha256":"5e9a4a1aad513778895ed6ffc2980b76512073ab29d4f8af14ae4e26e16fedfe"},"formatted_content_sha256":"e2c6271b02f2f53ca02cc8d25e64ff5633054d7c8d080fc11d7ea4f2182c8230","generator_receipt_sha256":null,"identity_record_sha256":"3c4356ae7b50150676b6594053c05745a4c6c37d4741a1a66db1c3426dfd2121","prompt_token_ids_sha256":"ec37b31ed3931f32732446c5e1b53853e57d83b71f6a60ee0973fc8544644cd2","ruler_category":null,"seed":null,"selection_rank":55,"selection_sha256":"257fcddb413e4d8b091f6e8d2f933e3a3702db72c89eba0d10b57f6f9f5e5204","sequence_length":103,"sequence_token_ids_sha256":"cfe2397b7ef9e35d26711486c836646e4598be218d361850d477f09eb5e24830","source_content_sha256":"fd2c6ff7f8bea809d74c434b1e914cc7cad37c2c16e71760033867b551f8b17a","target_token_ids_sha256":"e984af6102dc89335e836cc40d95bac564e888d124d337cc3b4d2293c69a593c","token_span":{"cache_exposed_start":103,"cache_exposed_stop":103,"prefill_start":0,"prefill_stop":79,"scored_start":79,"scored_stop":103},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b4b039a0bf0a677663a27b0e6225ea5659bf81bf4816ccd4aa2be56fbfe16c56","anchor_positions":[21,44,67,90,113,135,158,181,204,227,249,272,295,318,341,364],"anchor_positions_sha256":"2f350e6327acbc5ad68acdc17f3114f18631a5418462d1efa07225d17dee32b6","canonical_id":"839","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[21,44,67,89,112,135,157,180,203,225,248,271,293,316,339,362],"fisher_boundary_sha256":"b3aaea1f73f541b61f04c11357ca1b117be9c6ed377f11f60a993f9550d29493","horizon":1,"input_positions":[22,45,68,90,113,136,158,181,204,226,249,272,294,317,340,363],"input_token_ids_sha256":"14db27ee204411b76eac1110cc2eae63a9f7b2b51b8dbe5b39e057e911aad683","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[23,46,69,91,114,137,159,182,205,227,250,273,295,318,341,364],"target_token_ids_sha256":"db3248528512ace9a5926f28510801745ad1c3f595f00a85a70220785378a26f"},"formatted_content_sha256":"1fa99f6cf27a6488f0fc65f628a1282d325113dc82a07170cace8f0ceb591a14","generator_receipt_sha256":null,"identity_record_sha256":"6b6998a44c9be26bdd92e8fd5c97777ab53fadd0893edc4f08c5b3238dd07613","prompt_token_ids_sha256":"4be0192a68dd883cc6c905e04481fe02c58edb320e51cffa3148acdb235c27c9","ruler_category":null,"seed":null,"selection_rank":56,"selection_sha256":"25acfdc7bd97d8b8918d952ccd13767a2fe3d9db392ff96de0914fe990c96767","sequence_length":365,"sequence_token_ids_sha256":"64f01ee3dd9e450ba6406f15a3eb8206442ffeb1f8bad35135cdc10036187849","source_content_sha256":"e16fff97f3fb1d8593c7f03f52f248825604aeb9e11bb511b613867c12fe0d69","target_token_ids_sha256":"beb8edb12ebba49a31efbb316488aaea2754c69c9a528a3fc8ce2e25cfd56f2b","token_span":{"cache_exposed_start":365,"cache_exposed_stop":365,"prefill_start":0,"prefill_stop":292,"scored_start":292,"scored_stop":365},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4e7d65195091102780a3b899d1d9c169f8999fd52b4fce6cd4807da888ce72a6","anchor_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,207],"anchor_positions_sha256":"db093e77a831efd90fa9c853cbfd07cb4a1ce073a3d26ebd584589b17a67dcac","canonical_id":"870","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,50,63,76,89,102,114,127,140,153,166,179,192,205],"fisher_boundary_sha256":"e55af1a7964dd59bb74a21c91ffa723d2be196dc03399d61f54ed5e27acc11f1","horizon":1,"input_positions":[12,25,38,51,64,77,90,103,115,128,141,154,167,180,193,206],"input_token_ids_sha256":"7e51406e9532ccd83f5838a79143de8b54bddcb3cd4af48ff57ce8fd2068004b","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,52,65,78,91,104,116,129,142,155,168,181,194,207],"target_token_ids_sha256":"8dfb99d49da149bda798f4f1bea7693afcc685391f66174bd7dbd7f9f212d23a"},"formatted_content_sha256":"0671c01c177d3d976dc26f74b1532f125da8dbbcba201bd969a8df89be404212","generator_receipt_sha256":null,"identity_record_sha256":"80065f067cbcd3b6b24a83d3b0920140345cdb44209861d9942cfc40e6285afd","prompt_token_ids_sha256":"11ac1cd8c2c5a0bfd9a43b00c6dbafc129c255c7fcab76b0fccec71b0a1ca47a","ruler_category":null,"seed":null,"selection_rank":57,"selection_sha256":"25d5e31e60deea1f490886549d37023191fddfed4a7488c2ecb258dc7b690a34","sequence_length":208,"sequence_token_ids_sha256":"b95e7638baa81487e766778b92d51418007619c24c88596cb9d255d17707ad82","source_content_sha256":"25b998a219b3e624a18b48a424e9201d591646f6ccffd2c34ccd994d27c33467","target_token_ids_sha256":"2c9edc0f2e71ceb46be5cf74a768dd85d0f0568d35da9346930b4437902992c2","token_span":{"cache_exposed_start":208,"cache_exposed_stop":208,"prefill_start":0,"prefill_stop":171,"scored_start":171,"scored_stop":208},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7179a32a80e84307ba53490fd3d26ee4976520fb95463fd4d99c5eeea5f95e3f","anchor_positions":[10,21,32,43,54,65,76,87,98,109,120,131,142,153,164,176],"anchor_positions_sha256":"1a7be41f5dd5dc6797d88e73a587747cdd29dd38e9373f3f584d8051e8e93686","canonical_id":"862","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,31,42,53,64,75,86,97,108,119,130,141,152,163,174],"fisher_boundary_sha256":"1eb3dcd5d0f40487914047d0829182878435e58383687684c8e2311b203aae37","horizon":1,"input_positions":[10,21,32,43,54,65,76,87,98,109,120,131,142,153,164,175],"input_token_ids_sha256":"ab632089e4e539c8bd8472cc5c658a6c7457d8f007cff89f0e00204509423230","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,33,44,55,66,77,88,99,110,121,132,143,154,165,176],"target_token_ids_sha256":"a1faf4cf5f2ef1412bad68d667096b1c2dd483ffe1a6e641ce38ec7132214cfd"},"formatted_content_sha256":"16b2614d3f18e7e4f72755cb6542e4c740bbd15964d402f5b8deeaaf2091d4c5","generator_receipt_sha256":null,"identity_record_sha256":"40dd54e4e34693f385579ecbdbbfe9d627292003fa2bb0e3fa8bb8b2106964dc","prompt_token_ids_sha256":"a20a042120ba3b2dda09c804a843b20532bf4809c88d3fbe22620ef7ac56bdcf","ruler_category":null,"seed":null,"selection_rank":58,"selection_sha256":"26d4c81b5d924024df658aeaf2f322dcb51e74e59299d6c5bbd15f755f66be5e","sequence_length":177,"sequence_token_ids_sha256":"f51964e8ff8c85d8f090c0e12f2c0d8d6234b49c9b6256141d05ab65cbec2f56","source_content_sha256":"4ca1375a378716ffef02c6f4407d8935736ee95bd694754667893b4d008fe701","target_token_ids_sha256":"643aa7765da4c82ea6a56654b78d8a8b5489c8fe86d2c44a8b4ac1960295ddbf","token_span":{"cache_exposed_start":177,"cache_exposed_stop":177,"prefill_start":0,"prefill_stop":130,"scored_start":130,"scored_stop":177},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"c008fb00c4b75ed5a7d4956b63ea532e7def48f60c4527826df4af3da44bf419","anchor_positions":[4,10,16,22,28,33,39,45,51,57,62,68,74,80,86,92],"anchor_positions_sha256":"adf3767c4c3aef64d448368c378ec2e8623df9971d13f03dfbf0a245e0d32c61","canonical_id":"877","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[4,10,16,21,27,33,38,44,50,55,61,67,72,78,84,90],"fisher_boundary_sha256":"2b4cbf4ae8461adf7ece8f2d75ed8ba8fa4760a94e8ccbc37beabe48a2e53822","horizon":1,"input_positions":[5,11,17,22,28,34,39,45,51,56,62,68,73,79,85,91],"input_token_ids_sha256":"11d9eb276cb6e38836f4280a877421b19798f0a03c8c6a955cf798cffbc5e066","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[6,12,18,23,29,35,40,46,52,57,63,69,74,80,86,92],"target_token_ids_sha256":"9a807734bc0b75cf522407e78a2432d110e1c4f600034c299f2c1f1025bf651b"},"formatted_content_sha256":"f0d53963f87750fbc186ec354a9b0e5f12a9937a4a1e0aa5b14a393ae9209437","generator_receipt_sha256":null,"identity_record_sha256":"b69efa65a89b1b762f27440c2fa6d016a863c179526e602947c476c8cce0c500","prompt_token_ids_sha256":"decd083790a01b0d5ff5f3cedced5d9fc60db4ac239215f87d18bd13829340ee","ruler_category":null,"seed":null,"selection_rank":59,"selection_sha256":"27e1898502ba84f86677c9a8c281f209e5e5ca3ef3fa0ac74ccb2e5a0f96925e","sequence_length":93,"sequence_token_ids_sha256":"0c186015d98c4b3b835615571d680037c4e9ff03bc0bf360d7edcc64ea7e02ce","source_content_sha256":"a4d868515f48168874e3345764d92c8f9bed1be2211fa137d653955ce91e9231","target_token_ids_sha256":"05325f40cfa6f59d7dae81f8cf700730aebfff74fb1d0bfbf6df3b22888a7ec9","token_span":{"cache_exposed_start":93,"cache_exposed_stop":93,"prefill_start":0,"prefill_stop":71,"scored_start":71,"scored_stop":93},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"69bc3d226f2cb8a6dd05d7b71e6b82bd59279baf47672ff9264b7f90657b2ed6","anchor_positions":[9,19,29,40,50,60,70,81,91,101,111,122,132,142,152,163],"anchor_positions_sha256":"868d4ebf721b69ebc32ed80fb23287a93509d3cda6e8df7dfacdd2b56b0be622","canonical_id":"672","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,39,49,59,69,80,90,100,110,120,130,140,150,161],"fisher_boundary_sha256":"824632fca0be6e4fc45bd8a157d6c3d7331848be3ae62303610c9ced5cdc4fe5","horizon":1,"input_positions":[10,20,30,40,50,60,70,81,91,101,111,121,131,141,151,162],"input_token_ids_sha256":"4a43f0478e855c786a9ba41f412abef099a0aa6d336add8b63253a02536aff3a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,41,51,61,71,82,92,102,112,122,132,142,152,163],"target_token_ids_sha256":"5a469a6d9ca0e01f116bfc1713d0adff5f5c3384fa311978e20183e1b4e4cf16"},"formatted_content_sha256":"bd36f5a6ad55b181f08602b3596e02127fc18650dff36d4e78601272f208d8d5","generator_receipt_sha256":null,"identity_record_sha256":"b5b377fb132254b8031c1002255dcb18bec16d00c60ca2ad998a8649aee007f6","prompt_token_ids_sha256":"6b00d6059701e0be4b83e9dac571ad728ef3c38c00c26f87d6b67a85ef08f81a","ruler_category":null,"seed":null,"selection_rank":60,"selection_sha256":"28dac04649a15d3243c93bc4206efd4f0acd9190fdbbdfa2ab5345af799d2bf1","sequence_length":164,"sequence_token_ids_sha256":"db9853f0252606fe408f3f653940737ffa69d88b087eaf4b282f42551da66ed9","source_content_sha256":"2e61b0813b9f940f7d87c5121e73ed7a0b8b2d507012929e153f7b120c8c94ea","target_token_ids_sha256":"4282f90a8268a9ed7bc92f471b79152055c095686127816dd9ad5e82abe8893d","token_span":{"cache_exposed_start":164,"cache_exposed_stop":164,"prefill_start":0,"prefill_stop":87,"scored_start":87,"scored_stop":164},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1de6107371ef2883413c366d370b6b037d94c76ec7232ca2b34bb6d3efeed728","anchor_positions":[13,27,41,55,69,83,97,112,126,140,154,168,182,196,210,225],"anchor_positions_sha256":"87cf1b468f68c1b9ae4a445f5ca6ddac019379b906bcb9569e18bc77f8dab19a","canonical_id":"894","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[13,27,41,55,69,83,97,111,125,139,153,167,181,195,209,223],"fisher_boundary_sha256":"50541ef032b9ebd546fb5146735c5dc646a875ccea81f95e3135cfb092f777e8","horizon":1,"input_positions":[14,28,42,56,70,84,98,112,126,140,154,168,182,196,210,224],"input_token_ids_sha256":"29b3d6ebd87eeec326946bda12535cdd931e5c75481c27025e5044b0cf8d5ae0","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[15,29,43,57,71,85,99,113,127,141,155,169,183,197,211,225],"target_token_ids_sha256":"468a710354f96e062d0f534f2df0e373bceff5193c12fd9ae67b05776f4cb12d"},"formatted_content_sha256":"85fe4f431f987f9bcccff54e9fff464b3411229d14fbb421bec0c42bf02159f6","generator_receipt_sha256":null,"identity_record_sha256":"78c4954da5b1f2cd2dea1777612b390c51dd7f053d98f84f8dd93e4c16182ef1","prompt_token_ids_sha256":"366bcf6fff0a082cca6c159ef4ee352857b3adb5dc747a33fbe9d5a187cf1a70","ruler_category":null,"seed":null,"selection_rank":61,"selection_sha256":"2a07715411e7dcd5d13fd6bdab1eedf96aa272627f4dcec5410c9273f87bda59","sequence_length":226,"sequence_token_ids_sha256":"060aa83ed0ef449b43aebe0dba81de96624ea81a91b9f2bbf5346ec7ebba5d2d","source_content_sha256":"25f8c228a65fe0501ff126cf8215ef8c40f73ba446adccaac8696c0592d9cb99","target_token_ids_sha256":"5d62950c056f097d29c50fc4fa74da37364692566b85b25d8801ce7fd0cfae6e","token_span":{"cache_exposed_start":226,"cache_exposed_stop":226,"prefill_start":0,"prefill_stop":198,"scored_start":198,"scored_stop":226},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e9fc3e673c65e510fa4e60bee32295e16ef953843d59dc679325a149cb63954e","anchor_positions":[13,28,43,58,72,87,102,117,131,146,161,176,190,205,220,235],"anchor_positions_sha256":"a5ffcefdcb22ed36cff0992ac04165c50a214c58858b64309c2292bae4ad2df4","canonical_id":"911","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[13,28,42,57,72,86,101,116,130,145,159,174,189,203,218,233],"fisher_boundary_sha256":"a343b73549c45e626b75cab9d67da7ec20d01d1df92c5ea43c27940cd5ac7b0d","horizon":1,"input_positions":[14,29,43,58,73,87,102,117,131,146,160,175,190,204,219,234],"input_token_ids_sha256":"fd590f614ed1c54400b44a8923440159f0d548ea06103f72a56c7794eef8d355","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[15,30,44,59,74,88,103,118,132,147,161,176,191,205,220,235],"target_token_ids_sha256":"84f73363046430aa4e0c6c4dcd11b1cf627b0582b1c34fc1e0b65e35a6e9f9db"},"formatted_content_sha256":"3af453e7ae9642b7b3a6fcb1a5eeaa953e64e0f8b2a19e654ef7c76e72f1ff54","generator_receipt_sha256":null,"identity_record_sha256":"f4ea17bad40bae1caf3eba8695e7ad325112bd00019e2511ff3614fb4f3a37f4","prompt_token_ids_sha256":"0b618b4457d4e22f6ca61e219aee86cfa88f0ae704ed487315c854f83ad3c11b","ruler_category":null,"seed":null,"selection_rank":62,"selection_sha256":"2a3d19972c5985a3c0f494d32fc1cf627ec69e3ceffcf0239862d2c14a69ee16","sequence_length":236,"sequence_token_ids_sha256":"4b476aa638606f6f73086d180b8d59e8f59f03a7f0df8b934e603b7a121b0987","source_content_sha256":"fdcefafb28d63d4587a84cb4bad47b369563ee74a482a1216341f7f6dba5f2d6","target_token_ids_sha256":"de9d6a1e7f34b8648d9cc024467db18a0100561d5d78da9dd043028f0c4b1058","token_span":{"cache_exposed_start":236,"cache_exposed_stop":236,"prefill_start":0,"prefill_stop":171,"scored_start":171,"scored_stop":236},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7b01d8ec76dac2b7f8b13a0dee07ad2225b142de86be4c08fe417f349fab06b2","anchor_positions":[11,23,35,47,59,72,84,96,108,120,133,145,157,169,181,194],"anchor_positions_sha256":"366062d114b4b8abb7d67fff75b399eddce3c91d9f37bfd07f29cdd16e658363","canonical_id":"907","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,23,35,47,59,71,83,95,107,119,131,143,155,167,179,192],"fisher_boundary_sha256":"3e6fc1c5625bb4f9b6651ced2a909d037dc5aba4278dcfc13b85144e9142af0f","horizon":1,"input_positions":[12,24,36,48,60,72,84,96,108,120,132,144,156,168,180,193],"input_token_ids_sha256":"ede325c0962debe0dc616d99ffbf44bf49b11833fab47e5f51463bdf2ffd6011","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,25,37,49,61,73,85,97,109,121,133,145,157,169,181,194],"target_token_ids_sha256":"1d7cf9fe79dc3a0f99a9c1c0dc95972f78b47a89b20d976ae513c41b67a2b4ea"},"formatted_content_sha256":"99bd7655cb5e40f7f33cb82dee142cbc78854fc3a5fe6777c50f8ee8f6340cbd","generator_receipt_sha256":null,"identity_record_sha256":"8d7818a7f7ed0b3732c51529861e8d1f37b442f565e0755ae9412f4e544cb0d8","prompt_token_ids_sha256":"1d66bbe818bc1c82dece92e787302ed8be7bcca6dd90590e6de294c4d367d8be","ruler_category":null,"seed":null,"selection_rank":63,"selection_sha256":"2a42596fa08d9ebf6f8f895ed62f34cfc00f343a1b8fb9eaba4fce50e57632ff","sequence_length":195,"sequence_token_ids_sha256":"0239420cf0bc52b4ff133acfa7d4ba2d445aa4efa7ca99f95e10b1e7fb7bff32","source_content_sha256":"523274b507fb693b2bb38a9148d6f328b15704929202f43a1eadb3f695c038b9","target_token_ids_sha256":"8225f2a151273aa3c714667358b8a7f304f775c7bed4fa3e4aececc369b6aff8","token_span":{"cache_exposed_start":195,"cache_exposed_stop":195,"prefill_start":0,"prefill_stop":139,"scored_start":139,"scored_stop":195},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"dca37f2fe51bc394708fd6c2564b0cbf119163fd665608a4d93cd495b3998ad1","anchor_positions":[7,16,24,33,41,50,58,67,76,84,93,101,110,118,127,136],"anchor_positions_sha256":"b394d1b64998869298fd2b292d4ab84e6faf0b5a40eeb811c668f2563bb4df64","canonical_id":"744","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,24,32,41,49,58,66,74,83,91,100,108,117,125,134],"fisher_boundary_sha256":"31ee43d6395a61f1005ea89235db2d9169957c4fa0ecd817d09d92731e4bfff7","horizon":1,"input_positions":[8,16,25,33,42,50,59,67,75,84,92,101,109,118,126,135],"input_token_ids_sha256":"305b211372837f28d3c09a9195ff9da8870eb18255d1a5ade3aa23447ae066d7","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,26,34,43,51,60,68,76,85,93,102,110,119,127,136],"target_token_ids_sha256":"55671dc7b5a880e5aa6b49754d8ad5916d76392b74ba47f6eb14a67a99a65dd0"},"formatted_content_sha256":"24bfd2c8acd42c24dd0175dade66586d4ef9c8fba6740ab43be8a40d2344a8f0","generator_receipt_sha256":null,"identity_record_sha256":"0d3378f11bdab707d5a5e866bf31ece77bae782c5e98264606c8669811279e26","prompt_token_ids_sha256":"45cbb79cde4efb50926321115cb572632eaa8fc91f1b03e56edb1ec881aac0d1","ruler_category":null,"seed":null,"selection_rank":64,"selection_sha256":"2b946010dfbfbf789ca965f025d6a4ef990e3f3685bc221716f664a8af147cdc","sequence_length":137,"sequence_token_ids_sha256":"27302f59061cb4d77641b855190e68d1082efe4fa19ef17f6d1886ea443d271c","source_content_sha256":"388472fd029ca253a1e8ebe0b058b306b19e8d0a99d80ac0ff0cc80f6ef6401e","target_token_ids_sha256":"f529366ede3a080401140af1ed24d8b5dc889784a050980a1baf21988730f4ef","token_span":{"cache_exposed_start":137,"cache_exposed_stop":137,"prefill_start":0,"prefill_stop":106,"scored_start":106,"scored_stop":137},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"72f1915bbcebb3cd41116e7d18a62d81645a57d6ab6aefc5dd4d833040f03e86","anchor_positions":[10,21,32,43,54,65,76,87,98,109,120,131,142,153,164,176],"anchor_positions_sha256":"1a7be41f5dd5dc6797d88e73a587747cdd29dd38e9373f3f584d8051e8e93686","canonical_id":"858","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,31,42,53,64,75,86,97,108,119,130,141,152,163,174],"fisher_boundary_sha256":"dce1d589b941aa5d007836b839a80510d6ed27d69a6bc4826bc4f9d1be58fa8a","horizon":1,"input_positions":[10,21,32,43,54,65,76,87,98,109,120,131,142,153,164,175],"input_token_ids_sha256":"1f8edd352b430cc0ec42d176428e4dc421cd81f094568d0cef527f3c289d718a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,33,44,55,66,77,88,99,110,121,132,143,154,165,176],"target_token_ids_sha256":"bd43a32b864185de3d60dd13a588ea5c44371aff58aa9439eee1749afedfd427"},"formatted_content_sha256":"ffcfea4052a70d3b74318b9aa27367efd191b6bb39153e56b0d8c6be270bf7dc","generator_receipt_sha256":null,"identity_record_sha256":"4303eb5887d9002f18afa44a53d86830e876c3605b807968cb2327574864c1ad","prompt_token_ids_sha256":"7e2722d66f52a4b3e9bc2b499fe7271693a49bfccaf075db172096a06590501c","ruler_category":null,"seed":null,"selection_rank":65,"selection_sha256":"2bdf2de5737b5e9e9c804506d9b637c0b3e4c429c713402914877db9e163698b","sequence_length":177,"sequence_token_ids_sha256":"1bd4f0e350599ac78e054ba1f3313444f17aca30b3ab8a7d1b97b0a4db36795d","source_content_sha256":"30117a9c1f5c8fa1a71aa5d47b10e7b6a6ed631272d09a212703db54dc4e2b48","target_token_ids_sha256":"8597d05e46c539c2c6176333f41f2d43081ba451957d9e6fb3fab21ad0354486","token_span":{"cache_exposed_start":177,"cache_exposed_stop":177,"prefill_start":0,"prefill_stop":161,"scored_start":161,"scored_stop":177},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"90e208ba484962acfe5a450f2238cad06fa1b1e78cb9b693ad7f8e119d289e81","anchor_positions":[9,20,31,42,53,64,75,86,97,108,119,130,141,152,163,174],"anchor_positions_sha256":"3a23db0bdf537a9cc8cf4af3c5b682f02edafbf8e11933d0867e7f5a84a5d5fc","canonical_id":"908","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,31,42,53,63,74,85,96,107,117,128,139,150,161,172],"fisher_boundary_sha256":"3ddff55453bdfe39ff17279167d0bf79d250df7e3566058fbc8400d75148e0cd","horizon":1,"input_positions":[10,21,32,43,54,64,75,86,97,108,118,129,140,151,162,173],"input_token_ids_sha256":"a57955a9e52e60a3866dc92a0785d9ec79382d7f1e9d9a6fbe931b41cbb23f05","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,33,44,55,65,76,87,98,109,119,130,141,152,163,174],"target_token_ids_sha256":"08a89b4fe30bb44a957a32e4dbff9cce9f766b22eb990ab1c15ed78cf1409870"},"formatted_content_sha256":"542026924d7c553982b9f71b589a6b03aa17c1a9071232b6b352a8d6de880b4f","generator_receipt_sha256":null,"identity_record_sha256":"9ab67c828f5ebad51f93eefcc25a6f64e7b2efa682ddcaa0b13cdfaae34bb357","prompt_token_ids_sha256":"92dc6abd8d2bcf9552cb9117b89f426545c93d4bf69ad859fe2e22691583d830","ruler_category":null,"seed":null,"selection_rank":66,"selection_sha256":"2bfb0c5069394a29410fe17d912939039348a3279be0932efb83b1a7c531f5cc","sequence_length":175,"sequence_token_ids_sha256":"470fc1e9439dda64ef76f847132fd61d9cc29b4d1eae89d0d27b45dcf9b147bc","source_content_sha256":"7f2ca7ee05543293bafe5fc5ea510834c2eadcb6c383bdf04c72ec8b68a255ba","target_token_ids_sha256":"fad64c8dc2d584216528c7db193f37eff8c35a166fa9f081fea4f50fc1596f9c","token_span":{"cache_exposed_start":175,"cache_exposed_stop":175,"prefill_start":0,"prefill_stop":143,"scored_start":143,"scored_stop":175},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"ae2a13b9227b88c92f44ed18f4a1e74a1af336a5e505f3993199f634831d84bf","anchor_positions":[9,20,30,41,51,62,72,83,93,104,114,125,135,146,156,167],"anchor_positions_sha256":"eaf3a0e814f88177633695c30e9a05b6b492aa64c90bf98da98cb9be3917c812","canonical_id":"695","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,30,40,50,61,71,82,92,102,113,123,133,144,154,165],"fisher_boundary_sha256":"00b58a22e1ec099f9f89a453c68d32a00b7de79e01353970212f4486aea7a91b","horizon":1,"input_positions":[10,20,31,41,51,62,72,83,93,103,114,124,134,145,155,166],"input_token_ids_sha256":"0d02b321a2c7a93e523c4102c0a515f78295358243305ecf63572a9b9251f793","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,32,42,52,63,73,84,94,104,115,125,135,146,156,167],"target_token_ids_sha256":"479412a3d9bc5c68516840b01dfe7229e1ffffaec904e9405309df32f66e5318"},"formatted_content_sha256":"2c108150869bff59ce632e2f1cf6126ae8d1e5449fb81fa352e6080ece8e18d8","generator_receipt_sha256":null,"identity_record_sha256":"b8d1fbbaf80e50801cbfefb607f87f2ee0be785618f9d8733e4ec1cec193070c","prompt_token_ids_sha256":"12d9523360c9b9bbbc77331eede017f4df2b2073b06793fb6caaa4264ed356eb","ruler_category":null,"seed":null,"selection_rank":67,"selection_sha256":"2c8457d859c30b49c20f86b5a6d57cb9d3f87babdd78e191666700bc6b19c2a9","sequence_length":168,"sequence_token_ids_sha256":"61a8f24e31170e6510a3fcb1c13e03df674e6054bc07e8c1b50b2869262279dc","source_content_sha256":"f26ba058dff18925efba9c51a7c584d713db98910e092b2b8efad4050d705a97","target_token_ids_sha256":"331d6876269e5c72808d45d0ff8043245793020b7f0c47f8dff86c3d5896440b","token_span":{"cache_exposed_start":168,"cache_exposed_stop":168,"prefill_start":0,"prefill_stop":124,"scored_start":124,"scored_stop":168},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2776f19ea8ad43a4ca532ba12548c47a68fb5d577db33f2001594d30daae4b47","anchor_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,127],"anchor_positions_sha256":"be2b9b89820981ce1f31651cf7733d2ed9a608d27c6984c26e1c299ac3d463c0","canonical_id":"674","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,30,38,46,54,62,69,77,85,93,101,109,117,125],"fisher_boundary_sha256":"9fb839c7596b94297977a40da9a8bdea3ac37720cc556e470eac01fad7d8fd26","horizon":1,"input_positions":[7,15,23,31,39,47,55,63,70,78,86,94,102,110,118,126],"input_token_ids_sha256":"4f531382690861c06fb87e927c47ddec1b08260da41fc61f8ade85e2dbfa2447","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,32,40,48,56,64,71,79,87,95,103,111,119,127],"target_token_ids_sha256":"58f5f25b01c19e81aae2e0327e1c394b668618a5926e174aeb8d4db536ad7b5a"},"formatted_content_sha256":"11e4332442cb1751b96af99218e5ed832e0741b4bd279519954cf8ff75df3000","generator_receipt_sha256":null,"identity_record_sha256":"91c60f95fb1de5699a93eac54f860ec4b636acdf00d4bf36d47cb7038568d0e6","prompt_token_ids_sha256":"fc30e67c09e80fb8040481e2b49df5d4de0abb2ca503e59d5f7fe81b9053ea14","ruler_category":null,"seed":null,"selection_rank":68,"selection_sha256":"2c910566f006288010eaa2a19e63636d5cdb71c4dbe532bbb2f800b8bc2d83e8","sequence_length":128,"sequence_token_ids_sha256":"c6637a5c180bd2cb7b6ae0114077150e40aa103770a4414d373677e6197d3eb0","source_content_sha256":"b085122555255bd8b48a5d7202b27dcfa83512407504a3a7d36c26ab0971701c","target_token_ids_sha256":"bf8896134a16b2defd1fd3ad3a1e041bed0c2522bb90fb8750e2d750970281f4","token_span":{"cache_exposed_start":128,"cache_exposed_stop":128,"prefill_start":0,"prefill_stop":92,"scored_start":92,"scored_stop":128},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"62ab5323d823671ba9386621641396cc44f5d6d879e1ef5cedfd73f5d42ea49d","anchor_positions":[5,12,19,26,33,40,47,54,60,67,74,81,88,95,102,109],"anchor_positions_sha256":"04b0f7fc4a0267db36244bf79b1775bc8efc674979a8a894100f0164758c7107","canonical_id":"937","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,19,26,32,39,46,53,59,66,73,80,86,93,100,107],"fisher_boundary_sha256":"d621358bf5b31a9a6e1bf8b7861671a5c3b7d8f4b833117f22528a32214d9059","horizon":1,"input_positions":[6,13,20,27,33,40,47,54,60,67,74,81,87,94,101,108],"input_token_ids_sha256":"6b99376f37916da3177e56d8dbe872f5b54a721dad5a56e3d0f7641537953001","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,21,28,34,41,48,55,61,68,75,82,88,95,102,109],"target_token_ids_sha256":"6caf9fd439e8d278a1020748e4978b74310b3608c9d071b94f12ce03d0b9be62"},"formatted_content_sha256":"3f1c5b9ff9268e2a55000da43f44ffd96c58159b9099ecb106bfc6d0e882d405","generator_receipt_sha256":null,"identity_record_sha256":"2aef9e476b0e90a8d4c54f89251fdd678232c30c0588a40c71c3ea135c3a140e","prompt_token_ids_sha256":"de6bf797fe9bca2c9736422f1027832ef62d4311eea81a92442bdf94a157a32d","ruler_category":null,"seed":null,"selection_rank":69,"selection_sha256":"2db747f786476cd53b2b76e66050c7c9524096bc2cc634dc8949c6700bbb91e9","sequence_length":110,"sequence_token_ids_sha256":"f66fae77a66de49a19604647a7a1dcef25341e73c3beeb2dfc54b05038100bfd","source_content_sha256":"9427a32773d2dda1a56f98cb0bdbee88ed5634a53a96ba4f2adf32764392e2d0","target_token_ids_sha256":"9d11355be764566fab4daa7e657723edeee80c1c0d05f8dcbfe888b93d368076","token_span":{"cache_exposed_start":110,"cache_exposed_stop":110,"prefill_start":0,"prefill_stop":73,"scored_start":73,"scored_stop":110},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"be38fbf54d72a7f68805fbe18ccbdf7981341fa3db7c90146ed5584b8a5632cd","anchor_positions":[25,52,79,106,132,159,186,213,239,266,293,320,346,373,400,427],"anchor_positions_sha256":"2c63cecba171fd759882d1a320b87c2dfaec15e4490a663f9e1099a229e8e934","canonical_id":"783","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[25,52,78,105,132,158,185,212,238,265,291,318,345,371,398,425],"fisher_boundary_sha256":"df1eb852686d1c5fa6f464d0608a8ec972c4dfdb569e90a9d61ac80e9dbbd261","horizon":1,"input_positions":[26,53,79,106,133,159,186,213,239,266,292,319,346,372,399,426],"input_token_ids_sha256":"b4cb4bc55e5f1df30d752a6409003f2a8cb96722509b10f768d46b67657faa6a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[27,54,80,107,134,160,187,214,240,267,293,320,347,373,400,427],"target_token_ids_sha256":"22fcb95368b58ec5a4c13f727526665b92519bc8752f8ab11b799ee2ec751029"},"formatted_content_sha256":"0d8b83c24ac54c8b3c1a41092dc210a7f174f870033838f69387ee0a959cb7be","generator_receipt_sha256":null,"identity_record_sha256":"fd3b5341382c42b888558f7ec9d99892ab4bc6c62b5fea552dfd262f705be6f8","prompt_token_ids_sha256":"3c17fc671c321556046900784e286f9b2efcfec63411dcd19bc16d80f81efb33","ruler_category":null,"seed":null,"selection_rank":70,"selection_sha256":"2deccaa412b35e79530ed8315fcf11416ca3ef9bf945badf8e69214d6c67f0e2","sequence_length":428,"sequence_token_ids_sha256":"923d6ae58844ab43d7c92cc96780db2f0a68359b436399f8f4b84e8bd73850df","source_content_sha256":"3bf5dd95dc2a714c277cb345634f0add55f38f0c23e30c02e2fac976cc5e3560","target_token_ids_sha256":"89d552cd5c74ddddddc518b6c4d8776915650ab6c296da75bf7d8dc8d5e8a3cc","token_span":{"cache_exposed_start":428,"cache_exposed_stop":428,"prefill_start":0,"prefill_stop":201,"scored_start":201,"scored_stop":428},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"d8cc6a91dfe0b8725032b9dbe17d00c2267d1f3903f0f46de298aef09a0eceb6","anchor_positions":[7,15,23,31,39,47,55,64,72,80,88,96,104,112,120,129],"anchor_positions_sha256":"c03e2e14b7ad2333a14fc199839382794d3c90c5842d21c3c2d26acb100d0b9a","canonical_id":"792","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,127],"fisher_boundary_sha256":"36cbcaa5f2a243260e4b704ad965c1eb8729226a615436df507566bab0a15d14","horizon":1,"input_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128],"input_token_ids_sha256":"43ea65b95cf29e7cb303d7b58a8466e49d86ff3b94f481b088aedd213565b50c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,33,41,49,57,65,73,81,89,97,105,113,121,129],"target_token_ids_sha256":"09ed1cfb8661ecaf52c064021fdaac9e5e45d85d244adc29abbadd32cbb23a1b"},"formatted_content_sha256":"84324342a9988ef788199714ffd6a7b7f6493516f982634aeb0fab6e891dca3e","generator_receipt_sha256":null,"identity_record_sha256":"9ea4728a95faf70fc41786042f177a0a108a52c817d7f798a50fa559e6bfd72f","prompt_token_ids_sha256":"e2b25c9110fe041500ee35bda9d717a5b8764c0af9684a331d986da4d503a12d","ruler_category":null,"seed":null,"selection_rank":71,"selection_sha256":"2e13ba7ae758055511fc93fd8b21f4c8279b266645abb65bad13f929bab6d4ec","sequence_length":130,"sequence_token_ids_sha256":"173382b09d1a8419bd02b4281c9d8ea5342d4cee26d9006629bdfe38b23fc3ea","source_content_sha256":"8b4ec72b56e7b421cfd5660acfeaf5323dcfcb622b00a3b4f8a55c2bb8a599ab","target_token_ids_sha256":"45b965687a80da78d6902f0642a6a41c15d5b6065995e352b178741376df5858","token_span":{"cache_exposed_start":130,"cache_exposed_stop":130,"prefill_start":0,"prefill_stop":117,"scored_start":117,"scored_stop":130},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b526b8b7bf92e5b467bf63c4c44c32a7d3baa99496c16d829dfc10c90ef8e179","anchor_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"anchor_positions_sha256":"debc2f860d51bf30c9ccfe7850d3428800e80a005f976ab2943a8642c33fb808","canonical_id":"848","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,30,38,46,54,62,70,78,86,94,102,110,118,126],"fisher_boundary_sha256":"eb5082f8394245be99a854bb9e95a0e8a24b830d5b4e12d9acf89075a71132d3","horizon":1,"input_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,127],"input_token_ids_sha256":"2488867fcffe8e28aca7f38c1051a25ba6e3576b5816a851d585a91e5d1fb9b2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128],"target_token_ids_sha256":"67aae898043f0810d7b6435c2dc11fdfee0f51cf372142272e7060f3d549bd8e"},"formatted_content_sha256":"030fb32fa54afa33ec26144f6963d266d6aa4b969906d8ea12cf708055b33d67","generator_receipt_sha256":null,"identity_record_sha256":"992345ddf332a9b3994c60bd506b547f20b7a90c821dab684435a5333bc84659","prompt_token_ids_sha256":"f4528ab0cef6a0aedb91c8fdfdd0fd4d1d19f26b5c42b52cbb5e9d55c87e496a","ruler_category":null,"seed":null,"selection_rank":72,"selection_sha256":"2e8927b105ba240d08737397b6e4ae7e731e45937512629b5d05c9e422358ce0","sequence_length":129,"sequence_token_ids_sha256":"bdb29154778fe78320ce09bc48fb722bce1a1cb08026d6843fa036de3c59c397","source_content_sha256":"2a1f6cc18c5e358f530bd92c65c306de3dce59f6ab5bf9e55ea9c27c9608156c","target_token_ids_sha256":"d4d73429c6a8b75c9e50462ee645fac5d41af8e10ac5bbeb75778385684a24e5","token_span":{"cache_exposed_start":129,"cache_exposed_stop":129,"prefill_start":0,"prefill_stop":96,"scored_start":96,"scored_stop":129},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"a8b715cff28765151d67536c865a8213c840d7a8b7b255ed28fa0f4f4d86e4c3","anchor_positions":[5,12,19,25,32,39,45,52,59,65,72,79,85,92,99,106],"anchor_positions_sha256":"1ec66db99ded67a6b74f7dd5259127f2ba61fd9a33265762c1ea04710558d7bc","canonical_id":"973","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,18,25,31,38,44,51,58,64,71,77,84,90,97,104],"fisher_boundary_sha256":"9a38f0cf45f0d18c982e6cd4e3041a5c05aaa808ae9c3cc11acd1e5bd81ba1ef","horizon":1,"input_positions":[6,13,19,26,32,39,45,52,59,65,72,78,85,91,98,105],"input_token_ids_sha256":"beda690cd5123bf4bea71d0d29b89a37efdf1c91f271681dad45a48428687cfd","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,20,27,33,40,46,53,60,66,73,79,86,92,99,106],"target_token_ids_sha256":"b2959f450541356aa81bb0e5965581933b267e9e8e2f7bff19f500f4677cd5d4"},"formatted_content_sha256":"03a49d9fc1d2e40de0d006e558a4cc366fe5fc106b78e9dcce28bf991852272b","generator_receipt_sha256":null,"identity_record_sha256":"92e9205a716392965076d0c07b2927b8ceb264553c8ddf5e51d643b2c4447675","prompt_token_ids_sha256":"a60f5de6590c6b61e5ed13b415d33c77fd42dcd611c101939112501a21966e85","ruler_category":null,"seed":null,"selection_rank":73,"selection_sha256":"2f2ec7824d30ee016341d9347bf3033c9e8baefcf7f210237aece22c50a0c793","sequence_length":107,"sequence_token_ids_sha256":"02a78c628534fda7b94bf5fce8dc4c42f06805ffe169c5e88e972974217f23cd","source_content_sha256":"535cf2dcfdda3927e31a863cc6592099702642031c1efc3a80742b6f3ea4493c","target_token_ids_sha256":"8ad5d65ce29243858983f2e234e934222f9ebfb842ba646ed2857ea2ba6d2d5a","token_span":{"cache_exposed_start":107,"cache_exposed_stop":107,"prefill_start":0,"prefill_stop":81,"scored_start":81,"scored_stop":107},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9356ecdbe5e79b88db404f8fcc1cc4066488ab817a3a8d2c38916554dca11b22","anchor_positions":[13,27,41,55,69,83,97,111,125,139,153,167,181,195,209,224],"anchor_positions_sha256":"1b59984280d92d0f5b008fda81e8c1ee99edd8adebbc3555a45f9e3b6ccc7b43","canonical_id":"648","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[12,26,40,54,68,82,96,110,124,138,152,166,180,194,208,222],"fisher_boundary_sha256":"6a4524107689341a41d25534167f49a57a6232f42670a7ccb3c3c53953ca52d7","horizon":1,"input_positions":[13,27,41,55,69,83,97,111,125,139,153,167,181,195,209,223],"input_token_ids_sha256":"5d718cfe7aa9550f8596ed8028ffa484e5dde6a58cf75d5e564d46b03ceb80e2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[14,28,42,56,70,84,98,112,126,140,154,168,182,196,210,224],"target_token_ids_sha256":"29ce1ee9257823ee60fc561f31cf4387cbb83ef9304ba18e1878faf829fa4c12"},"formatted_content_sha256":"03ce2215aec8d5a70b9b7d5d9457a3049c63a14d3fb41d268cb7d640203b379e","generator_receipt_sha256":null,"identity_record_sha256":"47caea8e58a24ad7e9189aecc0d5cd34719c3296041ecb933c3966a471cb646f","prompt_token_ids_sha256":"c856d4839a1ad8ebc26aefea0a9fb07e5436dd63c6a305c7b4112bee6193dde4","ruler_category":null,"seed":null,"selection_rank":74,"selection_sha256":"307a74054631fd4611a4637c7dcc3b5902cc9b2530ad55e1817a7b66e0244a87","sequence_length":225,"sequence_token_ids_sha256":"996568b0b6b68ffab6ac7da97e43dcd7e5fdaabcdae5248fcf638e8c03d9454e","source_content_sha256":"c078071a1981a4c9334368583a6c9b84c5bca8b86cdedb85a12e8be1b1203123","target_token_ids_sha256":"c8d5a3a6d636c7fc1ee25da058f4b6f9c67be6202458fb01fa42861b8939a7a5","token_span":{"cache_exposed_start":225,"cache_exposed_stop":225,"prefill_start":0,"prefill_stop":172,"scored_start":172,"scored_stop":225},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"806ebe5a2818da11223ca67ee7cf2265f233845126be6d099c9dcb15c55fa08c","anchor_positions":[9,19,30,40,50,61,71,82,92,102,113,123,133,144,154,165],"anchor_positions_sha256":"f50a8793906ae0ec9418f440c4efb82f7805f24bf465b189ea08ad0bcb356f8f","canonical_id":"772","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,40,50,60,70,81,91,101,111,122,132,142,152,163],"fisher_boundary_sha256":"1b7250139134ecd49bde104b1a6a2510822a0f9dfb9d2d4e9e5eecf763a8b9a8","horizon":1,"input_positions":[10,20,30,41,51,61,71,82,92,102,112,123,133,143,153,164],"input_token_ids_sha256":"3e9f5ed8311d8d669c372b59caccce107e54dd1559931f673bbb25676156cadf","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,42,52,62,72,83,93,103,113,124,134,144,154,165],"target_token_ids_sha256":"43edf81c690bf9f561ba94748611186c5eed3e9186c085b2808196492de0f6e4"},"formatted_content_sha256":"b065a8530085bc045a0533e1eeaf8aaf8188f595824a24047f8870dde69f04dc","generator_receipt_sha256":null,"identity_record_sha256":"9f12e75bacb719e1dd7725bb446586e29d53441710a6bd0bee305318116aa5ba","prompt_token_ids_sha256":"deba03b89b8f2bb57ae57611e69505484898b45ff63c512c33597cb6fcdbb1b9","ruler_category":null,"seed":null,"selection_rank":75,"selection_sha256":"3116bf2b1674d3472c0f8a996afd05f7fc073f3720ffdb297b5d89cbc3eccae5","sequence_length":166,"sequence_token_ids_sha256":"beeb3758c81c70dda2bba27ac2041d61adaa245c5d8d0edf701cdf0f91f45c4b","source_content_sha256":"1ae2ef8f6c027aabd4091686f02cddd085364dc8ca4f35da3877f375ddbecf79","target_token_ids_sha256":"920ea1a628b222d262ce367d32d278f0b295a5028fbca3295770bcff04e208d9","token_span":{"cache_exposed_start":166,"cache_exposed_stop":166,"prefill_start":0,"prefill_stop":117,"scored_start":117,"scored_stop":166},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9b187e5fe961b945d44953c99d7e41a4cbc730ca0aee47b0408c755b6bd9ad40","anchor_positions":[6,14,22,30,38,45,53,61,69,77,84,92,100,108,116,124],"anchor_positions_sha256":"fe7a10db15891cb54d23372b10af49697c0401f54d8f6b37c01417545e9ba930","canonical_id":"764","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,29,37,45,52,60,68,75,83,91,98,106,114,122],"fisher_boundary_sha256":"e6be121ec0065a1a1aaee9d9e86ba55d2577c56a8617ecadebe8c4e3e6fa2ebd","horizon":1,"input_positions":[7,15,23,30,38,46,53,61,69,76,84,92,99,107,115,123],"input_token_ids_sha256":"8b87308fede122de1697b74353b230a58de366e630c17539cb050893d296a569","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,31,39,47,54,62,70,77,85,93,100,108,116,124],"target_token_ids_sha256":"a3e5f3dfab994ff9d370511f35cb4ce1a6409b31076b0c2fbfc672290cd7dbf4"},"formatted_content_sha256":"dd3955ad349be1f3bbe53adc3231a0d62ae00e248214be3a7b5e980d326fc9fd","generator_receipt_sha256":null,"identity_record_sha256":"700a6d8a14194a1306ce6fb464aea0085a25f117d07fcf265169ed7ae1eb2406","prompt_token_ids_sha256":"66cf8821e075a67969485c8f8e2ae3f57b8384381fd08c0db59c080fc7ed3267","ruler_category":null,"seed":null,"selection_rank":76,"selection_sha256":"315964c2edf3979db8272d920dfef4d6429237f6cc495a6ee3cd3d5549a6b648","sequence_length":125,"sequence_token_ids_sha256":"516180a745f3962c6178902c00099642c3d74217bedb6c055bb189918a4cff6a","source_content_sha256":"690c9b90840fcf0cd5119cfd171717a21c533ed2145ba5803ba93e78d2229bc2","target_token_ids_sha256":"17e59e0812b0551e0a212266dfde313e3b8b138e97f059cabb2ac99726994b05","token_span":{"cache_exposed_start":125,"cache_exposed_stop":125,"prefill_start":0,"prefill_stop":75,"scored_start":75,"scored_stop":125},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e95a3332e3f7eb017e36ba83694e18fd59804b725bb678bfc239e300e0f3db3f","anchor_positions":[9,19,29,39,49,60,70,80,90,100,111,121,131,141,151,162],"anchor_positions_sha256":"937bf130ee4ebb63677095c86a2168cbc15816e3cb8813d07c1ce8340b5fe4d7","canonical_id":"710","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,160],"fisher_boundary_sha256":"6be547ce84d00ff5e76edeb90cf3d18f3f42ed8471ed55e075a934fcf78abcd8","horizon":1,"input_positions":[10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,161],"input_token_ids_sha256":"110af952e2d5d1fb75f8a65fa643ba6711e84effc65a18d37b8308133a6bfb65","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,41,51,61,71,81,91,101,111,121,131,141,151,162],"target_token_ids_sha256":"b28186d7106e163a34e3f412a706d4588d43b0357f9c57bee118d5b51160b7ee"},"formatted_content_sha256":"88240f422a8a69eea08b888c6b9b3c0bb24a6241a7a906d6c5bd4e4f75a9e655","generator_receipt_sha256":null,"identity_record_sha256":"1781d9f90e6dd08f1aba55a3d324d8fa14693f707ca4f071b3b392360bbe0408","prompt_token_ids_sha256":"be734d8f240da897ca816e1ebc35469363413920aebb0b334817102e135917d7","ruler_category":null,"seed":null,"selection_rank":77,"selection_sha256":"3214d5450dc1d7505961e5b95aa8e71c3b102461efcea184dc5d581da13805e4","sequence_length":163,"sequence_token_ids_sha256":"8b2dd644e6f00860a2723ce8ab4d5c0a9ab6aa316b27d063fc4a39356a29e075","source_content_sha256":"15fd5fe74d1087e4941f9d296c749df402bbce707b6f506e7dfb634abdac93fb","target_token_ids_sha256":"5b619991d998bbfbccb9502f7a36880ca09dab9061d34c4dd351784267f88e5c","token_span":{"cache_exposed_start":163,"cache_exposed_stop":163,"prefill_start":0,"prefill_stop":130,"scored_start":130,"scored_stop":163},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"8623504af00a9b25be7bc8e91298e9bf3d79e889943b43f81d79a76ff7b2b08e","anchor_positions":[20,41,62,83,104,125,146,168,189,210,231,252,273,294,315,337],"anchor_positions_sha256":"d86064c4e7cc49d03ee7921a0260f3727647f42da5038fce7e70f58100a9b429","canonical_id":"615","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[20,41,62,83,104,125,146,167,188,209,230,251,272,293,314,335],"fisher_boundary_sha256":"f7c95c163c354425c2add0a88c1c4185cf084e48e914cfdd216090db1b963bf8","horizon":1,"input_positions":[21,42,63,84,105,126,147,168,189,210,231,252,273,294,315,336],"input_token_ids_sha256":"9379deab7e7b61df7302e3f0d09b7e567d9ba26caa1886692eba4506c94310dd","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[22,43,64,85,106,127,148,169,190,211,232,253,274,295,316,337],"target_token_ids_sha256":"80d5c49f9ff9f2ebebef47d67d9b86438b015a070f8c6e2dc84883e9240b9ab1"},"formatted_content_sha256":"b9645bc5ec977dfe41735d4ce5d1f4a734b1f86ea78c4a5646565ebd8c15566f","generator_receipt_sha256":null,"identity_record_sha256":"9b128354be7b44adac95032db55fe4b66adf4ee89cf51ce82d87f3159b662f3a","prompt_token_ids_sha256":"51676de173a112846b34d4fba7305f57b81fb3f9d18e638cc2fa74f8e3a3e293","ruler_category":null,"seed":null,"selection_rank":78,"selection_sha256":"3285b6f67988631b43e377c409844a0c984cfd47e40b19a98584df2d8f61b362","sequence_length":338,"sequence_token_ids_sha256":"6bebf94cb834d1fcfe47dc541cf15d6fc87989ddf47fecdd7be2a90d0663d24c","source_content_sha256":"fdb94f9efd3d2f52a851821fe1708964cb54b4e59675eb6fbb5b37edf3910703","target_token_ids_sha256":"9714d922e9b518657f61cc75eda60ded6a3f1c50ebe5b59d1d6800481296fdd7","token_span":{"cache_exposed_start":338,"cache_exposed_stop":338,"prefill_start":0,"prefill_stop":310,"scored_start":310,"scored_stop":338},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"96cee3de971a921588b61de2e2cee579332da466f101701e569463c8c6a8fe9f","anchor_positions":[9,20,30,41,51,62,72,83,93,104,114,125,135,146,156,167],"anchor_positions_sha256":"eaf3a0e814f88177633695c30e9a05b6b492aa64c90bf98da98cb9be3917c812","canonical_id":"929","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,30,40,50,61,71,82,92,102,113,123,133,144,154,165],"fisher_boundary_sha256":"a4409da62e8624d4a36ab120ae46d80d5347e0949ddb59fa84433d595ab39c86","horizon":1,"input_positions":[10,20,31,41,51,62,72,83,93,103,114,124,134,145,155,166],"input_token_ids_sha256":"ef3502c0e2121d90870bb103716d7830c424bfec40f477dca54ebfeac256169a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,32,42,52,63,73,84,94,104,115,125,135,146,156,167],"target_token_ids_sha256":"0afc3393efde3a0d92114c3ef929fe4bb20b15a93254a54991f7854ac5d8fb1c"},"formatted_content_sha256":"8720c8227a224ac2e19aea8120b6280befcba03b75052032de9bf8e95ca77b04","generator_receipt_sha256":null,"identity_record_sha256":"1386ffcc0e108e711ebccb2df6fc089ab088b8bb5bf75534b414141f41a74a75","prompt_token_ids_sha256":"d3e5efc050ed2ac9f9ae235e854ba1d481168daa7af573882848337902db8cbf","ruler_category":null,"seed":null,"selection_rank":79,"selection_sha256":"3296d51e868118ed66c05092fa54df28b0ca574c48a0cba377e10c955ebf1545","sequence_length":168,"sequence_token_ids_sha256":"e74df24c78d5f48aa647690ed6d124d47ca43f1d31e9c157bbbd81fae882c552","source_content_sha256":"bc0b73708502c4d4ef970ea6606ef292432812c718cd0e89507be65055cb1c76","target_token_ids_sha256":"e0d8a0f88aa5d40291247005bc203563ae5853bbdc6122d0bc21fb681b59aab3","token_span":{"cache_exposed_start":168,"cache_exposed_stop":168,"prefill_start":0,"prefill_stop":145,"scored_start":145,"scored_stop":168},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9209ac953f696c6753bd12b7e3aee86341dec2daa6e7890b20b1f6ca9c923fa2","anchor_positions":[11,24,37,50,63,75,88,101,114,127,139,152,165,178,191,204],"anchor_positions_sha256":"48082390de232d2e911509462169a4b3ebe8a03329ea579a66fe38623c52b3bd","canonical_id":"872","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,49,62,75,87,100,113,125,138,151,163,176,189,202],"fisher_boundary_sha256":"099898e807c53f06b4fefe6112588ccaf699892b69781fb1af96a8f953dceff3","horizon":1,"input_positions":[12,25,38,50,63,76,88,101,114,126,139,152,164,177,190,203],"input_token_ids_sha256":"e90e1cdb40861c5ce546f0f793eca7fb22ff9a2d1fcd28ac90c1c3c41b2a9a8a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,51,64,77,89,102,115,127,140,153,165,178,191,204],"target_token_ids_sha256":"dfe340d203294bbebece21c27b39e76b677b6c6f99f69c75f1c05d25a052a6b0"},"formatted_content_sha256":"ae7cccb0f44aa7dd8ae1d59a7bb423feb46060df6bccc538c0d729941cf62f09","generator_receipt_sha256":null,"identity_record_sha256":"33a68c486b6fb35b93a963a3c1ab2bc5ff188b6005bdce71a974210943d77edf","prompt_token_ids_sha256":"c587ed25a2f2bfc478fbc770f969490260f64e578d5a442189b63d9a9743cf1f","ruler_category":null,"seed":null,"selection_rank":80,"selection_sha256":"32eea1bbd2a2735f50e8ed7192ad280642b4f864877e2e12c74229735f3eb364","sequence_length":205,"sequence_token_ids_sha256":"5cd1de46f6a6a54a60046c16ee57b387b99886733c7d130b2e28f1ae13d4ae9a","source_content_sha256":"7a15fb9218af2f77deb400a4399d8ac77545eab7979f345c0ea992388693aa5f","target_token_ids_sha256":"27908577789a6b8b4d0224aecb42ddeb2609c5ec146dca642424eb2a076e6e7f","token_span":{"cache_exposed_start":205,"cache_exposed_stop":205,"prefill_start":0,"prefill_stop":183,"scored_start":183,"scored_stop":205},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9e18bfe18f9c4da802617b4dee6331466e0e733317487a86f7aef33753525ee9","anchor_positions":[7,15,24,32,40,49,57,66,74,82,91,99,107,116,124,133],"anchor_positions_sha256":"6c1029515cd9fd840ee9baeeb3bbf16de22d914596dc66d6c0b00a017e82aef5","canonical_id":"804","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,32,40,48,56,65,73,81,89,98,106,114,122,131],"fisher_boundary_sha256":"0654213b540ddd6054c3a7a54b4ed68fe2ce52d54601d589e8dd8432c5e03c9b","horizon":1,"input_positions":[8,16,24,33,41,49,57,66,74,82,90,99,107,115,123,132],"input_token_ids_sha256":"e6dbcef9e02b4ece7b578fe3e362efe755dbd02afcae77d6d6f4edcd4de4f4cc","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,34,42,50,58,67,75,83,91,100,108,116,124,133],"target_token_ids_sha256":"0b20024476a22a5a23bff385afa2fb325aef7aecc21280ce17cd65007c86f1e0"},"formatted_content_sha256":"58c227c113ff5dfa4a446ad44a803eb54659e0fbefb3f5f80dbef45d0994bdfd","generator_receipt_sha256":null,"identity_record_sha256":"01f7ec235052c97847150a68e3de4b9c2d022df8fafdb46a04092b4a7ee18c27","prompt_token_ids_sha256":"927da8d69d8b88559b5811091f4fd329f0e47607f28c399059242149e3c7336a","ruler_category":null,"seed":null,"selection_rank":81,"selection_sha256":"3379395523a918b0a3ecf21510bcc8704655d52083f91c7d3bd323ce8ea3e272","sequence_length":134,"sequence_token_ids_sha256":"bc6201b3951727d59c9b6c88db4d0722a0b56896e2812e646426b637a86b085c","source_content_sha256":"0b30c40b1ec355476196d7e4b579e4c837c30947a206b5da87928bc338b51aac","target_token_ids_sha256":"ce0bec16e6ba57588fa2e620c280ea4fdf1f2da0b33b6d8c1bf55efd9f79a60e","token_span":{"cache_exposed_start":134,"cache_exposed_stop":134,"prefill_start":0,"prefill_stop":93,"scored_start":93,"scored_stop":134},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3078f2a2b5f48560ec6b5e149abdc2ec94224715ce8b8b558250fe31cd4dbf52","anchor_positions":[6,14,21,29,36,44,51,59,67,74,82,89,97,104,112,120],"anchor_positions_sha256":"7f8463e6889d5cbbd872abcc6f675ca04c3c0d8d69d266000933478b69489b8c","canonical_id":"713","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,13,21,28,36,43,51,58,65,73,80,88,95,103,110,118],"fisher_boundary_sha256":"12bd3ed950f8f6e2d1ea75538e2472a2bb7fcef26eeb014dce2f68e39c0ef080","horizon":1,"input_positions":[7,14,22,29,37,44,52,59,66,74,81,89,96,104,111,119],"input_token_ids_sha256":"8bf93fb6683667ca762c7e68944d126e5f2c2b6c44cd212a8000cdc09acb27e1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,15,23,30,38,45,53,60,67,75,82,90,97,105,112,120],"target_token_ids_sha256":"bb0137c2b21eefda36fe32ac45aa551c534cfa2ebce4a392aed849f5496ed9bc"},"formatted_content_sha256":"681d439ccf9ce8b1da99801aad0074b3790087d6efe044ae79d7e49571d76ac7","generator_receipt_sha256":null,"identity_record_sha256":"bf5c49f9a99a125e0728cdbdb1103ef16d8f41d27323feb6d66db4c28cbdf8e3","prompt_token_ids_sha256":"689f474ef4fa8ccced0bd23ab2314dc97972bacba54c2a54dc4d440098402b38","ruler_category":null,"seed":null,"selection_rank":82,"selection_sha256":"36a09fe9c145944020ea5abafade01ab7f4a1c538f55c32b2daf644698464fd3","sequence_length":121,"sequence_token_ids_sha256":"df03ea9ab5af6fe2ad270a5fbae4db0bf61e07b1919c60f124a8070bb517a052","source_content_sha256":"9197df3cdc3a76aced1eefeae0cf812e2f43b1c25afc3bd73c6e27fa82f1aeee","target_token_ids_sha256":"6b06cbab3e1a4e869733d23ddc800cba139bbf3020eda6b3ab38843880a60fa3","token_span":{"cache_exposed_start":121,"cache_exposed_stop":121,"prefill_start":0,"prefill_stop":90,"scored_start":90,"scored_stop":121},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"63b9e82e307a72c058464d1b1d8a7f3c58d7fb5152977d0a46c053ff527a82dc","anchor_positions":[23,47,72,96,120,145,169,194,218,242,267,291,315,340,364,389],"anchor_positions_sha256":"8e2b268c6192e66f463da8f9c055e0c9184253267e4f791d54da3db3eed553a6","canonical_id":"709","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[23,47,71,96,120,144,168,193,217,241,265,290,314,338,362,387],"fisher_boundary_sha256":"967b90e282da8354335e13a711b1db1fc3dc1cd2d5f3733153901609f0b7284b","horizon":1,"input_positions":[24,48,72,97,121,145,169,194,218,242,266,291,315,339,363,388],"input_token_ids_sha256":"14ec48de9ee49ea137494da470356e32fd2626ad45fbf1122fb508edd8791526","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[25,49,73,98,122,146,170,195,219,243,267,292,316,340,364,389],"target_token_ids_sha256":"eee798bbc46c0f86ea4c32017a079b8d80703d9f4c5b22ed2e2e6a9edc5c6096"},"formatted_content_sha256":"377241d6ba849220fead3711b06a3790ec65647bdb509adac60123fd052d34e6","generator_receipt_sha256":null,"identity_record_sha256":"1d437dc4e8346fe37a0eee9d9a423bd60824e5834a88120acefb46d1be5ecaf8","prompt_token_ids_sha256":"fd90d23dea7f7557e683b5a171cd89091ccc9e500b8c27afb3c9221f5d9788c4","ruler_category":null,"seed":null,"selection_rank":83,"selection_sha256":"372cfe33288b920e01911b67999a2534620fe6fd180024291e1b86f7f221909d","sequence_length":390,"sequence_token_ids_sha256":"c1e45e96c9091049c070bbc66c111935f57c6b5469213a4d3ca91d3d27085868","source_content_sha256":"1bd1bf4c9a1573325504ef4695c6a08c9641c60e85f7cf68942681af59f0c13d","target_token_ids_sha256":"2c233f19acde716535f8aed74a2b4c7a6ff031e2df4d090d3f77cfd880649eac","token_span":{"cache_exposed_start":390,"cache_exposed_stop":390,"prefill_start":0,"prefill_stop":309,"scored_start":309,"scored_stop":390},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"755396a4616427092f5857ae9f027899125cc2f2dc290df40ac96b2609861e3d","anchor_positions":[13,28,43,57,72,87,101,116,131,145,160,175,189,204,219,234],"anchor_positions_sha256":"019a1d4f16a491584b0b921a03e82e05ab8f21be64e2ef381494510440a2dbba","canonical_id":"854","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[13,28,42,57,71,86,100,115,130,144,159,173,188,202,217,232],"fisher_boundary_sha256":"3978f984679fc70aed874ca972bca2f58c59e79999bd8b5dbaaf51586a103e4f","horizon":1,"input_positions":[14,29,43,58,72,87,101,116,131,145,160,174,189,203,218,233],"input_token_ids_sha256":"f50bcffcfe9946342cfe243e55c6a21e7d36fb6cc50a570f25fa3180e679507d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[15,30,44,59,73,88,102,117,132,146,161,175,190,204,219,234],"target_token_ids_sha256":"d2eb23c855ed7281d5d4f214ea2ec6aa0309bff553e222cc09435b17b3a89dc7"},"formatted_content_sha256":"ad6817ab94e23314f897ae56c55d36b1bfcbf69a7b8d35fb818c81c03d39b3c0","generator_receipt_sha256":null,"identity_record_sha256":"341e83cf4392f6688e220933fa083c53bd3f177ce67aba89005be759ab1ef581","prompt_token_ids_sha256":"c469e78789638c0e407409fe9aebd074eba88c7033481e56ef2dec96fd271eaa","ruler_category":null,"seed":null,"selection_rank":84,"selection_sha256":"374c60a9450ed67d1bb35eb2406c9e00aaf02021508333ed279cecdacc65f059","sequence_length":235,"sequence_token_ids_sha256":"3434905939b518880cc623791311c79432ac4833d7f9739c8a2fd83b9590d85f","source_content_sha256":"2f5b2b2d153d57ad5deccfd519cfcd97185b0cedd8a8e32fbcc6bb3429ff8755","target_token_ids_sha256":"8109cfc76d8f3759529fc35b4e397ce62b4a87dd10005164d11c77adb9a1a048","token_span":{"cache_exposed_start":235,"cache_exposed_stop":235,"prefill_start":0,"prefill_stop":209,"scored_start":209,"scored_stop":235},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3125b8323a121faaca1072275076c923a67557e8121f55b5373de0a2d7742432","anchor_positions":[21,43,66,88,111,133,156,178,200,223,245,268,290,313,335,358],"anchor_positions_sha256":"8cbf29aa79890d46e276ccafa41f46a9c68b76575359288a1ba31b9a687111bc","canonical_id":"698","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[21,43,65,88,110,132,155,177,199,222,244,266,289,311,333,356],"fisher_boundary_sha256":"c83dfd1738f12889d5c8d8fa14927532b8398e8d2ad099c912c8a526d1c14c23","horizon":1,"input_positions":[22,44,66,89,111,133,156,178,200,223,245,267,290,312,334,357],"input_token_ids_sha256":"b4fa8aaa429dfdeb479beb6db13cf536dad62e221419e87d8cd4cc6e49562547","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[23,45,67,90,112,134,157,179,201,224,246,268,291,313,335,358],"target_token_ids_sha256":"0e14956d01089b21589756a95abe69218dcb1a764bacd1f5efde70f9667949f1"},"formatted_content_sha256":"829bb0068e2180b28903791519d5fd2ef9c4873981556f5c4fb97915943ceec4","generator_receipt_sha256":null,"identity_record_sha256":"92efe3c0141538bc1fe1ae06136691ba5bfdf72bae549e16e258bdad5bc9a4ca","prompt_token_ids_sha256":"e985782bb265579625ef2c2a88762e93185a635ff5c9104060000f4dffeff366","ruler_category":null,"seed":null,"selection_rank":85,"selection_sha256":"383631dc03a19eb73d617831ef3de08685e43b584ca707ed72bff063479a80f3","sequence_length":359,"sequence_token_ids_sha256":"4f8db022db57407107163fbccfb9fa14d20b6c9ebb4167358a92b76de52a8cf6","source_content_sha256":"040369c5072294811f7d2b172d9db244cffb410fb58a0ff0ecfd4ac76e9ecb95","target_token_ids_sha256":"df7cdff2b03251bf8483757a1fe829176f9debe6a93f970241f7d8a5e45d2923","token_span":{"cache_exposed_start":359,"cache_exposed_stop":359,"prefill_start":0,"prefill_stop":310,"scored_start":310,"scored_stop":359},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"27e366c7bd906286182fb5657b877a3d983450e230b7e29de32c33658d233999","anchor_positions":[7,16,25,34,43,52,61,70,79,88,97,106,115,124,133,142],"anchor_positions_sha256":"231f3d432eea6372c0ecf521691b115566b5045b38a4ebc776666c6712167443","canonical_id":"847","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,25,34,43,51,60,69,78,87,95,104,113,122,131,140],"fisher_boundary_sha256":"8329a9007c7d4b358ba485f2d16e86dbb70099e16057eba7d09ee5bccd1208a2","horizon":1,"input_positions":[8,17,26,35,44,52,61,70,79,88,96,105,114,123,132,141],"input_token_ids_sha256":"c5de0c4ff62e358587b0368de1f213abd9ae23772b8a469ddfe3d5703c9d1012","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,27,36,45,53,62,71,80,89,97,106,115,124,133,142],"target_token_ids_sha256":"98c5bf9c6ce5ebcc0886bb8c1a84fb66bffb85cf9331303d2fd8e801f62f2549"},"formatted_content_sha256":"072b0e2d3b510043865ddae77cb50d35becf17f9e8114b0d23ee65976876342f","generator_receipt_sha256":null,"identity_record_sha256":"45013aa8aca9961b02b0a4a7cb8344d4f74b6e33cd7e17bde68f4bcc9830f5e6","prompt_token_ids_sha256":"cbc2d13d695e49c699eabbb0c37174336cdd76b1c47e1f754455737973e861a6","ruler_category":null,"seed":null,"selection_rank":86,"selection_sha256":"3856a5bb20d2c0009500867054a1e894dca05be3fb5db8c3123f8dcc626b3326","sequence_length":143,"sequence_token_ids_sha256":"d3b45de44d96d01b589d50a86f386609c7543385f067532a7174592d3748d795","source_content_sha256":"06648ba18a8fa484b8f21f1e5f0581005f1c25c15cbc72488833b9a12475640e","target_token_ids_sha256":"0fec8926851dfdf2d26e890a92aff900619119eb83bb757d608676dac4ded5d9","token_span":{"cache_exposed_start":143,"cache_exposed_stop":143,"prefill_start":0,"prefill_stop":132,"scored_start":132,"scored_stop":143},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"a71bad1c8e08458e094f4e03eef72e9b22e929c4ebde165642691319a2f08b8b","anchor_positions":[11,23,36,48,60,73,85,98,110,122,135,147,159,172,184,197],"anchor_positions_sha256":"7e13e3a1913a1aa40d4138d69b9cd0e10357d1735e6198f7373eda076b3142df","canonical_id":"627","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,23,35,48,60,72,84,97,109,121,133,146,158,170,182,195],"fisher_boundary_sha256":"fa4d10a3491f2cbd7a778d5d9cca07773dcb82334929e2637a6782e847dbba69","horizon":1,"input_positions":[12,24,36,49,61,73,85,98,110,122,134,147,159,171,183,196],"input_token_ids_sha256":"44dc69fff0abd93231a72100f67a65e70f884191a50e851eec9626b5f6d3ecea","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,25,37,50,62,74,86,99,111,123,135,148,160,172,184,197],"target_token_ids_sha256":"737d45bbae07d3bc02b3df44539a526e40a1a65dfb9bbd68a8552b4e3b8a3396"},"formatted_content_sha256":"0a0b695997f5097537b328213722620e421fc13a6da1c75ddb049d61551d4ed8","generator_receipt_sha256":null,"identity_record_sha256":"a708abf4f23637f8abca094ec7198056051b374088f997be873b42a6e20807ab","prompt_token_ids_sha256":"f19ff189894482dc79997369f05a455e1a3ef3d68bf8187d519557d6d6c6ab5c","ruler_category":null,"seed":null,"selection_rank":87,"selection_sha256":"3878b1b7a110136224b32091f63168693baa7e86e3c51b5a5536e13dad73f9d5","sequence_length":198,"sequence_token_ids_sha256":"0f9fcb1d27f0fb0d37209deaa4fa835902344d189e823d011638a2011ede20bd","source_content_sha256":"12ac418a9b4a0d9ae374ce607cfe72460e1c30e426106ea273a46ac4ba95ccac","target_token_ids_sha256":"cf5bb6259eaf6bc72ef3d80bf3723b99a1c4a093c8a643a5b9a663a259d73de4","token_span":{"cache_exposed_start":198,"cache_exposed_stop":198,"prefill_start":0,"prefill_stop":111,"scored_start":111,"scored_stop":198},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9c95bca00064ff088717e40df85e1ee14f4f14ac42b57c865b2a27f9d95db37b","anchor_positions":[17,36,54,73,91,110,128,147,166,184,203,221,240,258,277,296],"anchor_positions_sha256":"92fd03c25ec622578164ea2f5bba6632c7a3ee726d6e1c9caa74a3fe54646d5d","canonical_id":"936","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,35,54,72,91,109,128,146,164,183,201,220,238,257,275,294],"fisher_boundary_sha256":"e2680422efa4a11ce200109baaec769ad06a6773088de15eee3df9953e7958f3","horizon":1,"input_positions":[18,36,55,73,92,110,129,147,165,184,202,221,239,258,276,295],"input_token_ids_sha256":"06925d4de4cf692d9d7ae2fb8defd1d78ee16de18017fea89fc095caf7b42551","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,37,56,74,93,111,130,148,166,185,203,222,240,259,277,296],"target_token_ids_sha256":"a529ab740b332e7c438cb708428ab5907c7a59afc2ac4ac51145f549dfdc5580"},"formatted_content_sha256":"afa72e90063431bd6756ace794a9050cc1610077201aea2fd9b4bda86c662ffb","generator_receipt_sha256":null,"identity_record_sha256":"92f30344725d406c50725f35a3f5b576e19bbd592073bffc0ff773be3cd5ef1c","prompt_token_ids_sha256":"6a725c770525d60d682f99ffbb259336c6f4946dc97f839cd4ca68ee7d0cf425","ruler_category":null,"seed":null,"selection_rank":88,"selection_sha256":"38fb5a0b645f175f62d3e012cf15aec73e7982d5713c59591a4c1d1a08561d18","sequence_length":297,"sequence_token_ids_sha256":"9f51cb1048ef1942bb0af6b3e3d4087006b31ee6c48ec3a0f524b1e08cc51f15","source_content_sha256":"1effae4b4c21d10e095a0a5f927b7cffd241b64be8df419dc05137e160edabc0","target_token_ids_sha256":"6a6840a3339ec51d008e341f8b430ed9e10a11c573b431942795dcfdb1e62c77","token_span":{"cache_exposed_start":297,"cache_exposed_stop":297,"prefill_start":0,"prefill_stop":255,"scored_start":255,"scored_stop":297},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3b13375846101ec060f2a2ecefd25b66c836c8f9c53225732689232378b5cc44","anchor_positions":[9,20,31,42,53,64,75,86,96,107,118,129,140,151,162,173],"anchor_positions_sha256":"5eb476494a95db7e5ff69d760daae6c1b59a438e7d071342cc2f8648e535c378","canonical_id":"827","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,31,42,52,63,74,85,95,106,117,128,138,149,160,171],"fisher_boundary_sha256":"d2950d5bb22979ced35eeb6c35b7e0f442e13245822cf644ec5b7300565507ee","horizon":1,"input_positions":[10,21,32,43,53,64,75,86,96,107,118,129,139,150,161,172],"input_token_ids_sha256":"995c75651110d6eeaf5f7790a1e5988aa866d3ec3f148b43040167cca7bd163f","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,33,44,54,65,76,87,97,108,119,130,140,151,162,173],"target_token_ids_sha256":"4d18d212a638d29f456dfa535bd673c510d66ec32e43594822d9bb9241ca20e6"},"formatted_content_sha256":"4790f2d60d6a241d89f28294cf7110652356521e1216cc3a9e2ce7e0b065bbe2","generator_receipt_sha256":null,"identity_record_sha256":"e93e8a672f73ac122851223c0114eb0974c76a960f1a880f4d4f64c9eb90f40c","prompt_token_ids_sha256":"9cc3fc3392444e349135b1065a5227e3391859f3554be674981d424cbdfa9fbe","ruler_category":null,"seed":null,"selection_rank":89,"selection_sha256":"39165caf08bb54ef81c9f6ea0a6ae1eac6a1c3f0b6f9748dcd2e2d953b5a44c6","sequence_length":174,"sequence_token_ids_sha256":"f5f5521cd3bd481ca316528bee26f7f9355182d73550ff14da89d91f6558600e","source_content_sha256":"c60d7f94f875619bd70591ab5f80b6e8fb8d336938e07fbeed1b32564c1a0b38","target_token_ids_sha256":"fac10609f7b6c41ca9b4bbbf7495988b94cc3b4df19e5dac66d84079bd3d4b1c","token_span":{"cache_exposed_start":174,"cache_exposed_stop":174,"prefill_start":0,"prefill_stop":148,"scored_start":148,"scored_stop":174},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"ad1b36a9a3ef4aa1d78cc9a2babd28dfdd79a75b4a7e69563454a05c10b6e2bf","anchor_positions":[7,16,24,33,42,50,59,68,76,85,93,102,111,119,128,137],"anchor_positions_sha256":"451653062c723e6fe2316470c66684b5f06f6d96d14e7b3bb2fb0c9e7d88845d","canonical_id":"930","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,24,33,41,50,58,67,75,84,92,101,109,118,126,135],"fisher_boundary_sha256":"a46d5960cd9fe3147cb09c59211821b555240915a05f4d76e9a91b70e253bf77","horizon":1,"input_positions":[8,17,25,34,42,51,59,68,76,85,93,102,110,119,127,136],"input_token_ids_sha256":"e59435b4be9b10629e72f1905eae42cfc9f3fd8b77cbff58f09164e96872cde6","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,26,35,43,52,60,69,77,86,94,103,111,120,128,137],"target_token_ids_sha256":"e841c04edad63a6a7f6c695115859daa7a4c3eeb9fe4f2f7094561aff448ee28"},"formatted_content_sha256":"dcb79d95145dcc945bd4f4ac20fe07c33528e7e8d8ef4cfea0312e5b02e817b5","generator_receipt_sha256":null,"identity_record_sha256":"5b49551d1068cd3633b13af41d2b4e4ab38c2383efee9d8c5370832e1a78e65e","prompt_token_ids_sha256":"2eeaa907d87be72e61b726a9ed4672463343033ccb6bf2376754766f298a566d","ruler_category":null,"seed":null,"selection_rank":90,"selection_sha256":"3934218a3cdeb9fe23fe0820bbaf2003e6426a07af797c8318d3df525a7b9d57","sequence_length":138,"sequence_token_ids_sha256":"5c11b18fe521da6db909428adfb48961064e310ff957a9d2a1ade8b72d391175","source_content_sha256":"435474a45f03a3373a0a7f5f78526f3621d01a4daf6907075965fc04440e323f","target_token_ids_sha256":"3fa7542edfba5ed95e02424207a7fc3d1bdc29ef1bb3fb055507740f8a0860d6","token_span":{"cache_exposed_start":138,"cache_exposed_stop":138,"prefill_start":0,"prefill_stop":90,"scored_start":90,"scored_stop":138},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"926ef081a870c670e181e6f21a6dbb6e67418bbe21df0f8841a9da25ad315d93","anchor_positions":[14,29,44,59,74,90,105,120,135,150,166,181,196,211,226,242],"anchor_positions_sha256":"3859b527bfcde30950aa1fd4919bae6af3d771f9dcd051365cd8ca16d468c677","canonical_id":"751","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[14,29,44,59,74,89,104,119,134,149,164,179,194,209,224,240],"fisher_boundary_sha256":"af0bf62935327f2467d479d54e944e5f444b6c444118efcf42d997ed4da7cc37","horizon":1,"input_positions":[15,30,45,60,75,90,105,120,135,150,165,180,195,210,225,241],"input_token_ids_sha256":"f52f3b35b365e905e8af0a0912e656ae9d1e3ad2906698dfa15eff817cdc47e1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[16,31,46,61,76,91,106,121,136,151,166,181,196,211,226,242],"target_token_ids_sha256":"c71c870c1fd6b3ebc3243a03cd21567b7db14cc45512773ebc666af0317504a3"},"formatted_content_sha256":"b9749edd133bc0cefb8007a7358f767d918467015bff6d845fa78b29bdd5b3c2","generator_receipt_sha256":null,"identity_record_sha256":"bb8bf4056e66c85336deaf5c6bf0614c6c6eea646b46a3dcc0b05e265ed2293b","prompt_token_ids_sha256":"4d217c659514cbec1da37d7ca1cf02b1de4d40d095c106f7fd89d4517b8e1800","ruler_category":null,"seed":null,"selection_rank":91,"selection_sha256":"3972ee7b79e05ce309d4c65459acc2d264f7821184dc4baf556f892f50507741","sequence_length":243,"sequence_token_ids_sha256":"22a137779b1349921b5784c3f3a7da9f3700715398d21e5c69536216ef7c5d82","source_content_sha256":"04de6155b488684214357c82595e2a1b0b4162588221bfc1f959c3ad02c64302","target_token_ids_sha256":"4a34e467583136f16ea6deeb64a30a0f625052cbead11e0c2460872fb5a60d9d","token_span":{"cache_exposed_start":243,"cache_exposed_stop":243,"prefill_start":0,"prefill_stop":129,"scored_start":129,"scored_stop":243},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"bc742a938d14512badf1f6f2dc626345160bae405ccb62cc78f8fcf2fbef198b","anchor_positions":[11,24,36,49,61,74,86,99,111,124,136,149,161,174,186,199],"anchor_positions_sha256":"cbd5b119373faf027bdb11d72340904f9e8568f13150612cfe8c0d28d26f0a32","canonical_id":"707","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,23,36,48,60,73,85,98,110,122,135,147,159,172,184,197],"fisher_boundary_sha256":"f539aaf9624257146284435f180cb1ef2b1c3d2448b93f936510956f316e60d0","horizon":1,"input_positions":[12,24,37,49,61,74,86,99,111,123,136,148,160,173,185,198],"input_token_ids_sha256":"53699344c41ad6728511aaab294416c9cb8ab020e2cce05973c6abf72fe3830a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,25,38,50,62,75,87,100,112,124,137,149,161,174,186,199],"target_token_ids_sha256":"c13ae7cca2c350903d6e54179df99c8f0ca7fdd40021679bdcd12e59ab7d3f00"},"formatted_content_sha256":"b7943712ec68b060c30f63f9c1e79415e5d98891bad66512dd84aae0ea33779a","generator_receipt_sha256":null,"identity_record_sha256":"7e8098446b3e22dff19a34b8e6cb18c072889a0914269f5e6c46eda3e278f916","prompt_token_ids_sha256":"aedaaa76edf70a4335d76e3753a2e74b6358828bd7fd27089a9349b82dbc4f04","ruler_category":null,"seed":null,"selection_rank":92,"selection_sha256":"39d4884c3293b83c7da73a4192c68ac6ed7dd2a2b8cb6535c8bc27dabefb2152","sequence_length":200,"sequence_token_ids_sha256":"f84c8958f4ae3235716092ff6d50ea9f9eeeed151677fd0291eefd3f4c046e46","source_content_sha256":"835bcc97198da2e9313e8115ac8c70845ef11e086ca2f36a6cbb1bc74afb15e6","target_token_ids_sha256":"63a1e5e3435f4582095fe5bf45dca9f77c0635e287377e887030050e9576dfbe","token_span":{"cache_exposed_start":200,"cache_exposed_stop":200,"prefill_start":0,"prefill_stop":82,"scored_start":82,"scored_stop":200},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b368a5e3bf6a4d6a9d65ed071a6ae293e6addf944c619d41b4db74fd479ea1c6","anchor_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,208],"anchor_positions_sha256":"e3c9aaa391385675ab3475417e405da92d531052d937e046ddc609185dc2c7d4","canonical_id":"946","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,50,63,76,89,102,115,128,141,154,167,180,193,206],"fisher_boundary_sha256":"6717c7a65af61cfd12e44550f289c401161af90d68d57dd17e3c519d22a1613c","horizon":1,"input_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,207],"input_token_ids_sha256":"35c588f0b227da0babb6432ce2d27c1f6f0ddf842b8da0c7b8c5def4f5567635","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,52,65,78,91,104,117,130,143,156,169,182,195,208],"target_token_ids_sha256":"3cc14e363f767b18bf8fb37c1c7a1681237d5f34c0d8244fad4f8ea124be7de9"},"formatted_content_sha256":"6672cb487be13a90610c98af3babace38c6413c484e2a543e785501685eea0d6","generator_receipt_sha256":null,"identity_record_sha256":"f4090de3fc3464141d40f4b023f18db5c2e540193c894d2b56d314001a2cc540","prompt_token_ids_sha256":"23c086ef1490fd86a6b3f23899489b782e5aef4d38689af88dbd19f7fb7b1d0a","ruler_category":null,"seed":null,"selection_rank":93,"selection_sha256":"3a31bfe17e95e3f782d9b40a9bad5670a4dbab98abaa87ce2414402b4d4c9d3e","sequence_length":209,"sequence_token_ids_sha256":"8002749920d9aa312887fb27e1c8d185977ea0ce5f145e2a0dc684e46b759834","source_content_sha256":"bd64b5ce22599f2fceb61fda2a89a9caed49e3e0b63694d888f83b548c800738","target_token_ids_sha256":"d1d65dd1bea87ea89e2cb46bc0c015ee3a4e8df03fe4a02a29bb30c5549b96d0","token_span":{"cache_exposed_start":209,"cache_exposed_stop":209,"prefill_start":0,"prefill_stop":178,"scored_start":178,"scored_stop":209},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"122dd2605483f8c364752af43aece851c5117635a2c390bbecc2d2c38ee203cc","anchor_positions":[12,26,40,53,67,81,94,108,122,135,149,163,176,190,204,218],"anchor_positions_sha256":"34e0f2438d49fed8c871a33d3818a40fc035b239ea8639ec66d64d1e59a9bb66","canonical_id":"853","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[12,26,39,53,66,80,93,107,121,134,148,161,175,188,202,216],"fisher_boundary_sha256":"41be834ada2a5acf9617b5b43955804d2e6a4ebe3234bb01e8e5dad38bb0a1a3","horizon":1,"input_positions":[13,27,40,54,67,81,94,108,122,135,149,162,176,189,203,217],"input_token_ids_sha256":"afd2edb984710f6fc20bd321b1036090c05a37d38b4b24a9acf50e7bd4160e2a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[14,28,41,55,68,82,95,109,123,136,150,163,177,190,204,218],"target_token_ids_sha256":"b5093288c3e5810d5c3d19010a1acdd8082bcba55f9489bdabb69e3196c53db6"},"formatted_content_sha256":"abf4e533a3c9388da04e8ab3ba22c031aaaeb56e90338a4793f35ac968218c95","generator_receipt_sha256":null,"identity_record_sha256":"e80d081ddd92bc2a2b3317fc55c504dc53c4973ed5fdd77bb04345fc653a731e","prompt_token_ids_sha256":"54d2e4ab6ff2869f39d935df89b9c87e44df50e0fe4ecf67a377211040ceb3dd","ruler_category":null,"seed":null,"selection_rank":94,"selection_sha256":"3a95db7ccef488fa57c922c5fa561a269f79fc7c743e4d7a0ba136629ed14154","sequence_length":219,"sequence_token_ids_sha256":"7c11f008ccc4d7cd53ee721463f7a84f45653c8215e85fce7deab133839b9b0e","source_content_sha256":"6e23a95d9b3ebcb264ca654a288e4f63ec19e20f70c8c2cde95ec7ec6f5b65bb","target_token_ids_sha256":"5da4da15ffc85692e7f49aaf3e35fa21d41d697277e633358e0285f47b164471","token_span":{"cache_exposed_start":219,"cache_exposed_stop":219,"prefill_start":0,"prefill_stop":83,"scored_start":83,"scored_stop":219},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4475867faff977bd5c0d96f97b251d767fa8320ea73f71a38a236d7f33bd9db2","anchor_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"anchor_positions_sha256":"debc2f860d51bf30c9ccfe7850d3428800e80a005f976ab2943a8642c33fb808","canonical_id":"824","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,30,38,46,54,62,70,78,86,94,102,110,118,126],"fisher_boundary_sha256":"652add26cddc5b2b11115550fc5858cab1d3a2bf00d5a3de3dd88ed6f9ee6844","horizon":1,"input_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,127],"input_token_ids_sha256":"f5e28e53573eb8607c244c66e29a9057150911f5b5348a120bb3a756c686e14e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128],"target_token_ids_sha256":"f2aa9adbe82c7f97119740cd60b33102619d0355f0db44358231d88ca624f304"},"formatted_content_sha256":"37af9d0afbb100c1787445b2ccb87b0a15ea15c7b2fac0560e9fc296bf6fe512","generator_receipt_sha256":null,"identity_record_sha256":"3933b2a829550c994be7ec3c13432e7fcc7eaa7429de8d86f09f2fdd0d54c382","prompt_token_ids_sha256":"e63b26e32e2d08ba0e1878bb31f438a8b4f0754fac3b288bf8ff4bc11ebc726b","ruler_category":null,"seed":null,"selection_rank":95,"selection_sha256":"3ae7adc084daf636812058b087ea0414390f1d765f67235a978166a970420f8e","sequence_length":129,"sequence_token_ids_sha256":"325eb23014b521a2ed7488ed8be2e2022f039e91aa7939745a5ea7decd8357c3","source_content_sha256":"f3f9375971e245fb7d4b9ecb4368b1469f96e8826d3d290349822de215c9524f","target_token_ids_sha256":"98ee353303b490751f3b0bf4a9db8ee9f144fd843e84aee5b61e6b8b237bdca6","token_span":{"cache_exposed_start":129,"cache_exposed_stop":129,"prefill_start":0,"prefill_stop":96,"scored_start":96,"scored_stop":129},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"95c07a4ae2c995c71644eb87f395b0201ab61998c92c88acd0deec2bdf52eb7f","anchor_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,208],"anchor_positions_sha256":"e3c9aaa391385675ab3475417e405da92d531052d937e046ddc609185dc2c7d4","canonical_id":"755","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,50,63,76,89,102,115,128,141,154,167,180,193,206],"fisher_boundary_sha256":"630fdfb50def6a0e9810e25416f083ca388d391614da06d1ea421eee032302e5","horizon":1,"input_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,207],"input_token_ids_sha256":"211b86c0afe55079edbfe3277616d686ed0a24152163e4ad422e5c2071214472","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,52,65,78,91,104,117,130,143,156,169,182,195,208],"target_token_ids_sha256":"279608d1d01474978b20236572dd00bca3894556f73197ecfd6a080fdab2cea0"},"formatted_content_sha256":"d7d00e78af67fa7b8d4f1123bc771c6c1838a42ca7940f09be045fa8e0258d3f","generator_receipt_sha256":null,"identity_record_sha256":"163bb04292002d23bbb74050efcec0fe6cb3a616a2ed635b0e83b30ad3599dbc","prompt_token_ids_sha256":"649e3b31340e844cf9aca294f09bbfb6ea6f59b8fa4339754a8040e2939a14ab","ruler_category":null,"seed":null,"selection_rank":96,"selection_sha256":"3b77b38c7af3c05a391dfb6e4427c1d6c8e1408b314a52ec893944d4dba7a59d","sequence_length":209,"sequence_token_ids_sha256":"5eadaf66f4c2a5b1419e30b0e9445bdcf5e76923985d3015fb3da1ae81a6112f","source_content_sha256":"f132344866b31da506e676ecd7f787c4f0b9c30ce84dbcf234d1bec372a4a3bf","target_token_ids_sha256":"b0d0429e68642809416cf5dc819cdacda9612ad5ec72ede704a6165361970f31","token_span":{"cache_exposed_start":209,"cache_exposed_stop":209,"prefill_start":0,"prefill_stop":107,"scored_start":107,"scored_stop":209},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e588a1ee6a1182055ad85a99122b37dca3e494260dd21a56a3847938dd6650d8","anchor_positions":[15,31,47,63,79,95,111,128,144,160,176,192,208,224,240,257],"anchor_positions_sha256":"58b00b7c1a9bcbca3b1a752019d06b8217b207235aef4a33f8862dbb76f94f4d","canonical_id":"705","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[15,31,47,63,79,95,111,127,143,159,175,191,207,223,239,255],"fisher_boundary_sha256":"7ddf7690ae9f4e218fef449a93f7970eefb48a98743e2cdab0de23b0a0dec118","horizon":1,"input_positions":[16,32,48,64,80,96,112,128,144,160,176,192,208,224,240,256],"input_token_ids_sha256":"fce5f6a7ef57e2e29ac18c5c07d311d066b45dcd8cc0d5feede90040805886a4","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[17,33,49,65,81,97,113,129,145,161,177,193,209,225,241,257],"target_token_ids_sha256":"7e886f55d5a1f767632ff6ce8330179e78261592f007a8d785bcacf263c6b5b2"},"formatted_content_sha256":"79b5d499f16fb4783be74bd200b7cdbb4738662932b42d91a8cb390646bd7282","generator_receipt_sha256":null,"identity_record_sha256":"fabe611b8a7c9e97b13d178a8020c782eeb92caeab58708b2acab9e36c7c9739","prompt_token_ids_sha256":"0906a739ae0536ce684360464813702b15ce64ce3043e3fad9895d8c7a910e81","ruler_category":null,"seed":null,"selection_rank":97,"selection_sha256":"3b7d0fef73308384ee4b6d98b48fefed969da32df7231a61067116bb7b9b77e5","sequence_length":258,"sequence_token_ids_sha256":"c89f8002d735b00177a7842bc2ce8332555e0cb94e7fc1f651c7d9306085ae48","source_content_sha256":"249e819abcbbc94c533d4966c357933213dbe7b58282534d182cfdd719ab0b02","target_token_ids_sha256":"f81a0d07e6c10d8a36f2da0ff7490f0c0aadc3fe1411b385e79f96372bff8f0b","token_span":{"cache_exposed_start":258,"cache_exposed_stop":258,"prefill_start":0,"prefill_stop":231,"scored_start":231,"scored_stop":258},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"21ba94b504ab5ded4ffa601325cf7df425e7f81789df808b86d2e08ba2117dcb","anchor_positions":[9,19,29,40,50,60,71,81,91,102,112,122,133,143,153,164],"anchor_positions_sha256":"e70afdac88bd21ff5ffd057462f0868a1aaa408a52296b58ecd4900f7f13759c","canonical_id":"614","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,39,49,60,70,80,90,100,111,121,131,141,151,162],"fisher_boundary_sha256":"b4910baa9b90efc3b7ffe399e1e19b050f46dd444b4bfa0faaadd835b9a3173c","horizon":1,"input_positions":[10,20,30,40,50,61,71,81,91,101,112,122,132,142,152,163],"input_token_ids_sha256":"09f0051a5092a284f02162790ff187253c2a8be10b80c2ff7b64476a0924f7b8","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,41,51,62,72,82,92,102,113,123,133,143,153,164],"target_token_ids_sha256":"9c3d0858143570f2b17fd9c211bbddf1beedb863bd39887f1982c64284451830"},"formatted_content_sha256":"b0e77c735851949333aa5a07a80654370a25c6f4c29b682d0a7296c28db21615","generator_receipt_sha256":null,"identity_record_sha256":"9e60edb916d1cba4b8692f32df62fc1c57894b2aa815769ff9a17da063de91f8","prompt_token_ids_sha256":"591d12041b2684415e1d05a0ea815d006597cc26105e19c6a0769dc0ea26a8bc","ruler_category":null,"seed":null,"selection_rank":98,"selection_sha256":"3c33175d44e2cc571159f6914ce6a7c57a1949ccdaa58585114ece582b46e4a7","sequence_length":165,"sequence_token_ids_sha256":"964232f668fe81da4ef5e6945e2164c11b771718b74fc029dbc71ae25a213e61","source_content_sha256":"10dc2964ee279825eee885f98c2883dcf718ce9958fca29aabfd2057996b5a19","target_token_ids_sha256":"fc3e58f7036836c2a7f90f009d9d954dbc547bc6a412334c087a495ec6166136","token_span":{"cache_exposed_start":165,"cache_exposed_stop":165,"prefill_start":0,"prefill_stop":140,"scored_start":140,"scored_stop":165},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"41e7124eadf2c8d65f12d91df35d39b81b0af62f8f0339d10d48d23001bed1f7","anchor_positions":[6,13,20,27,34,41,48,55,62,69,76,83,90,97,104,112],"anchor_positions_sha256":"311e7702507c904e70997e0b842193b8c519ddd45b6861c3e52fdd12fcbe662f","canonical_id":"835","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,19,26,33,40,47,54,61,68,75,82,89,96,103,110],"fisher_boundary_sha256":"ef27b3360779ff829870ac1205d40a423bbd1857f9d29564cf1a4bf273508b37","horizon":1,"input_positions":[6,13,20,27,34,41,48,55,62,69,76,83,90,97,104,111],"input_token_ids_sha256":"622f2d82715555a3a862da50633d47a53da0f6d05afb23f037425ec2460ebf97","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112],"target_token_ids_sha256":"e4f44986dfe9f6afb783e79bc0788f586424b03e794cc20ab1129e7a75ca4b91"},"formatted_content_sha256":"16149587d9ee5b76bb9036394de7cc48051ba6deeced83ecf9552046bdf3d6fd","generator_receipt_sha256":null,"identity_record_sha256":"bcd976206ac1782d1fd75247a66068233d32764eef93e916f4303ac7216894e6","prompt_token_ids_sha256":"defa46cd3ab312e32247db6a9f87d5520f8ebbaa993a9af787a0d44ecc31a91b","ruler_category":null,"seed":null,"selection_rank":99,"selection_sha256":"3e545f645fe778609ea47f9d9d70db3a2d6fbf7c1b704f5c4b892a732ca4870c","sequence_length":113,"sequence_token_ids_sha256":"d87292b8689b41a713b9f23d42d1936617675fed9a389358046c37dee67270a9","source_content_sha256":"a2d12e2a125b7130ed036625cab03ca87319d5c2bc2f78d56e5b06e5d48967bb","target_token_ids_sha256":"d18a4e331e8bc3c665aeec8b4e8173eeaf216f691cec5deedcce9b89d1eabcf1","token_span":{"cache_exposed_start":113,"cache_exposed_stop":113,"prefill_start":0,"prefill_stop":85,"scored_start":85,"scored_stop":113},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4428369199e6df8c36558c477449442560d7dbefe1b1a34029eb0d3a540e8850","anchor_positions":[8,18,27,37,46,56,65,75,84,94,103,113,122,132,141,151],"anchor_positions_sha256":"99b598ab1b26514b0793ce6e9d9b8fd5fa21cfd91c15cf4ba944b02280eccc00","canonical_id":"785","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,27,36,45,55,64,74,83,92,102,111,120,130,139,149],"fisher_boundary_sha256":"224129c3bde0a7dc09aba43d89bfcf9df160c62a8d883b2421405de57da83148","horizon":1,"input_positions":[9,18,28,37,46,56,65,75,84,93,103,112,121,131,140,150],"input_token_ids_sha256":"d1cac9f469f540d35dd51eee4d84dc6489bdfbcb831d778ca3b910593caad5ba","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,29,38,47,57,66,76,85,94,104,113,122,132,141,151],"target_token_ids_sha256":"293d996a68091856a066009f8299d24def6c9ecd1505302fd1289596b4ce5929"},"formatted_content_sha256":"55cc81d66bd2973ef60996d3a13b658ddb8e10ba036a3a91808a8c4ae225b19c","generator_receipt_sha256":null,"identity_record_sha256":"70a4011b9bb314496b859d408f1205aaa618d08b8ca5504e4b0b7a8b3a794737","prompt_token_ids_sha256":"b5c378f6ed8ac42eb8be052f6d123a663ddfcd36c4bbcfc5decae012cf1c1926","ruler_category":null,"seed":null,"selection_rank":100,"selection_sha256":"3e7cbaec0d7be5d2de99697b23da0c61163c5bc4acecf283c4d9167e7b780eae","sequence_length":152,"sequence_token_ids_sha256":"ea3cdc2b1f803377a8cdf0588f013cebc2f019e5ad60fdf01294bf2d04d429ca","source_content_sha256":"06e8fb9b807385f5950f7335cb79a3b8105f621f1f5ac389155c9e737ff13d4a","target_token_ids_sha256":"a21eb422039b5c07e6a0e5f71858bbc8eaa8c315d06b39437977061a7cfdd253","token_span":{"cache_exposed_start":152,"cache_exposed_stop":152,"prefill_start":0,"prefill_stop":109,"scored_start":109,"scored_stop":152},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1cad8beb17ffee15f4de17ab4607544233d3683ae14b0cba6d063ccc9f11ff77","anchor_positions":[20,42,64,86,108,130,152,174,195,217,239,261,283,305,327,349],"anchor_positions_sha256":"3e58027edb04b4c749c71fe1c1753ad8300a599facd28b3d327dffaa160dd92e","canonical_id":"662","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[20,42,64,86,107,129,151,173,194,216,238,260,281,303,325,347],"fisher_boundary_sha256":"a261daa2db8ba7f05e7ff89074fdc6a0533b335ee67208179b8229ca3e70980e","horizon":1,"input_positions":[21,43,65,87,108,130,152,174,195,217,239,261,282,304,326,348],"input_token_ids_sha256":"d7e08d7e4f81bf68a6cf07dae075daaf7f79121f48d39ad9dba80fb057031e6d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[22,44,66,88,109,131,153,175,196,218,240,262,283,305,327,349],"target_token_ids_sha256":"8501644795ff2f929a4924fb2a3a1644ea894aa81320b17182c421175240b4b9"},"formatted_content_sha256":"91184e28eff6c3890010723ccfc61b5b4736a602d677efdffe59a220b3bac9b5","generator_receipt_sha256":null,"identity_record_sha256":"45b6269ce5d7d2ccefccb9d3aff10c4cc786d4482f036ebc1e0837d56e6c1923","prompt_token_ids_sha256":"ee07e3808c4e1142a8b3f904c8582078b899979e541c1ebe2297a3670112077c","ruler_category":null,"seed":null,"selection_rank":101,"selection_sha256":"41900ba952796bb2c024b96db8f08d2e63a757587e61fd96909dc934d50da156","sequence_length":350,"sequence_token_ids_sha256":"6890b69767a6c590160f23954c0338831fa0e3912f06dd6514c5b1552317c006","source_content_sha256":"3cf31926336b0e44d3630a63a486831d0b79632309d78e0b65f28e1c6f37353c","target_token_ids_sha256":"38e5b68f608a60d1fc61bff0194cd835196d4f8e8acb129c9e0744850d8641bc","token_span":{"cache_exposed_start":350,"cache_exposed_stop":350,"prefill_start":0,"prefill_stop":319,"scored_start":319,"scored_stop":350},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7d64f9e8b46b0ba25c21f37dee8e110df8862303d52cb3c2697353097a97db76","anchor_positions":[7,15,24,32,40,49,57,66,74,82,91,99,107,116,124,133],"anchor_positions_sha256":"6c1029515cd9fd840ee9baeeb3bbf16de22d914596dc66d6c0b00a017e82aef5","canonical_id":"732","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,32,40,48,56,65,73,81,89,98,106,114,122,131],"fisher_boundary_sha256":"2ea117a68a1f560fe1206940c2b26cdf6971aeeecd14dde580d013efe50b498e","horizon":1,"input_positions":[8,16,24,33,41,49,57,66,74,82,90,99,107,115,123,132],"input_token_ids_sha256":"46da8cb7bee3ade0de904115c405d0a4e2c1e215d9814066bfc5c50885b092d7","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,34,42,50,58,67,75,83,91,100,108,116,124,133],"target_token_ids_sha256":"3dd8aae9f6aff8d199075c33306d8bb5bd3b5f73666f2d236bd066a6d55fcbe4"},"formatted_content_sha256":"cf5f521e053dcaa96c55370f299514c9de27ccc98b8d846b0a51c4d2450341e3","generator_receipt_sha256":null,"identity_record_sha256":"1ea4e6a64df86bf2e3771dfc41f1716146cfac365a658d3864c030013a4cd84b","prompt_token_ids_sha256":"d705f66b234374474a57207de30809b5832cf7196bccb9dd73addbb879937264","ruler_category":null,"seed":null,"selection_rank":102,"selection_sha256":"41c7c18c717892ef0cbe99444d2f1d3d16142bd3128dc408df64987e27a8531f","sequence_length":134,"sequence_token_ids_sha256":"c98483cae5d714b2a8de61cd02257cc0ab084c30a0673971a4460e6d07f59d2c","source_content_sha256":"e5aeda9a803b146c76d557a6c8dc84720e938726b6f35982a5e61465a9a9d11c","target_token_ids_sha256":"15d06316804c670aad3848c3d01e3abb336656533c4feb76f93e90a0399aa760","token_span":{"cache_exposed_start":134,"cache_exposed_stop":134,"prefill_start":0,"prefill_stop":111,"scored_start":111,"scored_stop":134},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"a1bc1d7610d52bcb926390555c1cab77e3b88be528f3e256ea9e9a8106902d2a","anchor_positions":[7,16,24,33,41,50,58,67,75,84,92,101,109,118,126,135],"anchor_positions_sha256":"9058652c46869f6bf5cde3a730d23fba0f016461c119f6bbc8af3fb1c3031545","canonical_id":"679","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,24,32,40,49,57,66,74,82,91,99,107,116,124,133],"fisher_boundary_sha256":"810b5e7562357db03cc4e0193687f43bf7f4a08839cf3425a51f4e629ca7f5f6","horizon":1,"input_positions":[8,16,25,33,41,50,58,67,75,83,92,100,108,117,125,134],"input_token_ids_sha256":"03872be7640939a2770876b1cc144c05f59d3e06a6418658db2bceeecdbb5f11","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,26,34,42,51,59,68,76,84,93,101,109,118,126,135],"target_token_ids_sha256":"5c71155464943a602fc92f92c7507287819905cfa27f665ae40e126b898cdc40"},"formatted_content_sha256":"ffbf833b467a2b56b26bc50bcb320d893f41f60eed6341ca4c61813965020c7c","generator_receipt_sha256":null,"identity_record_sha256":"13c42ddbb25f335f5eaa4ef5df7c2cecd1ea19af1f8972ade37312c4fce78802","prompt_token_ids_sha256":"df73b549576d1784c30bb45b9c873ebb80267ac6c56afb4335eadca3c7c56849","ruler_category":null,"seed":null,"selection_rank":103,"selection_sha256":"41da3df61a258a1897eeac98083550d328825fdeec35836584dc0b7a76b060b4","sequence_length":136,"sequence_token_ids_sha256":"fb871500f94a5409b4acab60ab3e2aa6955924b9ceb87214232c4099d302dab0","source_content_sha256":"39571a4165c49d0610e00d19d3f5b5e1879232f2d0429100e2aaf915eaca9ab9","target_token_ids_sha256":"d6eaabcd23781ac4a910aadc33823fd08ab066d01331a124767f6e84f181ffba","token_span":{"cache_exposed_start":136,"cache_exposed_stop":136,"prefill_start":0,"prefill_stop":118,"scored_start":118,"scored_stop":136},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b8947897cdec3fc049ba3461c75adcd685b2264e6810778decb68c1db034f7b8","anchor_positions":[17,35,53,72,90,108,127,145,163,182,200,218,237,255,273,292],"anchor_positions_sha256":"c72256a78f816fa7bf8e540cc3aaf3fc1795369a3859efe48a9c4a8cce0d2489","canonical_id":"895","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,35,53,71,89,108,126,144,162,180,199,217,235,253,271,290],"fisher_boundary_sha256":"d12370caffbc875b01d6a59c8cf4c99d8fb4c93c519041b5500142fdac3a8340","horizon":1,"input_positions":[18,36,54,72,90,109,127,145,163,181,200,218,236,254,272,291],"input_token_ids_sha256":"a316e2b2888cea8097ab8420cf22051694e1d976c3b5b32f83f5207f4007d89f","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,37,55,73,91,110,128,146,164,182,201,219,237,255,273,292],"target_token_ids_sha256":"ea285c83c7e7736e7a436adedf9c66203844bbe2521ff7ad148dd51b419e98f5"},"formatted_content_sha256":"9b9360fb9e789a22496cf1d9c91e89cf7895942447ca7e16f15acbf2fe6515b8","generator_receipt_sha256":null,"identity_record_sha256":"080db8e8894bfd652b320dbf19c1d4e53c8b6ed2626bcb8f8d3cdf87268005f8","prompt_token_ids_sha256":"3ed029adce95edd3406c2359cabf621905d3ecaceb70e519fb1cc89a66899110","ruler_category":null,"seed":null,"selection_rank":104,"selection_sha256":"43113137520f4bdb68e26d41a096fa093a2f995386d5473618f0bbe4b2a0f73e","sequence_length":293,"sequence_token_ids_sha256":"24d56709934330dbe30c61a49434a07def76fdd2b31e47f84f71b050fc7aff39","source_content_sha256":"d1edefa8e37752a9da687c2a2e51086bdfe6401714de0bd593a892ecd4fe7bcb","target_token_ids_sha256":"e894e28b0e66238fd8988d8b0b978628f3354d87898164223cb7c7dc2ce5a596","token_span":{"cache_exposed_start":293,"cache_exposed_stop":293,"prefill_start":0,"prefill_stop":161,"scored_start":161,"scored_stop":293},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7d9d79586d3fd7a1376390bd1f085ee9ec35cb401fa471d99ddf45417339ce49","anchor_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,160],"anchor_positions_sha256":"cea919870992245499fb8d8a858d848e1e76f02a4e2230ada935442533c03198","canonical_id":"727","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,18,28,38,48,58,68,78,88,98,108,118,128,138,148,158],"fisher_boundary_sha256":"768624e8d0cb29118f002fce04ec89945eebe14e7587f370283fa04b2d90cf95","horizon":1,"input_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,159],"input_token_ids_sha256":"93e60eaabf27cffd7d1fee3301e42b3f62e183c9e40317db494ede3450b1ea7c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,160],"target_token_ids_sha256":"507cebbcf35bc57eb21aeff74f69ab933d22c31de894e77b573193730da7662a"},"formatted_content_sha256":"934eb88ee283eea8ac09e46d9333c08ae4e41a6910aa75e0638bc73403184864","generator_receipt_sha256":null,"identity_record_sha256":"aac6297b0c851ed51271119b880a93cb272776311e2b9e97f498c76785255100","prompt_token_ids_sha256":"ad50cbf08781e9a1429766f1a21e3ed4a76d9fc30e43b5b8d968dcda018da4d4","ruler_category":null,"seed":null,"selection_rank":105,"selection_sha256":"4338156536b7925e6044f19142d418110ae27757c47306d15815155b2d682917","sequence_length":161,"sequence_token_ids_sha256":"422567bd8e19fbdf69ab47274fe1bda49c8ac4eae311d7232ee13e0888b0fa23","source_content_sha256":"400a362fb6664f9084e588aa605b6e7ad1505849ab7c769574e0996829f6827f","target_token_ids_sha256":"c8e9d40bce2584a16cdb8ee232a9e1f86dd9c58c6fee7999125c37073c1f5bb5","token_span":{"cache_exposed_start":161,"cache_exposed_stop":161,"prefill_start":0,"prefill_stop":134,"scored_start":134,"scored_stop":161},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b50fd03bddce1923b315763ae4ef228d2b6c9cce2b80954554946c98c92a3760","anchor_positions":[8,17,27,36,45,55,64,74,83,92,102,111,120,130,139,149],"anchor_positions_sha256":"27ff593f69b04d82cc98f1ee839db50560b1d26eb9ecd645cc99a4e74dade2d0","canonical_id":"629","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,26,36,45,54,63,73,82,91,100,110,119,128,137,147],"fisher_boundary_sha256":"d8f311b4a33da5b28927ed11e2b7efbb2a1812456e5cd54c884287cfc1ce634d","horizon":1,"input_positions":[9,18,27,37,46,55,64,74,83,92,101,111,120,129,138,148],"input_token_ids_sha256":"f56a22b67052689f706d00c90e6fc5541583c78a589b0b3a26c6b93eda4162d8","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,28,38,47,56,65,75,84,93,102,112,121,130,139,149],"target_token_ids_sha256":"93c8a6c8bdc5c1ed56f7db15ae4424e87803630ba80e3845d421f97169235f34"},"formatted_content_sha256":"9c29f28fcb8fb63c6faa67311c4d67889b21525c18b625ca3fbd69bb6611342e","generator_receipt_sha256":null,"identity_record_sha256":"dd6a317d0ffc82ff3c3b71c470d6f7f4ee8c3276d55ea892d77526d03f80719a","prompt_token_ids_sha256":"ae80eeafd124153f30025382491abc9552e2fb6ac10aa0e25acea39bb6cab5a7","ruler_category":null,"seed":null,"selection_rank":106,"selection_sha256":"4357903439ea78adb2c1be88172806c1a5e5d57f1bfa26229be4d3d56bf15b5f","sequence_length":150,"sequence_token_ids_sha256":"58d577039503bfee517e7ec9f595d3378f0520d7b10780457232e42ac1f8b9e0","source_content_sha256":"d75386bde56ac72aa78e5b13019f0613da914d25303ec7ae96f618caf713bf8d","target_token_ids_sha256":"25dde1ab97315ca73c3e47f45297c03f015715e1dbf87dc40a90d1fff93faa1d","token_span":{"cache_exposed_start":150,"cache_exposed_stop":150,"prefill_start":0,"prefill_stop":109,"scored_start":109,"scored_stop":150},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"743e09d0b63e0d1d69cb1f4d92a19f1b612ce8d5ed318965c7cbc5ef10da4ad0","anchor_positions":[5,11,17,23,29,35,41,48,54,60,66,72,78,84,90,97],"anchor_positions_sha256":"3a7e2557ca5cb4f3e315ceb26b0546b57f9a7cdbff08065541d63492d4059513","canonical_id":"789","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,11,17,23,29,35,41,47,53,59,65,71,77,83,89,95],"fisher_boundary_sha256":"03a7963bc281465bc3ed237e9329c2d675fc5a2743cb29e2366d18c053e4354e","horizon":1,"input_positions":[6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96],"input_token_ids_sha256":"51ddc5b200c5449da711c0cb31fd958274770a19f7df1e0a919bfde5c62fa46c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,13,19,25,31,37,43,49,55,61,67,73,79,85,91,97],"target_token_ids_sha256":"5b9e09d236094c93cd27152a8362d6fc0907405ed836f4c806eb6a49ce662637"},"formatted_content_sha256":"95cda84841f0ed6748d2aa3eab9b3f2a1d602bef0648c2354cb235fbc49eca99","generator_receipt_sha256":null,"identity_record_sha256":"1b904dd9e776f5697d87583519b28c5869436344070e3c267f5d2f9cbe24f679","prompt_token_ids_sha256":"59d140fe1bf3dedbe286b918f769e76d9062fc0a8677694f8a29a4264c3dcfc6","ruler_category":null,"seed":null,"selection_rank":107,"selection_sha256":"435d524443c963ac0697492db5d755ed96c21825c6c12686e14ad1216a9d98d3","sequence_length":98,"sequence_token_ids_sha256":"f843f9ee0ba37389203b844c5d69910ebcaaa1266a694030f5cd56fa427be6b8","source_content_sha256":"528cefc12ccc28d572fc35698c3e55d5fa17974dcd83ab45fe269d5d96166d59","target_token_ids_sha256":"e052b469e73f16555fd8aae9d7e12029ab50d45bb5400880bce25ee85f7c0582","token_span":{"cache_exposed_start":98,"cache_exposed_stop":98,"prefill_start":0,"prefill_stop":75,"scored_start":75,"scored_stop":98},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"56264414b48afbfd02d06a09b83695f1be6a109c5e5917043688018a40e3a3c2","anchor_positions":[23,47,71,95,119,143,167,191,215,239,263,287,311,335,359,384],"anchor_positions_sha256":"5ecef0009b1f2b5ca1c05ae59fc5945152ea0233cf7a7da1d75b9641a0da44d4","canonical_id":"819","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[22,46,70,94,118,142,166,190,214,238,262,286,310,334,358,382],"fisher_boundary_sha256":"83c8e0577de0e56d18d8697c6d54b87b15a0a615623c44993bd8b9c2163d30c8","horizon":1,"input_positions":[23,47,71,95,119,143,167,191,215,239,263,287,311,335,359,383],"input_token_ids_sha256":"e56c492befed4cd096e4eb5258b978187c18cba26036c7ea511ea7b3828aea8a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[24,48,72,96,120,144,168,192,216,240,264,288,312,336,360,384],"target_token_ids_sha256":"b5f4bffe29cf172dbb04ddcf7fd6b1f217b698f7016f31ff8f6e48c727e2c9be"},"formatted_content_sha256":"616a3535fe1ec791a293ac6da5c856a7712282d6db633cd638270d4013cec624","generator_receipt_sha256":null,"identity_record_sha256":"c33ba3115c44c968c4ff9d9c33e133ea4f975549b4e7717c6581b9f976c187f0","prompt_token_ids_sha256":"de127a7cae4376da7d66c0f3f1eae83db093b0eee64a2a21dde2ac28c80f0322","ruler_category":null,"seed":null,"selection_rank":108,"selection_sha256":"439c6f2d57f5500b6cc21860ee22a1ebce8b83eeeb2bb034371733828ffa361e","sequence_length":385,"sequence_token_ids_sha256":"5ff88ef079fd63302a6d16c9b8d2bb597b0886ed6975c9ee0d7005fd42a6852e","source_content_sha256":"1b57006191af74fc5cabf13ded0936d54e33d0eb2d92e12170ffbdf83e11e029","target_token_ids_sha256":"f0b593d02c8c7048d24a55e2360b1e42ecae542816dcf5a6d3a7a5f0d3baa103","token_span":{"cache_exposed_start":385,"cache_exposed_stop":385,"prefill_start":0,"prefill_stop":268,"scored_start":268,"scored_stop":385},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"f83dda5f76b6c77a2f8f34d32d59899a5438aa93e284ca72500f57061c684a29","anchor_positions":[8,18,27,37,46,56,65,75,84,94,103,113,122,132,141,151],"anchor_positions_sha256":"99b598ab1b26514b0793ce6e9d9b8fd5fa21cfd91c15cf4ba944b02280eccc00","canonical_id":"886","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,27,36,45,55,64,74,83,92,102,111,120,130,139,149],"fisher_boundary_sha256":"18cbe59ca514ee6ee4de7a9ff59d5f2173303011464fcc35cb15114c03969d2a","horizon":1,"input_positions":[9,18,28,37,46,56,65,75,84,93,103,112,121,131,140,150],"input_token_ids_sha256":"7e8df804a72a77f8e56f283f6f3d3e0ccf7bdd663763c7ba3cf49b6cf2c8c479","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,29,38,47,57,66,76,85,94,104,113,122,132,141,151],"target_token_ids_sha256":"fa410d4aeb53ce61e53c7bf6ae6bf9008a5152a79caa4db0b21713c71de06b6d"},"formatted_content_sha256":"1be2ddc47aaf0ef4c6ce6979f16a7daed6383578cbb0e12a913e8c76157fe911","generator_receipt_sha256":null,"identity_record_sha256":"16353e62969e446cb1fd7fc9ca7a99ffad2f73e31b73381c626a08c7293cb32b","prompt_token_ids_sha256":"fdf8cba22d99356b2668bf60f4e3c5f33e9981ae71ede27ef250e9945224d770","ruler_category":null,"seed":null,"selection_rank":109,"selection_sha256":"43d5d553cb2e28075980d0bd0920a10d225c7497df856fe96c4367fbe7e6bace","sequence_length":152,"sequence_token_ids_sha256":"8853c262834e9d1226bdfde776d4ef0e7a0e7c8778a464180efae64bfb4f3c20","source_content_sha256":"434c4f97415e9ae869612f384ad8eb1d4459b91e71891d13598de2c0ecf58ce9","target_token_ids_sha256":"c73711379fdd03655ee8104d211db2039b2bd1b1d8757202f91122e2e8800f82","token_span":{"cache_exposed_start":152,"cache_exposed_stop":152,"prefill_start":0,"prefill_stop":120,"scored_start":120,"scored_stop":152},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9bfe15074048c1b3cc4e9a64f9f82a7762f5e3fc7b6286e9505bebdfc8ce78e6","anchor_positions":[17,36,54,73,91,110,128,147,165,184,202,221,239,258,276,295],"anchor_positions_sha256":"54ac14bb1cffe66a9bd2a063df3998236174c0775660ded8a46c6b4f3b34b4be","canonical_id":"669","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,35,54,72,90,109,127,146,164,182,201,219,237,256,274,293],"fisher_boundary_sha256":"e64ce615c2670462c794e8f67c34ff9ba26d820a1d46601098cb0321f7249f65","horizon":1,"input_positions":[18,36,55,73,91,110,128,147,165,183,202,220,238,257,275,294],"input_token_ids_sha256":"2147411fed9b01bc77a54da5a575818dd51a16507fe185b59c616201c9dbe92e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,37,56,74,92,111,129,148,166,184,203,221,239,258,276,295],"target_token_ids_sha256":"d9cc42057cf2f44e2193c2fff4ec48c3d54db14f1369ce5096749eb0a37be2eb"},"formatted_content_sha256":"fe004ffe4d515aa06edc2d7a6ff8f2900da04c2c06fe055101a34ba872ba040d","generator_receipt_sha256":null,"identity_record_sha256":"5a4fdfb2bb3bdb18316e26b3524adf61c217b8661a9502ef4888ec80cf44a517","prompt_token_ids_sha256":"545545ebb1988d32125cc7c0b6360272908d9ddf46dbea3b96b9923e8f61a5da","ruler_category":null,"seed":null,"selection_rank":110,"selection_sha256":"454527a603157418a471cc4d85900b7bc64b30edc7df17206aecd43f4b706a81","sequence_length":296,"sequence_token_ids_sha256":"0c56d0026a937592d71b7773e5acad5e8774badceb5806f30884aac130918445","source_content_sha256":"76e713c3e769794fd21cc2b169d3fd25155269e05a87202fa2ac7130dd7d9534","target_token_ids_sha256":"e5afdd086f0ae8795afe5c271dd9f9a4fae2f74fda8cd0e04e3f697b2d39bb25","token_span":{"cache_exposed_start":296,"cache_exposed_stop":296,"prefill_start":0,"prefill_stop":113,"scored_start":113,"scored_stop":296},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"bba06f38d7ef46d2274fcc3c62d6b99545aea08fc9a970bb2cf4112497f86d55","anchor_positions":[18,37,56,75,94,113,132,151,170,189,208,227,246,265,284,304],"anchor_positions_sha256":"94f33a91bb15b5b174db14bd5a18a220943ce79ec8eba0ee2655b965e84d64e0","canonical_id":"846","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,36,55,74,93,112,131,150,169,188,207,226,245,264,283,302],"fisher_boundary_sha256":"e56eaa1d53ef9cac5e4ea05eac5fae5d2ff32456ec4fd16aa432bb0ba682df16","horizon":1,"input_positions":[18,37,56,75,94,113,132,151,170,189,208,227,246,265,284,303],"input_token_ids_sha256":"b57a987371cd39d03b724b08c3b6837260a5e8ad281a18e85c9901110811e452","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,38,57,76,95,114,133,152,171,190,209,228,247,266,285,304],"target_token_ids_sha256":"adb018d6d0b2d7716ff0afe479b6ee4cb1e352197e41003e0e3962d666cf8a0f"},"formatted_content_sha256":"d783edb3ae5e40760a243586ed62d48e4d1fab758d36793b9e509e3e34c74a3d","generator_receipt_sha256":null,"identity_record_sha256":"3c715e7b82c94eee9feb5fcc68ac303cd191e6ec2068a3d300d3ca8e4dd6e5de","prompt_token_ids_sha256":"604c21c3a8e0575c89ca1b7911771a2a7ef7b4a915f2b165be5a3a2c56b206e1","ruler_category":null,"seed":null,"selection_rank":111,"selection_sha256":"45dda3a343ef8f05e6b253db4cedda79669dd8cedb0368cd7ba98b9b569952cd","sequence_length":305,"sequence_token_ids_sha256":"b0ea9f94ab26fc47af030e71000bddd166cb8e1001febd6121f29acbdf792437","source_content_sha256":"e03f7274bd5dfa833b5d69f144a32a2866a4d5e2df73779e455ab0a797e8b9f6","target_token_ids_sha256":"7c55e00d0effcd9ceef682e102dd49697117fe7a162f11e2bcd35a18523b96dc","token_span":{"cache_exposed_start":305,"cache_exposed_stop":305,"prefill_start":0,"prefill_stop":182,"scored_start":182,"scored_stop":305},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2f002330a14ccd49bbdf1186b94d7f4a660f3c79ca0935f6733429506fbac4dc","anchor_positions":[9,20,31,42,53,63,74,85,96,107,117,128,139,150,161,172],"anchor_positions_sha256":"9bb32a96de9c1958e4d75c68046a769ab85020ffa742f7286cba9f5cfe66172b","canonical_id":"809","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,31,41,52,63,73,84,95,105,116,127,137,148,159,170],"fisher_boundary_sha256":"b791fc560913737b45746c61831af949c9b84432325347422a7a866d99f55180","horizon":1,"input_positions":[10,21,32,42,53,64,74,85,96,106,117,128,138,149,160,171],"input_token_ids_sha256":"35bdf9fe7875294a945c0b7fa771287b6e892aade75b41750c2e0144945a8d4f","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,33,43,54,65,75,86,97,107,118,129,139,150,161,172],"target_token_ids_sha256":"901d22acbaeea26debc37b388abaf43ac50248b0422016d07fd3a292798c4cd0"},"formatted_content_sha256":"7d79ec67027ef058aa16aa49df49541f7821cc1e06a8283c93d05218bee5c30d","generator_receipt_sha256":null,"identity_record_sha256":"00931944a29fecfc4c8c6b60832e5269dd3d431f1fc0fec8f21cfc3d5b2e56c5","prompt_token_ids_sha256":"d2c4a745f10a0a0263c72f290f8ef232ab4aafe62f10358bad7a317ba0990e25","ruler_category":null,"seed":null,"selection_rank":112,"selection_sha256":"4639d4a26db93822b4b2e18576a8169bd874398e85f3a592904ffc3f972fbb17","sequence_length":173,"sequence_token_ids_sha256":"97308fe18d6a1e87ca3681748a23734c46402fa3f1992ea007b186d80d049540","source_content_sha256":"197f56fd23a55154213be6170da63f959f105eb46b7cd9cb707820e74a06529e","target_token_ids_sha256":"c3f58b0c22170cfdfc5245c921af88598385a2623da14949ba7503f68723c427","token_span":{"cache_exposed_start":173,"cache_exposed_stop":173,"prefill_start":0,"prefill_stop":128,"scored_start":128,"scored_stop":173},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"696c3eff19f8cd3ba60be744325a381bcade3a5bde2058ce2b7057bd86a602e7","anchor_positions":[8,17,26,35,44,54,63,72,81,90,100,109,118,127,136,146],"anchor_positions_sha256":"95e4d2457eadd7ff2ebe49af36c11afbb6034a7c8d08013efa9de5325f17bb5d","canonical_id":"797","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,26,35,44,53,62,71,80,89,98,107,116,125,134,144],"fisher_boundary_sha256":"0f290f46b842b2d9555c6342ee65ad579f7f6195e2840a98edaf138b7f77e879","horizon":1,"input_positions":[9,18,27,36,45,54,63,72,81,90,99,108,117,126,135,145],"input_token_ids_sha256":"45d165859afaa71c112d2d4db0282de95319684d4d160e6d34be3255f19e378c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,28,37,46,55,64,73,82,91,100,109,118,127,136,146],"target_token_ids_sha256":"dd11e1c91e7cf97f459b2696ad0d206a7e54b2e224552b8590c9c6431e8584a1"},"formatted_content_sha256":"8e313e746cb70b0f39a63b254855866da5978faa757f114d540a3e7ae9d63cbf","generator_receipt_sha256":null,"identity_record_sha256":"6952ce6716bd8931f4ca2c58ccaf72f466062aa0785261289d59a5104678a7b5","prompt_token_ids_sha256":"04b1d2f628e08d770e64feea8b70cbb7a64b347ed50b692622c6c3e0c7c8029d","ruler_category":null,"seed":null,"selection_rank":113,"selection_sha256":"46503a4924e57013ffda3af84d33a2db536b465afca8ee866ff2a2e87961e472","sequence_length":147,"sequence_token_ids_sha256":"dbf6194f0a08074850f461642c989526f8fe9049d05306dfd6a2da9a9987f8f8","source_content_sha256":"aa80156f16bf70bd62cebda84762b9a1a8ae437dc8871c0acb7d61e9d83b1b8a","target_token_ids_sha256":"d91b2a60c8ed8407d0c17107efc9a4cc4e719d8a094196cb04d089c6d8aea011","token_span":{"cache_exposed_start":147,"cache_exposed_stop":147,"prefill_start":0,"prefill_stop":91,"scored_start":91,"scored_stop":147},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b907107ca4056a13d77b90b11cffa6104f002742e48e4d73d84701f35cda3ce3","anchor_positions":[7,16,25,34,43,52,61,70,78,87,96,105,114,123,132,141],"anchor_positions_sha256":"4a3ecd9aedde633817cc01e0837692eec5dd57faddeca3f7bcf5e18333ad7c64","canonical_id":"803","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,25,34,42,51,60,69,77,86,95,104,112,121,130,139],"fisher_boundary_sha256":"8c5dd3de4221407fb31879a8831c58de078b7bd96edff29efabeb0d4f941ca5d","horizon":1,"input_positions":[8,17,26,35,43,52,61,70,78,87,96,105,113,122,131,140],"input_token_ids_sha256":"612f42435d82b79fa57bcb8dc07f12f964553d764d942f85ac5c989daa0ac60d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,27,36,44,53,62,71,79,88,97,106,114,123,132,141],"target_token_ids_sha256":"20ecbcd0e6f4d0f38e6e34055d4306d4556676db803bddcbf4efb50dde35e518"},"formatted_content_sha256":"b80b90161540d8a9b1c0ba6896342db1e035ccefbe96663db9df9b74d82a3eae","generator_receipt_sha256":null,"identity_record_sha256":"cdbc48a4424d2515ce6141290d69310f055340010d03518c808f7666de9eecb6","prompt_token_ids_sha256":"9a94491891f327307e6c2d541dae50c594d32f7e7113f10a4ad421cd16f7efee","ruler_category":null,"seed":null,"selection_rank":114,"selection_sha256":"47a81fc27a92568642839ad9294c190dcd0b21f98ad4eca4472567609f81bac4","sequence_length":142,"sequence_token_ids_sha256":"4e6d4b6ce7a5a64ad6b5d03e8b6c0786f2a1206a63cd36a35f88e41282093685","source_content_sha256":"fcc558736fc46d07648e59309cc19de0322f28060b2265985cd5cb46f9ccac97","target_token_ids_sha256":"16fcd9ca012cf831e086d70b1c2f66e51e595f470fb2610a600125f50807abae","token_span":{"cache_exposed_start":142,"cache_exposed_stop":142,"prefill_start":0,"prefill_stop":82,"scored_start":82,"scored_stop":142},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"61f0fff09b788c1dde584ff85418eee082c77947d695e4df4e831eb33cf048eb","anchor_positions":[10,22,33,45,57,68,80,92,103,115,126,138,150,161,173,185],"anchor_positions_sha256":"fde5a3b64a487819cb5be9d572133781e4744390c11836e2f115ba2608fe7216","canonical_id":"644","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[10,22,33,45,56,68,79,91,102,114,125,137,148,160,171,183],"fisher_boundary_sha256":"3c448eaba6e976f4d8540a8018f1b6ee02f1a078e5ab97372fe15de20e37c79d","horizon":1,"input_positions":[11,23,34,46,57,69,80,92,103,115,126,138,149,161,172,184],"input_token_ids_sha256":"61a5b967b5a2460665ba8ac148a2ed8fda9b1a0e95c8d66a509db1c89e384bb5","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[12,24,35,47,58,70,81,93,104,116,127,139,150,162,173,185],"target_token_ids_sha256":"e6285297df83884b53ce8b429f39bfbc76734d91c897b1ef6b45e02c90bdc53e"},"formatted_content_sha256":"cc789f8b54535a69935595ebbc2d15ace8da8a517f3e3a766f4c9801ab975e73","generator_receipt_sha256":null,"identity_record_sha256":"add76763c425e50ca33682da8956ff29cc4547518dd84c9fea5b339de6f6125b","prompt_token_ids_sha256":"a672313ea736e1353ebd2a843db4d5ae378ddbec92e54cb81d21376a13087e88","ruler_category":null,"seed":null,"selection_rank":115,"selection_sha256":"49614c4b89fd4e064236884bd54d2b6744d30da34cae36de98a3f8d6dc005295","sequence_length":186,"sequence_token_ids_sha256":"2a629a0ceb8ed135d2d3814402c87b6b4d4925c16ec65dcf756469c7f70d28dc","source_content_sha256":"6ca1519c2cb7b633366e44afc37a261e40427c751412998cceb863343020cf4e","target_token_ids_sha256":"147bf03e5bc17953126d30a0640ae17e4b57cbaf4d65322290bee14c6f8c12a8","token_span":{"cache_exposed_start":186,"cache_exposed_stop":186,"prefill_start":0,"prefill_stop":160,"scored_start":160,"scored_stop":186},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0a09087a0815d40c6da7249de80b96629effaed44659b49eb51e112de7acf916","anchor_positions":[5,12,19,25,32,39,45,52,59,65,72,79,85,92,99,106],"anchor_positions_sha256":"1ec66db99ded67a6b74f7dd5259127f2ba61fd9a33265762c1ea04710558d7bc","canonical_id":"647","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,18,25,31,38,44,51,58,64,71,77,84,90,97,104],"fisher_boundary_sha256":"bbab335f18e2c0ccb810bd85c071826d79d9b490e590cef0c6d681341c0cc08e","horizon":1,"input_positions":[6,13,19,26,32,39,45,52,59,65,72,78,85,91,98,105],"input_token_ids_sha256":"46b16340d489ab9d5e99f73c42d4a7005f7436243eef5fc34fa92767754e2622","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,20,27,33,40,46,53,60,66,73,79,86,92,99,106],"target_token_ids_sha256":"761af83bc90049bcb9676e87032fb087e505ce54ccdace5bd3157cfe531b0e6f"},"formatted_content_sha256":"892bdd77965161d3f33ecae4371aaab245852c1a733575ea82a03f4c5162b6e9","generator_receipt_sha256":null,"identity_record_sha256":"096bf170f6323441d41cfdcd82e991e5330ce43bbca166fdaff8e2867cb3901f","prompt_token_ids_sha256":"9d1498f0e195703f3d0a9bab6ab2b95760b77762a386814e263445e614eebe8e","ruler_category":null,"seed":null,"selection_rank":116,"selection_sha256":"49bf4e9209616a951134062596b8e1762dbe5ae3d6768f1c37fc3ca1a831c8d8","sequence_length":107,"sequence_token_ids_sha256":"e9307774f416105649348d7b962aed8818afdd9670718378a8011aab53794a9f","source_content_sha256":"a4c7ff187da9b88bed24a5ce4e33b1d245e061cc2f7fb731157aeb3dfaf250ab","target_token_ids_sha256":"d3f2b417944e705153e271af9eeac8e255eb09fdbc9a8241cedc49106c05cb62","token_span":{"cache_exposed_start":107,"cache_exposed_stop":107,"prefill_start":0,"prefill_stop":82,"scored_start":82,"scored_stop":107},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4294762e8478186f4347187eec4203bd212ed4716abafee16d6c831b211a92d0","anchor_positions":[6,14,22,30,38,45,53,61,69,77,84,92,100,108,116,124],"anchor_positions_sha256":"fe7a10db15891cb54d23372b10af49697c0401f54d8f6b37c01417545e9ba930","canonical_id":"605","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,29,37,45,52,60,68,75,83,91,98,106,114,122],"fisher_boundary_sha256":"c098a5d59dd920acf5e6ecb372e0ff5df7d0b3ee1501bb011f699481cf66e206","horizon":1,"input_positions":[7,15,23,30,38,46,53,61,69,76,84,92,99,107,115,123],"input_token_ids_sha256":"faf2e5d622e9951baec003f38847af5d1319f255169b8c041a433d5b46952e3f","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,31,39,47,54,62,70,77,85,93,100,108,116,124],"target_token_ids_sha256":"26f6b36890856124d1f7f7a676b24ea8aeea3382d92cd1489d141c22982a041b"},"formatted_content_sha256":"80aa72fd69bd10f4c2e5403802378e6fb00d15a1e298252ded65c6ce760ab095","generator_receipt_sha256":null,"identity_record_sha256":"ee34d2c3691827aa084c190eddddebe9d1345888c7f48e25c87d60d765f6dfcc","prompt_token_ids_sha256":"9fd32241d4f2b36ee9f01d68e556fe7d2b96791f996bcb4bddb46b56225c1199","ruler_category":null,"seed":null,"selection_rank":117,"selection_sha256":"4c310c24967e920116a6713c985dc574fc27f51aa0e5f96102923b5848a2fea8","sequence_length":125,"sequence_token_ids_sha256":"d5c71f5ac8cca279a5091bc56ba019e4c7f2a97540c957843ce3658f1edad1ac","source_content_sha256":"dcf527a4f6986a9167a459248d6567bd87701d5fa34856a31f9be5ee02c91168","target_token_ids_sha256":"741ab7638f051c1d782ffdca7cafb68a673af7ec2773fdff58abe66e46d7eef1","token_span":{"cache_exposed_start":125,"cache_exposed_stop":125,"prefill_start":0,"prefill_stop":68,"scored_start":68,"scored_stop":125},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"05c8e3cbead3af43c5c1e6e8d87467ab8c03ec4648a4298153b82d936448e5ad","anchor_positions":[19,39,60,80,101,121,142,162,182,203,223,244,264,285,305,326],"anchor_positions_sha256":"f3e5839dc1785a2c00f87ad2ea0e816ae94270b3238251e62817f25d09a21cbc","canonical_id":"630","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[19,39,59,80,100,120,141,161,181,202,222,242,263,283,303,324],"fisher_boundary_sha256":"de76b71c7d9a5873e8f18d38d9bfe274b70bf5ccb7ad95749ffff39ac8269bce","horizon":1,"input_positions":[20,40,60,81,101,121,142,162,182,203,223,243,264,284,304,325],"input_token_ids_sha256":"b71f4ccff2a296d03b9b8fada60f1b7538853521101cb758ed05e1b6e41628d2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[21,41,61,82,102,122,143,163,183,204,224,244,265,285,305,326],"target_token_ids_sha256":"939dd654b199e687770731bddc84e26956b5c40203089d198687d9c12778ed94"},"formatted_content_sha256":"b39458254cec5c52b4b1ffb7f1dc476b0e99bab62c4d66d0998f147e9cd767cd","generator_receipt_sha256":null,"identity_record_sha256":"d20e005d97d9c08185e9a25884618ab3451741d9e00a6f7cf4842daa416fb1a1","prompt_token_ids_sha256":"92effc02a358ea378746b418ac301dfe5bacb980d7a8a9bfee9363f21a656396","ruler_category":null,"seed":null,"selection_rank":118,"selection_sha256":"4d4a3d035c7868292b96361ee3387aa0653c72bf82ff2ffbcdeee1c544f9b310","sequence_length":327,"sequence_token_ids_sha256":"11215c622904358795cd7771cb56cd3e68586e31dea00e30330fdece99fcd790","source_content_sha256":"d026e046adcf26a5522065e389d1e74a8d7ef963fd53e4f8db8952c3c3058226","target_token_ids_sha256":"59f027a42d76a9550f2a9d95174c87b9fd402df01c12dd6433c83e8bbb1834ec","token_span":{"cache_exposed_start":327,"cache_exposed_stop":327,"prefill_start":0,"prefill_stop":236,"scored_start":236,"scored_stop":327},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"865d6b59e048496b3bf7c4b287421ac71174d11943f8a146bf5f215f6e662bb5","anchor_positions":[7,16,24,33,41,50,58,67,76,84,93,101,110,118,127,136],"anchor_positions_sha256":"b394d1b64998869298fd2b292d4ab84e6faf0b5a40eeb811c668f2563bb4df64","canonical_id":"868","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,24,32,41,49,58,66,74,83,91,100,108,117,125,134],"fisher_boundary_sha256":"7f4687722337e0596bbc45f63cbf6b70e8988c03af12154439f27a9db973cb09","horizon":1,"input_positions":[8,16,25,33,42,50,59,67,75,84,92,101,109,118,126,135],"input_token_ids_sha256":"8d01b29f328a257d664952e75b1923e83d2777cbcd4b7dd6cee1855ba4bd0e23","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,26,34,43,51,60,68,76,85,93,102,110,119,127,136],"target_token_ids_sha256":"8b8ebd92cf550b5df42ff3360fbfe39d8efc120a7e745a701b2476dc01b5b305"},"formatted_content_sha256":"44d9a5afa4c9790a32cb254c2d7e1c8151d16636b7c10318c889176cec4bab1e","generator_receipt_sha256":null,"identity_record_sha256":"fa45bce52100c237b5694c601b0c6d6d61bafa11dc08e55e2816e2dc5931e145","prompt_token_ids_sha256":"b9f642149302b4aad7f0cd78c10cba633079e023b978bce75611464d15bbca52","ruler_category":null,"seed":null,"selection_rank":119,"selection_sha256":"4d7d9814d186699e85046419caf35e8bacfed3ab6650cbd36ef4705e2fe2367b","sequence_length":137,"sequence_token_ids_sha256":"b282f3d8b38678f37b26da12ae374fa98bd55719947e98f5670fa8a8530270c8","source_content_sha256":"eae1ffd3380c73bbc35bb8929b99e7549668c0ee1f54da998081ad923ebc2b98","target_token_ids_sha256":"92e8679c291f99696b3ac568510fbe994ddeef46b7ee37dd2252d2f7a9c4bf84","token_span":{"cache_exposed_start":137,"cache_exposed_stop":137,"prefill_start":0,"prefill_stop":78,"scored_start":78,"scored_stop":137},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3053356b37e0ec018499d9df09100d5d459ba083855ec3be6383459c76336c2f","anchor_positions":[18,37,57,76,95,115,134,154,173,192,212,231,250,270,289,309],"anchor_positions_sha256":"1d1fbb7874404570c3a9cce52a9bba801a787fd59ed00a041f9aca9922c13383","canonical_id":"902","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[18,37,56,76,95,114,133,153,172,191,210,230,249,268,287,307],"fisher_boundary_sha256":"b03addf9ce28714390d0c8aa72e4cf5bfc4d998e1e89125d619cc5de86bb2d27","horizon":1,"input_positions":[19,38,57,77,96,115,134,154,173,192,211,231,250,269,288,308],"input_token_ids_sha256":"a9d774c7e1c1f9d59bd7b757d0d21d9849e8eec8052afe6025037cde871559f7","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[20,39,58,78,97,116,135,155,174,193,212,232,251,270,289,309],"target_token_ids_sha256":"52cd56fb6fa5b1573d45b6aa37a594fb645c879bc5efa69d6edc441f4e348a6c"},"formatted_content_sha256":"5c1d9ec33b3a94931255d1220cce10d4ee097868c37884a8e19ae9f9a448966a","generator_receipt_sha256":null,"identity_record_sha256":"11c597e2cf580e07f77afc5041e62139cfccff5f5ab2c548565673fb0b500832","prompt_token_ids_sha256":"bf15bcbdbc30699f9cd0081bb00efafed21ea1abf4e5381da4e70c8a8ea2907e","ruler_category":null,"seed":null,"selection_rank":120,"selection_sha256":"4daa86751a582244f7d590b6ba730fc4e01e1d3eb676941a3552f1d490faba8f","sequence_length":310,"sequence_token_ids_sha256":"376f09bf804c7e1d57cc7e70ae46ecf6050c3c815f6a7cee4a95bd11199c01d1","source_content_sha256":"57ceedd084b1dac9a04f6d5be3b4a7e63c4474855a5cbb2577214912ffc07f8e","target_token_ids_sha256":"49502604cb2f2efcb1c5cee4993331ea6bb17ff438c88e61241bb8472ffbd75a","token_span":{"cache_exposed_start":310,"cache_exposed_stop":310,"prefill_start":0,"prefill_stop":278,"scored_start":278,"scored_stop":310},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"afd54f87610f85707a02f20f8b511f975ef944c51b61fa08f447b37ed40c6a78","anchor_positions":[6,13,20,28,35,42,49,57,64,71,78,86,93,100,107,115],"anchor_positions_sha256":"5af6cea1df262184ac4897d6d91d25605392e5fb08e41b5742c64172e9670d87","canonical_id":"955","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,13,20,27,34,41,48,56,63,70,77,84,91,98,105,113],"fisher_boundary_sha256":"1c62dce0ea57f80fb1a56a3ea511f2ca9f646f6413e8db3d99f5d99de6521811","horizon":1,"input_positions":[7,14,21,28,35,42,49,57,64,71,78,85,92,99,106,114],"input_token_ids_sha256":"ef01c4ac7ec6c6b186fdd4eb31a7f21ee8d3e4dc566355d91c713471887db336","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,15,22,29,36,43,50,58,65,72,79,86,93,100,107,115],"target_token_ids_sha256":"785091482f15a4ef0949a945686a1778f4de4cf07b8104861aa31a9db7b18f4a"},"formatted_content_sha256":"ef1384fcf9c059d64323dd5fa9880b686c5b7ea953ce70d245534bb1c6ed7137","generator_receipt_sha256":null,"identity_record_sha256":"f520632d9fe13e3b9ffeebdc9ea540d828dee798f54926584e9872d0fe7b87da","prompt_token_ids_sha256":"010438e2e3ae3ee5626141c874feb0f74e50fca32fd27c7d8e49a07e790ec560","ruler_category":null,"seed":null,"selection_rank":121,"selection_sha256":"4dd5c5d56c9d92dfa81aec10aa5beae38c2061e96e799b813b238234a0ce8e4a","sequence_length":116,"sequence_token_ids_sha256":"fcc6077ed67145358daf5bcf3eb1ece0e289f68f139666cb89192eec34ca4386","source_content_sha256":"8408dff25eaf845d972ce9ae510b9d76eaa3e7a6bcfb8b8c7a628d70d2dd6cf6","target_token_ids_sha256":"76411fdcb68c0c051819b5bc2c0fed5b4048dcb9000413fb60e5de16d537d568","token_span":{"cache_exposed_start":116,"cache_exposed_stop":116,"prefill_start":0,"prefill_stop":72,"scored_start":72,"scored_stop":116},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"6bfb1e634cbb65a8f9c6d30dcee64be9d19b2bf1774c4fa850c4b7b5ce5abb9f","anchor_positions":[6,13,21,28,35,43,50,58,65,72,80,87,94,102,109,117],"anchor_positions_sha256":"f40be3d1cf7064d7e86e3a810371d8f2d368583e1dcff4660d5d02fc4a37f43e","canonical_id":"640","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,13,20,28,35,42,49,57,64,71,78,86,93,100,107,115],"fisher_boundary_sha256":"261088f8bd3fe25be23146357491bf2699628f4ce493685b028cc1ddbac01eb3","horizon":1,"input_positions":[7,14,21,29,36,43,50,58,65,72,79,87,94,101,108,116],"input_token_ids_sha256":"db9064f4846367e8570c2c84fe797b835d18960e9211681a5e0daac0654b2347","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,15,22,30,37,44,51,59,66,73,80,88,95,102,109,117],"target_token_ids_sha256":"0a9ab2151f96a33137ea1ad001a0d76f499d390891517e6c83bd2ac4188c1207"},"formatted_content_sha256":"06826303346d9d86d61fc5abbffb92ebac62b2e090a9f70b82927eb026441a29","generator_receipt_sha256":null,"identity_record_sha256":"7b2ff5b497e0a82c3b0d260a880af1a82e4a4c08c8185dd3ce186597015d3a20","prompt_token_ids_sha256":"05a2a260c33e6e500d66729f3e56aa8f9ccd92ecf9825c6449d5f8fe340e67a0","ruler_category":null,"seed":null,"selection_rank":122,"selection_sha256":"4ddb9210eb7ae0fdab8792259d081bdba6eae451141c47a811a0245cec7ee62d","sequence_length":118,"sequence_token_ids_sha256":"20509fd474053e85eba21ed705bd626c6c6f337445f4daf2177bf8b506211954","source_content_sha256":"6438ac844fd844aa7880ed46075764fa684488592b7d562f4ae65218f8b35da2","target_token_ids_sha256":"fd3de534e5c7f8e743fbfaeaa6eb5d1a4c3d2acfdfd4a83cb4e9f6b0a9383173","token_span":{"cache_exposed_start":118,"cache_exposed_stop":118,"prefill_start":0,"prefill_stop":85,"scored_start":85,"scored_stop":118},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"f5ff8bcd127367dd786bcc2b0c253bbddf5577a9692a04a945dfd7611e658813","anchor_positions":[7,15,23,31,39,48,56,64,72,80,89,97,105,113,121,130],"anchor_positions_sha256":"4b59ffbfdc6171d05db7e4ce13fd81c13fb1c2704ef54cbfaede26eafe2b1382","canonical_id":"602","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"fisher_boundary_sha256":"4366416cc3edb9f1e2ee5e0a55178de659ab8d05d59a8cbbce27f688cce7b2e6","horizon":1,"input_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,129],"input_token_ids_sha256":"2db46a104b4295959d6a16c851488604e471fef800ab5243426a1d23cce719c5","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,33,41,49,57,65,73,81,89,97,105,113,121,130],"target_token_ids_sha256":"2b0416f82f51105c2f972ed61b19f300631a81a49326605f7f5bf45fc3d5957e"},"formatted_content_sha256":"48559307e617b830c4e7a2ef8b20df2f6a53a7ddd15f766c7415c7b8597ff2f9","generator_receipt_sha256":null,"identity_record_sha256":"baa5a63154617acc29a3b5192d3459d2edae4c0959c00d24e2dadeeb3837d757","prompt_token_ids_sha256":"0bb478e0d8162f79fa8ab3da8baed5b87413927171e2f2c8071c49ab30da5521","ruler_category":null,"seed":null,"selection_rank":123,"selection_sha256":"4e02d4ed1f0c6d170eb518cc4020d82ff4ab1bd18d4949c5913e319fa7cc9e83","sequence_length":131,"sequence_token_ids_sha256":"e7b30f61231d01e87b3afa4dee3faf2008c8e20ffd1b3a478e7a94932567aac3","source_content_sha256":"59b47992e5e199dff2a5b0af7ea0a02713ea803a5dcc6fc3db6ecda2c4879288","target_token_ids_sha256":"78bd9c18ec9dfb40ae35c333bd6dd45162c4cc291b415b84d5ae2a87dd1ea334","token_span":{"cache_exposed_start":131,"cache_exposed_stop":131,"prefill_start":0,"prefill_stop":86,"scored_start":86,"scored_stop":131},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"bf2d404007f3e1af973b816cfc665fdfc8257f337deb35322ac0077058cea4d9","anchor_positions":[12,26,40,53,67,81,94,108,122,135,149,163,176,190,204,218],"anchor_positions_sha256":"34e0f2438d49fed8c871a33d3818a40fc035b239ea8639ec66d64d1e59a9bb66","canonical_id":"712","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[12,26,39,53,66,80,93,107,121,134,148,161,175,188,202,216],"fisher_boundary_sha256":"957de9da750e31aa4980cb34b1fb160bb11d67e08faf67af18d3cd3e5289f959","horizon":1,"input_positions":[13,27,40,54,67,81,94,108,122,135,149,162,176,189,203,217],"input_token_ids_sha256":"f56a9ec29643c9eba2eb44791cb09e84d9f2b231c902427ad5756e74ec534316","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[14,28,41,55,68,82,95,109,123,136,150,163,177,190,204,218],"target_token_ids_sha256":"c0a62a7a97676edbafdec6adb2cfe3c67719d5ccdcfcc3a0ddba0c6331624ede"},"formatted_content_sha256":"b47cde36a934283ca86c0366ca05754f6267c0fa319aa8c9ab18bc5ff0e5d9b5","generator_receipt_sha256":null,"identity_record_sha256":"4cc5c35280b65e6893e928c4b6a5b2caa568e303f0840d0df3438ecc6b258676","prompt_token_ids_sha256":"4ef7f3aa586f42e443494b0dc4987d24f11b6cc50acb5df627b135819de32092","ruler_category":null,"seed":null,"selection_rank":124,"selection_sha256":"4e132dddf48254f73a1575b1a311842bc8036e6c5310eeac68d27fcfa9705827","sequence_length":219,"sequence_token_ids_sha256":"45c3e814d2308b690e19384d22ca303456c689d9292a9292a9974a96d2c27dfb","source_content_sha256":"5355ae6f3f743535c93a53ce5c0f05092f5c9e89585eeb219a5a424a29b6a20e","target_token_ids_sha256":"8c478cb16206b569ca9079b27faaad0f0bf99e2182d4dd8ff62cd8328afac5ca","token_span":{"cache_exposed_start":219,"cache_exposed_stop":219,"prefill_start":0,"prefill_stop":183,"scored_start":183,"scored_stop":219},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"36c3b31211a995bad7f56d142b515472a78c8c0bab6bb376f23f53c90f2b7139","anchor_positions":[8,17,26,35,44,53,62,71,80,89,98,107,116,125,134,143],"anchor_positions_sha256":"486e7f47d72e2b3d4a862156a2b41e9dbb05a70d78349fdcac0b01e09667750a","canonical_id":"606","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,25,34,43,52,61,70,78,87,96,105,114,123,132,141],"fisher_boundary_sha256":"bcfe34f783ad5a75f3e21e034bca0957d966c976fd05656202a26e596ce4d242","horizon":1,"input_positions":[8,17,26,35,44,53,62,71,79,88,97,106,115,124,133,142],"input_token_ids_sha256":"2b725b30a4b9b4a7e69480982037a0a07c57e454ad8e62b46c0f0d34a3b1774a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,27,36,45,54,63,72,80,89,98,107,116,125,134,143],"target_token_ids_sha256":"872de4a20f2b57da5f4caff92476bd417ca5b3e9d701ef7d101a9fd65f42abed"},"formatted_content_sha256":"c6fed4dd2cb64cca6e833bf23797bb7c1d862a9c875ccc65acb9b0897618cc79","generator_receipt_sha256":null,"identity_record_sha256":"a9e7062d3f002e0a39343fd4f410be18e7409cd9643a09c6dedbf4bf79332f30","prompt_token_ids_sha256":"67adef290bf71e36a6af78dc46a182ea33221b75714d4b89977dfbfb25d08ad7","ruler_category":null,"seed":null,"selection_rank":125,"selection_sha256":"4e211b7dc81626479060883502f3e73d8b09c6e4da35910a1305a8e23d6e76a7","sequence_length":144,"sequence_token_ids_sha256":"e35fd3a90e9e66af3e142095f74dbbe6c47e3a63d75cede17749f5f6d49e6dd0","source_content_sha256":"3a119437b4f93db7da36fbe2d787607dde294a4a838f8b29d327bc68a0eb0478","target_token_ids_sha256":"764d32b0ec71bb9f1d078ccc8e4a39c3ce7dd359fa503899ebfcef2b1653ed9f","token_span":{"cache_exposed_start":144,"cache_exposed_stop":144,"prefill_start":0,"prefill_stop":117,"scored_start":117,"scored_stop":144},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7d1f918958bc691d31836ac50e195b86b748a5336a8a243a1b427eb6b289303d","anchor_positions":[9,20,30,41,51,62,72,83,94,104,115,125,136,146,157,168],"anchor_positions_sha256":"a040c10bf4081d1ff5f0840fe94899e3d5f4cc8b3513ac78d8caa23e146870a7","canonical_id":"656","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,30,40,51,61,72,82,92,103,113,124,134,145,155,166],"fisher_boundary_sha256":"c9142db1a8573b215599b5dc489eb1c6d88615d3cbf8ad1c104742ef7bfc5041","horizon":1,"input_positions":[10,20,31,41,52,62,73,83,93,104,114,125,135,146,156,167],"input_token_ids_sha256":"c32bee20d7de9784fc14e7f94e2b1e165c75b8c09339f3ecc44be1cbad01f8e1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,32,42,53,63,74,84,94,105,115,126,136,147,157,168],"target_token_ids_sha256":"abafaed5c5aabca6d515ea1a705691d431f604b08089f1f0b25c6df8b45c8305"},"formatted_content_sha256":"374f6e67c0234e75d236ea8f2069dcdf8a631ce916aaed870d7810e3347cde44","generator_receipt_sha256":null,"identity_record_sha256":"770edcf1d6f1f10e0e3b4c54ae7a7e8a532a07b1a3d17003f426dc095bed8679","prompt_token_ids_sha256":"b45a43c9c890b88c42ea8fe0a8f0a5d71c55ec6f01a772f30a952fad7f8431f5","ruler_category":null,"seed":null,"selection_rank":126,"selection_sha256":"4e7a2318f515292e8fc2e6b77f63c13218916891ac0b3f71650ab59f87fc1a5c","sequence_length":169,"sequence_token_ids_sha256":"c61b8bdf63e3f2d95107fee666460e20eb1e4a5d5420d41d523360fc8916c990","source_content_sha256":"7164e42f57848f0ec61d9fcc2a78bec0f09e6a54082574820bd69a7607d84721","target_token_ids_sha256":"52ef9360d90d7b7b754a31958af231bec9dfcb171dd416721cfb316918352b68","token_span":{"cache_exposed_start":169,"cache_exposed_stop":169,"prefill_start":0,"prefill_stop":118,"scored_start":118,"scored_stop":169},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9613ae532d1ac6180936e714bd20e0522fe6966715f1926f3a3b8b7af31b80be","anchor_positions":[11,23,35,47,59,71,83,95,107,119,131,143,155,167,179,192],"anchor_positions_sha256":"97c036645b9b6a78e53e848879a7e3ee08a70ff938ed6e98aa7098b21ab4319c","canonical_id":"671","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[10,22,34,46,58,70,82,94,106,118,130,142,154,166,178,190],"fisher_boundary_sha256":"730eb3d8f69aea1c1666e7307d73b83daf05f241bec613d1628b92a50ec3bcaf","horizon":1,"input_positions":[11,23,35,47,59,71,83,95,107,119,131,143,155,167,179,191],"input_token_ids_sha256":"1100089821e0f1740660a914b8b15a1db0534566177c6e09bcf34f670905693c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[12,24,36,48,60,72,84,96,108,120,132,144,156,168,180,192],"target_token_ids_sha256":"8d49d123f2607a365571c71836e7acda4676f3df261e0a4254afd127a53299ec"},"formatted_content_sha256":"22bd9c0d68e48a76c31dfe254036ce6008c9abd1f71c4c290e76fb4135d09ad0","generator_receipt_sha256":null,"identity_record_sha256":"4cb68a3b9440b00f53a5e655ff7024c46d2ecdc277ae03326b75f41268b173cd","prompt_token_ids_sha256":"63bd2370a08ace97ff440307492d1bee8479e8e14cd40ba9b40b1487a9ad53fa","ruler_category":null,"seed":null,"selection_rank":127,"selection_sha256":"4f4ba031f801028fbf78f774438c1e6ef3c15a58f65dccc230632c90a7fe6b5b","sequence_length":193,"sequence_token_ids_sha256":"d653c8bb051679b00cd66c5370bff1e217b990629e9ca260192942d1541dc892","source_content_sha256":"85295da947527bd6a8cb51dbe0f81aed64a26a245e12c289dab42daa0830ecdf","target_token_ids_sha256":"5233a54b205f9eef3872026a1b044403750ba40d368571e658bd36b909a1a85d","token_span":{"cache_exposed_start":193,"cache_exposed_stop":193,"prefill_start":0,"prefill_stop":89,"scored_start":89,"scored_stop":193},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"797956c3bb60ce0f54962ed315c69368b4d9168e36abfd675d14ccda36e8b2fc","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/36808","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"7e362e252ec4500305135b0f2ef134e2fe129c787fb0a31326c35439cc9ff4fc","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"d0e5311a6d6bde4f201973a4428fa08ba4fa97dc6cf2580fce9e2a4604922291","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"5b2da07174c560f02d4940468e335ad6777df3f471b2be9a3550cae36b1fd62d"},"formatted_content_sha256":"f6e3c61d5723ac6698db28d6198db5b270f51e83e2c8800f5ab2cd181a993a21","generator_receipt_sha256":null,"identity_record_sha256":"e7274663db91b5411d8af35ca6979c6b6038b747658e034486c7161db217a12d","prompt_token_ids_sha256":"5233822ab87654bb7050a9eff9f77a58c8ca4a74c176420316866641587b5ac1","ruler_category":null,"seed":null,"selection_rank":0,"selection_sha256":"0000d64fc1c2791eedccba1b0a014ae30d36a69ee95be69a7d2dc5541d9ff914","sequence_length":2304,"sequence_token_ids_sha256":"5233822ab87654bb7050a9eff9f77a58c8ca4a74c176420316866641587b5ac1","source_content_sha256":"ec7b13f2bb2766533058782bbf50f269e8c0743722581ba193e8dcf47008ada9","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"5ca98bf769dab4ebd4692e9fbb560efcc0f0f474c5a49e0732d605a2683da881","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/32024","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"2b3d247e247806b4dc881d8599ac43f4212a4abfdf27c4a4984cc7b6e286bcde","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"231119f3847a33472d36eeacd144d75fbf1c16d0971ecebd977dec5f84685ed7","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"bcf8ceb1b07052b45eaec59a1762d47a3fd14770e2014af6c7462da3ae912a8b"},"formatted_content_sha256":"056228535dc5d8d08c833c37eb05eb57a524d567c7137e2d8314c181aeca5137","generator_receipt_sha256":null,"identity_record_sha256":"48c93886d09d16bbd4b1c300f10b64a7a5908445969fd866133d7dbd65c13133","prompt_token_ids_sha256":"20d38634a5307135f41953611e0e9d2a8e22a29a1f912d78af0414bcca3e0dc2","ruler_category":null,"seed":null,"selection_rank":1,"selection_sha256":"0002ed283941f3da5106f697d697735d5064de6f22aa84e512fd5fb4792c49a8","sequence_length":2304,"sequence_token_ids_sha256":"20d38634a5307135f41953611e0e9d2a8e22a29a1f912d78af0414bcca3e0dc2","source_content_sha256":"eb06d835db68d2ccee63052eac9d025e120ca9331c84186892dbfca0b8c7b5e9","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"870cf6deead4c720c1131fda048c46f654bb50af3e403058e9370e06d21978fa","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/20279","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"9a2ec9cafe47a0a911539d7cf75996655cafccb2071b17063c4d74a9a215b6bf","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"b6c9a50b23af8aa52a00688ce9f3e2f1b86afb30dba305513f107d56499b7109","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"484ad2459b1ac7e91b8a406d092192990e1eb1f2ef4295cbc9ae9b0d8db832fe"},"formatted_content_sha256":"ad22fff4026303176b09131a60d9a8a6d1a6385ccaa59768e5f6a72ff8117f2b","generator_receipt_sha256":null,"identity_record_sha256":"01c99745bb7d41ee57ea15450fc853f25c79f09d2f8bc063c0bdd3bb66dcc3da","prompt_token_ids_sha256":"aa29883de5d02657b8eddaf1e1caece900a2d6c56fcf63a868f35f285132dee9","ruler_category":null,"seed":null,"selection_rank":2,"selection_sha256":"000d6a7d93865e2955a912d7fcafdf0e79cc24fdca9b059114ada982813dedf6","sequence_length":2304,"sequence_token_ids_sha256":"aa29883de5d02657b8eddaf1e1caece900a2d6c56fcf63a868f35f285132dee9","source_content_sha256":"b899a1b6c5b3ba61252a07cb53ea7d0f15f0c63315df7b872a9c7cacc58017fd","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"c3a77259d6b11a251eaddefe696ec8873dac79b38e498e4c19008e424264366d","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/35786","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"049cec2caddadd6c5a22d5eebe0f37af9a7d660e1f8a0134c85404ef60e34076","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"34af22d248242c1ad9044c3b1cc5988e257371a5867a4bc4f686e53596cf32a7","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"e400ca0c9462a4d8e94a16cb560f6a54f948fead3ee81377ae34f4b435d27bf6"},"formatted_content_sha256":"7e0c728729a3f6e258bf44c314097e48e5bd7a3d2ed6a9f7c33b0dc333de0830","generator_receipt_sha256":null,"identity_record_sha256":"182e8520c6e45d8da1ae7d54afd83292ddcbb68a76cccbeb96d22ffdde1b9464","prompt_token_ids_sha256":"a074a6e0e6624af64b673167b490ce4de6ff1fc5bc6a5a0429a482706d5047aa","ruler_category":null,"seed":null,"selection_rank":3,"selection_sha256":"000f8dbba0aa87b15dcf3bb9e900e6a1c95b96affc23284c0aad2871def7d41d","sequence_length":2304,"sequence_token_ids_sha256":"a074a6e0e6624af64b673167b490ce4de6ff1fc5bc6a5a0429a482706d5047aa","source_content_sha256":"d99d106567585d3e0dbdbd545794876258507c655b4b92af1dd705f6e4c984ce","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"6e96a1becbdeb74006e814bfe6a5fb240a5de9f6134d609b13117dbe7211fb78","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/29413","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"f68c33e180124100446c92def52974f274452469befeff1421363223dfd67323","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"9c4a80f3b25c53840a44de74f52c06907a16b90bcc7a8c077dc4eea1fccba752","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"85a930c5f1ed93345b9259991c2d9c7a90faace00b3ca7248e61b28fbbf27372"},"formatted_content_sha256":"aa37e0852c2a9d938e44d1f6050bbe8db2ca60ee6d89c2e40ed5ddc5f9c36a77","generator_receipt_sha256":null,"identity_record_sha256":"c8f11e6e9703b0eb65e3a6f5f79135730de277373c9e3a94e68eab94f5054d7b","prompt_token_ids_sha256":"5f2026b82faac8988768fc7330d60a392741c3a5f7b364d0b87bf8d596e74bdb","ruler_category":null,"seed":null,"selection_rank":4,"selection_sha256":"001145473f2b3b765745639cd14dac4b421dffa50f63df360f9059d362dc9b5a","sequence_length":2304,"sequence_token_ids_sha256":"5f2026b82faac8988768fc7330d60a392741c3a5f7b364d0b87bf8d596e74bdb","source_content_sha256":"12b0ce7832facf193492ad1f568ff0a0b25bc5e52381a9690df089bc8a9b1db8","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0244917a01282995b95e9ac9304e280f4253bfbab1626b6c1c57bf69a558b519","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/24903","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"d50289a8f2b5eec9977321d118785e94090fabd5a2f4938b5e6f1c63045b237e","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"ee472d2e8db03638388106e721863599f643316a3422a5245d0628a1edb69010","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"4b9781c37044a00962c0018332d1f3f966c9e3b9ff49f267b124cbe927314707"},"formatted_content_sha256":"32ac15efb7073af0ae11d3032d1da75babff6e7bd6406334879047edbd6e821a","generator_receipt_sha256":null,"identity_record_sha256":"b26e48a4638561a3f7f57dc2f6915accfeabce22cb26f87ddd65919ddf7a653f","prompt_token_ids_sha256":"7b786394510a818db1c521f6d71eb3b4052359b0d28d0a57d166057d80326905","ruler_category":null,"seed":null,"selection_rank":5,"selection_sha256":"001280ff6af9eba470ab76ff48bc2983cffe1995efb051d68cffd40cbaac4d3e","sequence_length":2304,"sequence_token_ids_sha256":"7b786394510a818db1c521f6d71eb3b4052359b0d28d0a57d166057d80326905","source_content_sha256":"14074cbd779ec8e06e0e5e91c6c6e6c02ebe2d185bb5310ab0dfdf64230019d7","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"67b1ede620a9bdb86c619e3124eef21332be86df3795cb4169fdf93c1bb4c46c","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/10609","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"51f6c6102f8fa772b2f8e1665ab91ad76a5f4f11205009bb3a8af2e9c9d4c06e","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"3d06c4debf0e8c528b85d65d243bbe561bcd13afdcb83d09b2d604fed1853965","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"9bdcf5cea67426ee4ade841fed2e6833ccbede63b7b5497449fa2279f5e54182"},"formatted_content_sha256":"8639254a29be3154bbc110c673738678316be09fe6f456878dac249bdeb1f4e0","generator_receipt_sha256":null,"identity_record_sha256":"a21cb3bd2e34a22b91569c2942d19d8bafaed2f11827ee9c58af3cbe366d479a","prompt_token_ids_sha256":"0697f0eea8df7d85bbeb6a688ab15d4294212694298ccd864b429ddac3ed7a4e","ruler_category":null,"seed":null,"selection_rank":6,"selection_sha256":"00131239a626272348186654735088a9b4c813c3bd83827262f7891e6056d43f","sequence_length":2304,"sequence_token_ids_sha256":"0697f0eea8df7d85bbeb6a688ab15d4294212694298ccd864b429ddac3ed7a4e","source_content_sha256":"49ff1f749aab76c1df7c47e6c79fdee9dd3fd191d073320f0ab1f5f361059fc3","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0d6376f0009c55e4f4efbf0d650cfe974856c049c8b14c303c991dd3f6c05c19","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/39447","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"7914f39477c00d3c1516c2e43c441114314ef252fe82526ad293b346ab359284","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"7d6865c318074116ceae5d7735c27fed5066ae326d49cbc3280f9f0289cfb7cc","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"ec95c049c76add2ddc5fa9d5301e2bddb64906cc8e42e8438a5c2aeeffb46499"},"formatted_content_sha256":"033b6ca5d9572a314560961826373bc34657c08759a1919296e81a4cb84c6471","generator_receipt_sha256":null,"identity_record_sha256":"f59cda6a20ec53af40493bf6be8ad0f898023c32531b81fe0ff0fb3b13340ea9","prompt_token_ids_sha256":"81890a7b8319487c5f4c312ca211202afd2d039e46e4931b65e09c6e446fa634","ruler_category":null,"seed":null,"selection_rank":7,"selection_sha256":"00166693ce38f4c5185c40c1258a1a602a290506112c351fa067daa039c2d960","sequence_length":2304,"sequence_token_ids_sha256":"81890a7b8319487c5f4c312ca211202afd2d039e46e4931b65e09c6e446fa634","source_content_sha256":"2d19f7657219986df6eb9bcf188544ab7660f95cb5521668082b48da38ac52bf","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"6f5f79cc3c1180f2cf00501aeafbc8d0f1dd6e6e7ccc69a8648c032e3469df45","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/33622","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"9b4c1f8e6fd129beb4bc474e6514815a95af657b4e2bc5fe7a7f32362dfb40bb","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"dcd27f6003c74d9c8a85711934161ee0f2636d2f0d27c4137e156f0d5064649e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"536e8bc4cb2331a406028d335233022e273cb3c563ac84e3db83762692164b14"},"formatted_content_sha256":"de944eeb38c1a6f2dfea1cb1657adb820f3d66ebe3aa361c72a39823aad3fa73","generator_receipt_sha256":null,"identity_record_sha256":"344ad881a15a994684e1ad39c738fc64b472325c7c46c63c837f255455d9f5e0","prompt_token_ids_sha256":"93be74d13c5df9550e8923cb9af9086891766e59060bc9997c00a1c14458302a","ruler_category":null,"seed":null,"selection_rank":8,"selection_sha256":"00191187cdc6cab1097f558bd88f30baeb7d1f08e9eaffd47828d8a1345ed53b","sequence_length":2304,"sequence_token_ids_sha256":"93be74d13c5df9550e8923cb9af9086891766e59060bc9997c00a1c14458302a","source_content_sha256":"dcf794cb44d57515b528594025b879dcacdc8e06e892b1a0e62d26319aa8d8d7","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"f602f879f02588bab48170a79cd2e407d35c19e5a4e05337a2615b62c1d0c64f","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/28065","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"3ab0b82eda5a33b4272654c9de3e352bda591b27e5499e28d3642e5a138ec7f8","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"f29080ea5125a83d114344a4897dcb5e6cf6135a3d28a1b55374cdb2836dd1db","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"34dd2e61b1ba68a0bd87b1f9e8650c6869d66f35aa98c549feab516d17c3afd8"},"formatted_content_sha256":"f500fe5cd15c60256f64965b23d5784b6badce6cc0da7c4792ac3017524b173a","generator_receipt_sha256":null,"identity_record_sha256":"73be35853dc6427eeab216a511cbdeba7dbfbab63feb81bd2642b1d99bbe5f6e","prompt_token_ids_sha256":"1c701cd7ff6691ab6d74de0874349dc37eac3d7dd6450cdb43b876a5aa7ea76b","ruler_category":null,"seed":null,"selection_rank":9,"selection_sha256":"001ac4d7e72067b598bc36c7582913c54df342db338498dd73a106eec9c8ffb6","sequence_length":2304,"sequence_token_ids_sha256":"1c701cd7ff6691ab6d74de0874349dc37eac3d7dd6450cdb43b876a5aa7ea76b","source_content_sha256":"0beeb0285771e4bc2cee42c31f1b35d1570cda724294b426b193cdac54b83393","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0d2186f26771b127e5a2350edc81554ef21f6eb9da3e69d77f7377a307b8a2ec","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/1253","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"4db1f2865c58e3a6faa8055be8d46b6c3f57be277f07fb5e39670a879cc599df","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"e20502727d9945c19d9f78c314ab8a1be19622e5a4316dd86e9823225d38135d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"be880e0cd3b16a5b9d737b8b2b7cd71353c9aad69f7dd469e024c5803d69f411"},"formatted_content_sha256":"a6e5b442be8b8fd1b45487216940754b214f379ae9a2021d30f82598049ca5f2","generator_receipt_sha256":null,"identity_record_sha256":"02492d4c2ba2a4f9435fd461ed91ea404250adb616636b5c5f6930fc4c350595","prompt_token_ids_sha256":"c55c7896671653ced092d5fc45ea5527e9457780b57375667e77dd586f161da4","ruler_category":null,"seed":null,"selection_rank":10,"selection_sha256":"00223684c9261647b075a573c94999d1d273100f31524ec62c5e6c73b4787787","sequence_length":2304,"sequence_token_ids_sha256":"c55c7896671653ced092d5fc45ea5527e9457780b57375667e77dd586f161da4","source_content_sha256":"6d1b9548166f4f9d82806a3af5ceb3ef3a1f644c2c946deb8aa86ec78d2c2fe5","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2608f932273743963113f1c10bf0e1c74703b9252284d4ab16394eea6ee00a1e","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/17774","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"ea92971f2ed12679772d304c1af6ce6e5d20a0a07f9c6e29ab3b8b95eb0c9c7a","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"de2670faad91b518e2bdc4f93edd92e3cf3b6d9e12fe24803a9dbb2d6ee7d8c3","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"d1ca5e91572227a6a1cdbeac2c3e0023a33d11172238246640c98d613aeb61a8"},"formatted_content_sha256":"2a4bbc348f9010bdfc1eec2d9ac77a487ad6f51468cfdedc35c10316e95564da","generator_receipt_sha256":null,"identity_record_sha256":"aba94b34901cf76209e4f0517738794062a4e8978f75d8d8e46bd007b32f2d61","prompt_token_ids_sha256":"25656de4135fa58df4d27bf5e1bb9122210cd21754c03b8b5f83f9412a6e2f5e","ruler_category":null,"seed":null,"selection_rank":11,"selection_sha256":"002494cd7b130c2e8101df6704ee2628630640a6ea6d8c21e9a0419c810d8f25","sequence_length":2304,"sequence_token_ids_sha256":"25656de4135fa58df4d27bf5e1bb9122210cd21754c03b8b5f83f9412a6e2f5e","source_content_sha256":"c71f6b5999e91ecf1857fb9dfb3f2573827e4ae4c5c3c4af2b08e3ba6a349e07","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"112223ea4e8c7a17eec0768b0bef76c663105f97785440668c062c5080b3859e","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/26008","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"2eb3fc663ac2fcca42c750d5cba63ad3d0d118921b38a48382c1cac4734faa8b","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"ecda06dc336cf17705f48dcde65a60f0b7ff432af083c07918dd0f1dbf09d79c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"e7eb1385302da9031a1227b768aaabecafafdb5614b3a0e2873e45ccfa3f7325"},"formatted_content_sha256":"fb608858dedf19c2f0a8e14d8710aa1d1c393be5a72ac8554731f0af33714904","generator_receipt_sha256":null,"identity_record_sha256":"af695a09abd9495d4e87989cbe7b352aebdfea88272863b3cade1adbd9d9c1a1","prompt_token_ids_sha256":"8eac346c569d9163ec8a9bde2355446690ca3c72a9a9f5c9bc00b745b6648f63","ruler_category":null,"seed":null,"selection_rank":12,"selection_sha256":"0027603c9d271c580c43673b3431be3adf061650ddff27e01b7869ea18673d9b","sequence_length":2304,"sequence_token_ids_sha256":"8eac346c569d9163ec8a9bde2355446690ca3c72a9a9f5c9bc00b745b6648f63","source_content_sha256":"469e817fab0a0502150988f96b4b16d5c4de03fb7f7bd185e1cff0715aee4b26","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"c447c8e41acc4393a51daf3ef28ed95971628266812cc78c05b887e341a38f02","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/38160","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"849f6474e4eb97b56439c66b9aaf2ac1fca275e49c29e33c48b2de0e806d1193","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"d11cb3a40f35a5034111c7bbbf1a8ff895d15396fea10ddb0cdbd97e8a47a5bb","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"ae53490b8b58ad6da63a300819db55d4a14e7b1bf25c07bbcd54b06469cc8b62"},"formatted_content_sha256":"b87156fd6d95b79ffe144b1b7448682cc84e8c2d06709db21214d82439b58315","generator_receipt_sha256":null,"identity_record_sha256":"f629ef55f22acb269a1825e56238e5770bf1bffc327d1d71d03370911d984632","prompt_token_ids_sha256":"55fb1408407a280bc2c65e78161f5cefb0ceaa4ceb04073817e0973d6a982b53","ruler_category":null,"seed":null,"selection_rank":13,"selection_sha256":"002a54c1b2c80d534795a77a68dfa9f3411107c3896ba33d31a868339d813293","sequence_length":2304,"sequence_token_ids_sha256":"55fb1408407a280bc2c65e78161f5cefb0ceaa4ceb04073817e0973d6a982b53","source_content_sha256":"f59885bfc6d88898f97c7bda7625006b3be0651da5929a1e272907e85a9772e3","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"14e443d1ba9ca076a4a9d065c90f0b3ccb2c24defaced1591fe7cc77f71e7527","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/2773","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"5a0b47a49833f2e6114f9b3e2d475b69a889ed7945db09507580c9536ef76448","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"f0dbd8116db99645f9271fcd040a856c59b111c6243748f64ddc1422928c64ff","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"78a78a32fdab2f986b289b32d0a4612fb33ea2014dccbbce3a60d65c16e765ac"},"formatted_content_sha256":"68c11a24ba674cdac7966cbaf0f71a9d2fc30a62a0d90a649a6554907ef00125","generator_receipt_sha256":null,"identity_record_sha256":"ddbf6728a3fc2c07defaa4546c34ea401f4fd05c829819301903562cd04a9519","prompt_token_ids_sha256":"8b935a7676a8137367066b0b1af7a39f635e0e3592fe3c1982a28bac24eb96cc","ruler_category":null,"seed":null,"selection_rank":14,"selection_sha256":"002d0ca3ff81dfc79dea1f54c4f28547f367149283e13dc83a14c114322353bd","sequence_length":2304,"sequence_token_ids_sha256":"8b935a7676a8137367066b0b1af7a39f635e0e3592fe3c1982a28bac24eb96cc","source_content_sha256":"be1775efe2cd9be13c65cbd65200b0cd27ae7d29b52ff493b27ed85e4b549c61","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1fb610bd6210663f10a294d4b45df223b7ee4e3b6a8c0de390f3e50612ea1219","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/1219","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"92e822869a8cdca6f8c09e9495406404c8a345c0a99d06dc98a948b7bcd010d6","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"7ea3f80cfbc818f9210b87b0153e90af5720869a1fe3f26b0d5f723bfa9eead2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"c1725ca6a384a182c8f164ec35f4faded719127f4cef2baec3d812545eff3e6f"},"formatted_content_sha256":"cd30711364e668a3275113e8a3f92ecee4b268935f5a095cdb0f0c1f11205c1c","generator_receipt_sha256":null,"identity_record_sha256":"2d205b87fc59eabeed0bda29df3e0789eef230026c46ad11796d41b866d18b2e","prompt_token_ids_sha256":"65e443512311bce75a506b694195aa105b7831541ecb92c1768fe5a0ce285ef2","ruler_category":null,"seed":null,"selection_rank":15,"selection_sha256":"0032c753604107e647b4ccd7ef877562bc0ca9c0e232a26a761388befd002fea","sequence_length":2304,"sequence_token_ids_sha256":"65e443512311bce75a506b694195aa105b7831541ecb92c1768fe5a0ce285ef2","source_content_sha256":"6cb2a106560ad166c025fb7a83708e519f95bca68cecb9a7bff61e261a0b8384","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"52b78dddc7c06fdc5e5ab3937802742ef3b3414f9743427d2bd71994c5cb1671","anchor_positions":[113,227,341,456,570,684,798,913,1027,1141,1255,1370,1484,1598,1712,1827],"anchor_positions_sha256":"9289ece2ebec2ce8ec20865a50beecbecea8ea6a3dc4b5c5c8ed9f6ad0d784ac","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multikey_2:length=2048:seed=12340:sample=0","config":"niah_multikey_2","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[113,227,341,455,569,683,797,912,1026,1140,1254,1368,1482,1596,1710,1825],"fisher_boundary_sha256":"a9016c04cfbd31eb6f72106eb0693c1545bbaa0451e228e6e447a7814d921eb7","horizon":1,"input_positions":[114,228,342,456,570,684,798,913,1027,1141,1255,1369,1483,1597,1711,1826],"input_token_ids_sha256":"dfd87b8cdcdc45c3aed78576c14b5dd53c5f3a8de7a4a69358857943c1f8ba1a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[115,229,343,457,571,685,799,914,1028,1142,1256,1370,1484,1598,1712,1827],"target_token_ids_sha256":"4ed30d7e42a7866174c94f532bdcc6947a15ad58245c18f3aba71a4612f05990"},"formatted_content_sha256":"d1e07461677a5fe92f27926505552cc8a343cd60a5b5be86c2fa00090fae27f0","generator_receipt_sha256":"801e44bdb609773d985e8fd74f1d853f1afb30cf0a66d2cee0f7013ae8749c9c","identity_record_sha256":"a1cb4798036f5d2c63f1da08e09141e23d4fdec224e854eeb2d770ac8ba11721","prompt_token_ids_sha256":"793e38d71d7e91af366df94fb359ffb203e20cafb77f7afbb412db3ac5cfc5d2","ruler_category":"retrieval","seed":12340,"selection_rank":0,"selection_sha256":"26538bdeb4a99f8904324b26add0826a13744b51ec94fd9a004c72a2ba68c287","sequence_length":1828,"sequence_token_ids_sha256":"793e38d71d7e91af366df94fb359ffb203e20cafb77f7afbb412db3ac5cfc5d2","source_content_sha256":"801e44bdb609773d985e8fd74f1d853f1afb30cf0a66d2cee0f7013ae8749c9c","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1828,"cache_exposed_stop":1828,"prefill_start":0,"prefill_stop":1828,"scored_start":1828,"scored_stop":1828},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"75123434e18fce7bfe0e4b995fd12769a05a08d35fc315c68b8438e286e23f77","anchor_positions":[246,494,742,990,1237,1485,1733,1981,2228,2476,2724,2972,3219,3467,3715,3963],"anchor_positions_sha256":"dff504ff5227ef8cb24123e368d13dfb8cef71f15a3e715472122831820e3e93","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multivalue:length=4096:seed=12340:sample=0","config":"niah_multivalue","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[246,494,741,989,1237,1484,1732,1980,2227,2475,2722,2970,3218,3465,3713,3961],"fisher_boundary_sha256":"b36b4cfcb1258559a6eeafb235f1efb9a3bc5d2d3ec31389b39290f7e7209d82","horizon":1,"input_positions":[247,495,742,990,1238,1485,1733,1981,2228,2476,2723,2971,3219,3466,3714,3962],"input_token_ids_sha256":"dd9ed3a5d8b1bac29e9c251660e29d7a61b1d0cbb9846d333768dd85128cd708","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[248,496,743,991,1239,1486,1734,1982,2229,2477,2724,2972,3220,3467,3715,3963],"target_token_ids_sha256":"80f466fa56b4c9b1c24b8b51bb8cacb08c7290a7b0f1fd9ceeb119534231b14d"},"formatted_content_sha256":"f6bd8b2f8c5eb25a8ea8e08f2f4fb8e581a8d2080ff40194bce0a86f5deccc0e","generator_receipt_sha256":"fc75f4bdae8929325589536065fb7c8f5bb1fd737001da616d339365aa94920a","identity_record_sha256":"8091f9201ab1213aa14972869be942841763c3897bee5b30b77fb3d34877dd86","prompt_token_ids_sha256":"d498355b8380354e768abc57a79afe8cc73c5849b98f01081fd945329db5e77a","ruler_category":"retrieval","seed":12340,"selection_rank":1,"selection_sha256":"34e79c93add91edaaf2f631ffbc9e4c5350f61dc96769cf971f03ae113ccd1b5","sequence_length":3964,"sequence_token_ids_sha256":"d498355b8380354e768abc57a79afe8cc73c5849b98f01081fd945329db5e77a","source_content_sha256":"fc75f4bdae8929325589536065fb7c8f5bb1fd737001da616d339365aa94920a","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":3964,"cache_exposed_stop":3964,"prefill_start":0,"prefill_stop":3964,"scored_start":3964,"scored_stop":3964},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"532ac060a0cb86e868a3d0d5f0e7217475df142f3480655a5f9cc923addbf97e","anchor_positions":[124,250,376,501,627,753,878,1004,1130,1255,1381,1507,1632,1758,1884,2010],"anchor_positions_sha256":"baf07dbe6e81bfe17433570a4cb75b5ed267ca19947bc93119fe6c078f612806","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=2048:seed=12339:sample=0","config":"vt","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[124,250,375,501,626,752,877,1003,1129,1254,1380,1505,1631,1756,1882,2008],"fisher_boundary_sha256":"9a1bd7b90a51362eb876693f97e6c20ec0fa6bff1077ed5acd1237a823804b3a","horizon":1,"input_positions":[125,251,376,502,627,753,878,1004,1130,1255,1381,1506,1632,1757,1883,2009],"input_token_ids_sha256":"d43a3eb571f5bfc8ea7dc04191b8336f22f066f154918abe65e057791d13a43e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[126,252,377,503,628,754,879,1005,1131,1256,1382,1507,1633,1758,1884,2010],"target_token_ids_sha256":"453114002e40922d3386b14ea8e4915c4aa872297dae554dfbf91fe2f3b3e9ec"},"formatted_content_sha256":"221e10f0973c46ef1f20a688046e4bffae24656998c1059a831d7b20726fede0","generator_receipt_sha256":"b9a9b27500644e9aa2532304d577b85ab988c4359dd5edf7630fc8a8bee4b60e","identity_record_sha256":"3c519c0d4df294b9bb4d57dfaf44f3fe4934ae8e697dc924d17dbf39644026e8","prompt_token_ids_sha256":"cd95e7b2f95f4b705b1b43dc6b201b8307b182fe379566cb2e900ef21733b21a","ruler_category":"multi_hop_tracing","seed":12339,"selection_rank":2,"selection_sha256":"48145a1ddcbd5ae2492d753c9ffdf69047346dcba3d4c88e131c036be6968d9e","sequence_length":2011,"sequence_token_ids_sha256":"cd95e7b2f95f4b705b1b43dc6b201b8307b182fe379566cb2e900ef21733b21a","source_content_sha256":"b9a9b27500644e9aa2532304d577b85ab988c4359dd5edf7630fc8a8bee4b60e","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2011,"cache_exposed_stop":2011,"prefill_start":0,"prefill_stop":2011,"scored_start":2011,"scored_stop":2011},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"deb02a9e929ebd4d409deb018ab87063b47401d9993356ae1689a17e44d9121f","anchor_positions":[248,497,746,995,1244,1494,1743,1992,2241,2490,2740,2989,3238,3487,3736,3986],"anchor_positions_sha256":"a5ad1bc8d8afa780351af0b99deecbf3b31c05390a133ecb9438c32584b13b51","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:fwe:length=4096:seed=12339:sample=0","config":"fwe","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[248,497,746,995,1244,1493,1742,1991,2240,2489,2738,2987,3236,3485,3734,3984],"fisher_boundary_sha256":"3ce3ab1d00aff5ae9a8f93284d14aaa6f1744a79acc88d0772877941983042b6","horizon":1,"input_positions":[249,498,747,996,1245,1494,1743,1992,2241,2490,2739,2988,3237,3486,3735,3985],"input_token_ids_sha256":"87a331f7d1b5d5467dcce6012238bb065bb86881942f6c071315f3911a04e910","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[250,499,748,997,1246,1495,1744,1993,2242,2491,2740,2989,3238,3487,3736,3986],"target_token_ids_sha256":"0c3be397f087a6670d953bee273680288fc932be2e83d55d3086f5f6090398bb"},"formatted_content_sha256":"0fead2f4a48fe6c038641241dc6133e35cf0290088d4b06bf388bbf505180d9d","generator_receipt_sha256":"7fe1d0a912ee13f2dc1aae6839a7f8e87a623d5df30a3408ac5c6059f88aed13","identity_record_sha256":"bef04af160238768f27800f450d76a76013d80a3c96282a8a684900ba5807d4d","prompt_token_ids_sha256":"1610d294ca00f9745a960e213dd10784d8a523eca88bdaded2c7bde663d09a1c","ruler_category":"aggregation","seed":12339,"selection_rank":3,"selection_sha256":"51065012a292ff5d1577ac8988306f3ebc15a9c955ac9c104f135c18f7ed7cc9","sequence_length":3987,"sequence_token_ids_sha256":"1610d294ca00f9745a960e213dd10784d8a523eca88bdaded2c7bde663d09a1c","source_content_sha256":"7fe1d0a912ee13f2dc1aae6839a7f8e87a623d5df30a3408ac5c6059f88aed13","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":3987,"cache_exposed_stop":3987,"prefill_start":0,"prefill_stop":3987,"scored_start":3987,"scored_stop":3987},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"8a6b31f4f399feceed15eefe68a896dcd67ad7e19c3225837e50ada41fc0a847","anchor_positions":[252,506,759,1013,1266,1520,1773,2027,2280,2534,2787,3041,3294,3548,3801,4055],"anchor_positions_sha256":"e3a8a2871a86f2cf789e76ba6df6ede1df61047b14b69480ff1217c0a911147f","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=4096:seed=12339:sample=0","config":"vt","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[252,505,759,1012,1265,1519,1772,2026,2279,2532,2786,3039,3292,3546,3799,4053],"fisher_boundary_sha256":"159f8ad58c26093c2f75744da0f9502f7a2906b033bd5fce2665cf525fefb38b","horizon":1,"input_positions":[253,506,760,1013,1266,1520,1773,2027,2280,2533,2787,3040,3293,3547,3800,4054],"input_token_ids_sha256":"e8a2ff60bb805b206d15a2e0bf12885fedc537e7082f756e2c445a6bf70c326d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[254,507,761,1014,1267,1521,1774,2028,2281,2534,2788,3041,3294,3548,3801,4055],"target_token_ids_sha256":"49fc01abb0da9ff437c1e5a9aefc9158a1f8db110f275f2946581b8be9491118"},"formatted_content_sha256":"946e2f3a8214ecc488a65888ebeb1dc5b42d3027357523816a5ce8d3414b899c","generator_receipt_sha256":"8ea9a21c91b1b5203697a120d21824bea8dc810455d228e58a7ae319dc72dd82","identity_record_sha256":"5556eeb12c7c154e817d9c46859eaeb64f6a54512b787f1b1f38bf9fcf284542","prompt_token_ids_sha256":"9973907d38c28c89e9a53bec04ecb4c695cc4c94f35050f705a22d2e6ab2c897","ruler_category":"multi_hop_tracing","seed":12339,"selection_rank":4,"selection_sha256":"5c5a0c61d80b5c3a76e5bbf1062b35594e4a4995d482c429634e381eaaadcf21","sequence_length":4056,"sequence_token_ids_sha256":"9973907d38c28c89e9a53bec04ecb4c695cc4c94f35050f705a22d2e6ab2c897","source_content_sha256":"8ea9a21c91b1b5203697a120d21824bea8dc810455d228e58a7ae319dc72dd82","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":4056,"cache_exposed_stop":4056,"prefill_start":0,"prefill_stop":4056,"scored_start":4056,"scored_stop":4056},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e6965757239ac98f811d9377515f2e5110de3926f0390ca3416c9f2e3ef74bd3","anchor_positions":[79,160,240,321,401,482,562,643,724,804,885,965,1046,1126,1207,1288],"anchor_positions_sha256":"a02a7b889641d261dbd8927a54b0e9bda873159838da4973760162d67a2adbcd","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multiquery:length=2048:seed=12339:sample=0","config":"niah_multiquery","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[79,159,240,320,401,481,562,642,722,803,883,964,1044,1125,1205,1286],"fisher_boundary_sha256":"57d7fe986325f9a24836dcf6f4afdba3cd327289b6aa8077b19ec4aaff566ce9","horizon":1,"input_positions":[80,160,241,321,402,482,563,643,723,804,884,965,1045,1126,1206,1287],"input_token_ids_sha256":"0a50d970537d979d05cfd20781db749fecbfb1eb42f1426875d990cadb4a07ee","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[81,161,242,322,403,483,564,644,724,805,885,966,1046,1127,1207,1288],"target_token_ids_sha256":"88866037b9a7485b03068ce42d898b0e5484251bc087329d421a95fea8993da1"},"formatted_content_sha256":"bfb4cc6fdedde1a2578a09daf8fa4e779725a810722080b59de46588e1d5ab2b","generator_receipt_sha256":"0e8034964c18899e91a435a5737bfa29f7791fc41c29f42e4f2d9d75fa3a98ff","identity_record_sha256":"6bb7156902c5f36fc7c3e3c92666c510e1a2d572d3c6c557cc98ab440dd94db2","prompt_token_ids_sha256":"c68672f5e746622c7ef9c8d45f9312057365f04182c7af0576723d8089244ca1","ruler_category":"retrieval","seed":12339,"selection_rank":5,"selection_sha256":"65f813250e251ff50bf38bf6b227abf56f6e43ea5dedef6b18e6cf94874e510d","sequence_length":1289,"sequence_token_ids_sha256":"c68672f5e746622c7ef9c8d45f9312057365f04182c7af0576723d8089244ca1","source_content_sha256":"0e8034964c18899e91a435a5737bfa29f7791fc41c29f42e4f2d9d75fa3a98ff","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1289,"cache_exposed_stop":1289,"prefill_start":0,"prefill_stop":1289,"scored_start":1289,"scored_stop":1289},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"128a207210316772af37d86a91c97448a5eec69418db26c573a15daa57fff7db","anchor_positions":[252,506,759,1013,1266,1520,1773,2027,2281,2534,2788,3041,3295,3548,3802,4056],"anchor_positions_sha256":"ab52ebab779359ace36255ddbc6dfccb71733c90923120903537310d833e1be5","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=4096:seed=12340:sample=0","config":"vt","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[252,505,759,1012,1266,1519,1773,2026,2279,2533,2786,3040,3293,3547,3800,4054],"fisher_boundary_sha256":"c02daef8e0847ca8ef8f6795ed064d750cbfb33070533ceaf84d3f199f95bdf4","horizon":1,"input_positions":[253,506,760,1013,1267,1520,1774,2027,2280,2534,2787,3041,3294,3548,3801,4055],"input_token_ids_sha256":"758c575c40dc855f729daac2fb4c427f22776d59517e67e7dcd8ad13b9015cd3","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[254,507,761,1014,1268,1521,1775,2028,2281,2535,2788,3042,3295,3549,3802,4056],"target_token_ids_sha256":"e7ee891d8494cf6174e6a03c1da26e912c52473297bcba994663141abda1e73e"},"formatted_content_sha256":"84db9b2d31c7c5cfb2df3227d24ed8fa03d50cc474dbaf560e4f7aee2e52bb97","generator_receipt_sha256":"5b6cd316a368db4420845994f9423aaeedc0fd75802aba9eafb7deb4f4eddc73","identity_record_sha256":"ce937620a8ea8bbf5af1e1b522055853a1ec44ac92520d74486cc8e8f0bae348","prompt_token_ids_sha256":"42167ea36f4f43f21c1eadf1bd9fb391cb7d16904f553e91eac1401c360e902c","ruler_category":"multi_hop_tracing","seed":12340,"selection_rank":6,"selection_sha256":"79c3cb58b42a4b44fb604b28cd22ae1aa71bc49e05a3eb8399ca698c0044ee51","sequence_length":4057,"sequence_token_ids_sha256":"42167ea36f4f43f21c1eadf1bd9fb391cb7d16904f553e91eac1401c360e902c","source_content_sha256":"5b6cd316a368db4420845994f9423aaeedc0fd75802aba9eafb7deb4f4eddc73","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":4057,"cache_exposed_stop":4057,"prefill_start":0,"prefill_stop":4057,"scored_start":4057,"scored_stop":4057},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0fe15004ffd8e39b3cd00a623cb86c9ae2e0570216811c075442338f6c8c6963","anchor_positions":[141,284,426,569,711,854,996,1139,1282,1424,1567,1709,1852,1994,2137,2280],"anchor_positions_sha256":"9ea473fa5cae3aef9908a7ca4fa168866d469f3650f18f07e5dce4ee02714d7c","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_2:length=4096:seed=12340:sample=0","config":"qa_2","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[141,283,426,568,711,853,996,1138,1280,1423,1565,1708,1850,1993,2135,2278],"fisher_boundary_sha256":"12c58ae371c707d325c0b7db5868000c046d213a4b92b2344f6482266e812bec","horizon":1,"input_positions":[142,284,427,569,712,854,997,1139,1281,1424,1566,1709,1851,1994,2136,2279],"input_token_ids_sha256":"e6b8490ddd4e543ee4bbe0a2096b3a34397a5c6e4ee16c9efb438fba25a387d5","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[143,285,428,570,713,855,998,1140,1282,1425,1567,1710,1852,1995,2137,2280],"target_token_ids_sha256":"644343be1688f80c9846724beff4e6d47423e4ced93613951f85b480fb1aa6bf"},"formatted_content_sha256":"1d3faed8d004df01dbd7309b8dd1ed0e326c1bd1c5fa54306abe22855487372d","generator_receipt_sha256":"fba640bf322cb8c1f6175cc1a73e6267b18618e8555d9601e202268427ebabe8","identity_record_sha256":"1e32be0f57d7bfa1580c73b4f1c5e8a06ff8f429ff4611e766534940463ad32e","prompt_token_ids_sha256":"dab144649219fd1b15323997ba14f284492db0e725562192c789c0f766c8de3c","ruler_category":"question_answering","seed":12340,"selection_rank":7,"selection_sha256":"7a527495d7e22511d3058e2b2a4a846cd8b244d93a021ee51622ee975c2530ea","sequence_length":2281,"sequence_token_ids_sha256":"dab144649219fd1b15323997ba14f284492db0e725562192c789c0f766c8de3c","source_content_sha256":"fba640bf322cb8c1f6175cc1a73e6267b18618e8555d9601e202268427ebabe8","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2281,"cache_exposed_stop":2281,"prefill_start":0,"prefill_stop":2281,"scored_start":2281,"scored_stop":2281},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4a35e619dc657f107bad1efd0356762a6276d17f1c44fe7c843454ce15350f14","anchor_positions":[84,170,255,341,426,512,597,683,768,854,939,1025,1110,1196,1281,1367],"anchor_positions_sha256":"52762b5ae990ebf341f9864370415ddc0be98c34b1415f0fb53c8f4bb5237780","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_1:length=2048:seed=12339:sample=0","config":"qa_1","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[84,169,255,340,425,511,596,682,767,852,938,1023,1108,1194,1279,1365],"fisher_boundary_sha256":"71d2d40de65a4fd2d9d9e691302e52d3a9fc059b1629b02c3b6543f8201e463f","horizon":1,"input_positions":[85,170,256,341,426,512,597,683,768,853,939,1024,1109,1195,1280,1366],"input_token_ids_sha256":"673327b5691ed16299c8dd34b55f88b87d1fe642535708588295ca0f00f89422","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[86,171,257,342,427,513,598,684,769,854,940,1025,1110,1196,1281,1367],"target_token_ids_sha256":"aefebb06df48a7ba94a6d5de230228ea289d46553d344edc66924e98dea4b336"},"formatted_content_sha256":"9463b4927878ab07df8a32bd0b0584b76f5d42182dc21d90193275757cadc022","generator_receipt_sha256":"8d5f10bad274c66452a465f737da20099210de1d161c27a36c6331783da5f4f2","identity_record_sha256":"9198d8dbcec2c86f25636e37bc394d9d761cea2bb93559b1786b527f69e7fd7c","prompt_token_ids_sha256":"1102e50da7352084f75b9fa23ab6cc2d70708530625fc296126f13ff696d77e9","ruler_category":"question_answering","seed":12339,"selection_rank":8,"selection_sha256":"8bfbb4cad87302a003026059836508ab6e2d89448865d0217704654cd5b9bc8e","sequence_length":1368,"sequence_token_ids_sha256":"1102e50da7352084f75b9fa23ab6cc2d70708530625fc296126f13ff696d77e9","source_content_sha256":"8d5f10bad274c66452a465f737da20099210de1d161c27a36c6331783da5f4f2","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1368,"cache_exposed_stop":1368,"prefill_start":0,"prefill_stop":1368,"scored_start":1368,"scored_stop":1368},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"56eb209f76f5b84c51f3a6dfa390051c3e6179a589094e42a630548e33c79ab6","anchor_positions":[117,235,353,472,590,708,827,945,1063,1182,1300,1418,1537,1655,1773,1892],"anchor_positions_sha256":"0f267c5222d70787fd8238bd3de2e5bcbdfa19d30f6519cbc9abe6eb49aff8f8","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_2:length=2048:seed=12340:sample=0","config":"qa_2","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[117,235,353,471,589,708,826,944,1062,1180,1299,1417,1535,1653,1771,1890],"fisher_boundary_sha256":"433027cb3aa35bbaaa74d9db1d137366f2b08b50e51181e868b602a2a55b6952","horizon":1,"input_positions":[118,236,354,472,590,709,827,945,1063,1181,1300,1418,1536,1654,1772,1891],"input_token_ids_sha256":"eef4188a93a14820c4b8529c64e375c3dc36f29c2314d289c81684b9e3d1db9d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[119,237,355,473,591,710,828,946,1064,1182,1301,1419,1537,1655,1773,1892],"target_token_ids_sha256":"828ccb325b0d5b2cbe71be59f39c620de4093e298de9f0dda10d92926f2263fe"},"formatted_content_sha256":"696b7eb132f4a3d5f49052ac7680047c4a0a1ad1fc05f0bfa2bb25b4ddd67353","generator_receipt_sha256":"e58aa114cbdb8502b1dff17517a049edca17022eacfe90cd5afc0e2eda097c3e","identity_record_sha256":"67d8a257b752e3b5c57062870f7fde1fc0efbf2427ed7c5aaa4b1f1b94419ddc","prompt_token_ids_sha256":"ca3e9180e70700d36dc0d15f1ca4e7e4febee3a9058814b3f575e89f58f04daf","ruler_category":"question_answering","seed":12340,"selection_rank":9,"selection_sha256":"8fd992ea1ad818ee9cb171e5d9070ef59e24c95c1dfb3549cdc3963f904c96d3","sequence_length":1893,"sequence_token_ids_sha256":"ca3e9180e70700d36dc0d15f1ca4e7e4febee3a9058814b3f575e89f58f04daf","source_content_sha256":"e58aa114cbdb8502b1dff17517a049edca17022eacfe90cd5afc0e2eda097c3e","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1893,"cache_exposed_stop":1893,"prefill_start":0,"prefill_stop":1893,"scored_start":1893,"scored_stop":1893},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"cc7b46a7e50bc99f0cdb680c8631375304bb7983eedd6e6bec9dbbd2fbcfbdf5","anchor_positions":[251,504,756,1009,1261,1514,1766,2019,2271,2524,2776,3029,3281,3534,3786,4039],"anchor_positions_sha256":"7360c0b207ce24b236c422e4e664bff53fcafb7e524bd09aaebed6ea7c3ec7f8","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_1:length=4096:seed=12339:sample=0","config":"qa_1","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[251,503,756,1008,1260,1513,1765,2018,2270,2522,2775,3027,3279,3532,3784,4037],"fisher_boundary_sha256":"e551a79a205e6b53d8d809cc13da1c1d6f98faf520fcf9e40ed135aee7b9bb8e","horizon":1,"input_positions":[252,504,757,1009,1261,1514,1766,2019,2271,2523,2776,3028,3280,3533,3785,4038],"input_token_ids_sha256":"eb303aed866e00a6665d3e56e5ccdf9775fbd1e8a9bb7f5652c3fd7bd9707424","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[253,505,758,1010,1262,1515,1767,2020,2272,2524,2777,3029,3281,3534,3786,4039],"target_token_ids_sha256":"fdd7b7b8e4d53d09f5fa32c50c573deb2dbc0fe9474bed57416b8fac4ca54c1e"},"formatted_content_sha256":"9c7b681fe9e8d8f416aceaac08bdfd92d885e97dc140320d76ff0d9fefb803db","generator_receipt_sha256":"2dd908d09753092d8a358262f5073246919dd0069dfb0a60eca406aba764caa5","identity_record_sha256":"de00cb025c394a25eaa32bd222daf198e16413a112f3bccd5a72b628d593cd51","prompt_token_ids_sha256":"a089b295fbea580bdf8d050e97c37b242dfd30da2c3c7a3e620b76a54ed9c79f","ruler_category":"question_answering","seed":12339,"selection_rank":10,"selection_sha256":"a114fee36328bb536916462de6cc07e89d3dd7b3809c5295bc63e2663d7973f7","sequence_length":4040,"sequence_token_ids_sha256":"a089b295fbea580bdf8d050e97c37b242dfd30da2c3c7a3e620b76a54ed9c79f","source_content_sha256":"2dd908d09753092d8a358262f5073246919dd0069dfb0a60eca406aba764caa5","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":4040,"cache_exposed_stop":4040,"prefill_start":0,"prefill_stop":4040,"scored_start":4040,"scored_stop":4040},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"271a989874c4cc17ca1a739347bba23628a9e467e85d3219bb363f9ed7b07bf0","anchor_positions":[115,231,348,464,580,697,813,930,1046,1162,1279,1395,1511,1628,1744,1861],"anchor_positions_sha256":"cf6b395abd6929080fc35256e6179d2e3c657bc70c02f16093cb0533b0035bcc","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:cwe:length=2048:seed=12340:sample=0","config":"cwe","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[115,231,347,464,580,696,812,929,1045,1161,1277,1394,1510,1626,1742,1859],"fisher_boundary_sha256":"4015c686931b188bc2932b79e3bfa9ac30c8393959c5972c66fb8b82816a3bfd","horizon":1,"input_positions":[116,232,348,465,581,697,813,930,1046,1162,1278,1395,1511,1627,1743,1860],"input_token_ids_sha256":"7a7df1462084ce8ab589c85b605a1f4311c0ec91dc306a02673a84285b192131","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[117,233,349,466,582,698,814,931,1047,1163,1279,1396,1512,1628,1744,1861],"target_token_ids_sha256":"77743f96c0e111a7c8ad0cb12178be0abbe535b25edec949d8812bb2522c07a0"},"formatted_content_sha256":"3b5a450e8c283afa9103e3393bb8414b1d833a42c88cc5d7d4bd3f26a6120467","generator_receipt_sha256":"1c23f946536f8b05ad81d768edf4b1a5cc74ce39a3e9048124593635eb45ad57","identity_record_sha256":"21e8e431c773084606e1bb52807ebb59255d6a84e9bff884ebb333b51b0833f7","prompt_token_ids_sha256":"52bbc9b8942f395aaf5ad8a6b712b521a6a64d665cf085cb5fba76caccdf5b8f","ruler_category":"aggregation","seed":12340,"selection_rank":11,"selection_sha256":"a4f7cd62de6cc19b84e61c4bbedb2f095811b5b15e0045c070a67a003ff2110a","sequence_length":1862,"sequence_token_ids_sha256":"52bbc9b8942f395aaf5ad8a6b712b521a6a64d665cf085cb5fba76caccdf5b8f","source_content_sha256":"1c23f946536f8b05ad81d768edf4b1a5cc74ce39a3e9048124593635eb45ad57","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1862,"cache_exposed_stop":1862,"prefill_start":0,"prefill_stop":1862,"scored_start":1862,"scored_stop":1862},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"134c9730064b61e43d80673f0cb931058bd1b3199bca91ad233138c0226ce11c","anchor_positions":[120,242,364,486,608,730,852,974,1096,1218,1340,1462,1584,1706,1828,1950],"anchor_positions_sha256":"1e64ef366c8307e3a3410c30274175d81a4400831d9e53231d196d85e95f722d","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:fwe:length=2048:seed=12339:sample=0","config":"fwe","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[120,242,364,486,608,729,851,973,1095,1217,1338,1460,1582,1704,1826,1948],"fisher_boundary_sha256":"6f240a219eecca56c401dc3488a6be62114cf717fb5d3354698e9b9bd01cb16c","horizon":1,"input_positions":[121,243,365,487,609,730,852,974,1096,1218,1339,1461,1583,1705,1827,1949],"input_token_ids_sha256":"cb97fb7a28acb3307ed6ee8d8c153283de017a8ff009aa8791bb6bcb457ee5f6","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[122,244,366,488,610,731,853,975,1097,1219,1340,1462,1584,1706,1828,1950],"target_token_ids_sha256":"317374ba87916932cb7eb42091c9f2e9f86931a246150c2f199b1253ca4deba1"},"formatted_content_sha256":"6133db426215f641d319c7f7a251c50725f112b597bacfa38b5b397f559b9bc6","generator_receipt_sha256":"bb2518d8cd4ac6e41fa7aafc28a2951da762de64e8c2843d63b20f731b44ed69","identity_record_sha256":"fda1b358c53c26d14ec53f9f1307c3a894b7bd9b2265f34dc4835ad657df108d","prompt_token_ids_sha256":"5dc1507b9a08d033e3d20d61fa912a594f04959a9046d7a45a38e8828444ba30","ruler_category":"aggregation","seed":12339,"selection_rank":12,"selection_sha256":"a802c5ae87d368a5b0da4639128597266434ded4c54487021d47309dd1c7d305","sequence_length":1951,"sequence_token_ids_sha256":"5dc1507b9a08d033e3d20d61fa912a594f04959a9046d7a45a38e8828444ba30","source_content_sha256":"bb2518d8cd4ac6e41fa7aafc28a2951da762de64e8c2843d63b20f731b44ed69","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1951,"cache_exposed_stop":1951,"prefill_start":0,"prefill_stop":1951,"scored_start":1951,"scored_stop":1951},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3a9bc7a5fff3d80eb04fe1bfebb5c1e9543f99c72342a46af4069af857693b87","anchor_positions":[222,446,669,893,1116,1340,1563,1787,2010,2234,2457,2681,2904,3128,3351,3575],"anchor_positions_sha256":"f02061347ae4b281dab9fe2e3493b73bfb716ffe8fbf61cda899a3009674e03c","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:cwe:length=4096:seed=12340:sample=0","config":"cwe","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[222,445,669,892,1115,1339,1562,1786,2009,2232,2456,2679,2902,3126,3349,3573],"fisher_boundary_sha256":"44f51387dfbe3723bf9be8b472cc53ae9a75aa3358aacbe82ce774bca098b167","horizon":1,"input_positions":[223,446,670,893,1116,1340,1563,1787,2010,2233,2457,2680,2903,3127,3350,3574],"input_token_ids_sha256":"7d7dec828b0e79f43e590036098693e354e401b432f63c491d9b74ef31439c57","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[224,447,671,894,1117,1341,1564,1788,2011,2234,2458,2681,2904,3128,3351,3575],"target_token_ids_sha256":"8a65c1ab28b83d19798e54204b1019dbe72fa0be387f3d6e56ba27d82d13e7fa"},"formatted_content_sha256":"bbb3633bd44e955d1f33e80a2c1f71f0ae0e778f671a8a282b12b48eb76e3e53","generator_receipt_sha256":"3a6698b2ea721d388d72f28e1c4d4646616d6952fe01ed8038c01dc53dee3a72","identity_record_sha256":"cad2c6009d9d8080acff02b0fc66babc81f232bac219039d597c2889f240a957","prompt_token_ids_sha256":"679f814d93763a3c0db75fbc0cbf70d684bfd0ccde3a62fe15ff9d09dbc1a014","ruler_category":"aggregation","seed":12340,"selection_rank":13,"selection_sha256":"e09c4998af87e1e33e97059c684d3b0558e6e1e107295e2ccf3655bc3d70e7c8","sequence_length":3576,"sequence_token_ids_sha256":"679f814d93763a3c0db75fbc0cbf70d684bfd0ccde3a62fe15ff9d09dbc1a014","source_content_sha256":"3a6698b2ea721d388d72f28e1c4d4646616d6952fe01ed8038c01dc53dee3a72","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":3576,"cache_exposed_stop":3576,"prefill_start":0,"prefill_stop":3576,"scored_start":3576,"scored_stop":3576},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"8046d74f592f47985b3070756d1e18a618db723abbd079cab7a5a30d4efcf596","anchor_positions":[124,250,375,501,626,752,877,1003,1129,1254,1380,1505,1631,1756,1882,2008],"anchor_positions_sha256":"d296c9d479bcd5abca26602f7c8ff6f9e1668dd7bcce7512dc2b7a9f6612803c","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=2048:seed=12340:sample=0","config":"vt","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[124,249,375,500,626,751,877,1002,1127,1253,1378,1504,1629,1755,1880,2006],"fisher_boundary_sha256":"bddd9058cc4355edbe7a8c638de06d849c2134d8813b5401ee6957f2d201a918","horizon":1,"input_positions":[125,250,376,501,627,752,878,1003,1128,1254,1379,1505,1630,1756,1881,2007],"input_token_ids_sha256":"56e2b2e1f3b426b0a06e44e2cbb64564b09850f5d87d64bf62b5cea5466bcee1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[126,251,377,502,628,753,879,1004,1129,1255,1380,1506,1631,1757,1882,2008],"target_token_ids_sha256":"bd6a1f6ec03f5d3adcfb37a0b1ac11dadaa147d56888994f667aa7841dc1257b"},"formatted_content_sha256":"b17100d16d1771ca1122c2b9937cacb984da6216ccf38bef7be0228979e43248","generator_receipt_sha256":"e8b50ab62299b2f7306b66c83a8c6f2b471e1198bc02aba090f1ab406da24110","identity_record_sha256":"ab943760500ed5247a613c9eb87ac98c621b16c8919c9b1bcf5bebb2f1da258f","prompt_token_ids_sha256":"c6dfdc419e73d825ef29b21a8e2369076e96964690bfab85b8262a2299d05ce2","ruler_category":"multi_hop_tracing","seed":12340,"selection_rank":14,"selection_sha256":"f833f67c79e0d9e87d7d872ac8220d32c02483b56c47dbcd2385976fff75fe5f","sequence_length":2009,"sequence_token_ids_sha256":"c6dfdc419e73d825ef29b21a8e2369076e96964690bfab85b8262a2299d05ce2","source_content_sha256":"e8b50ab62299b2f7306b66c83a8c6f2b471e1198bc02aba090f1ab406da24110","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2009,"cache_exposed_stop":2009,"prefill_start":0,"prefill_stop":2009,"scored_start":2009,"scored_stop":2009},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"163e62428e8cb8f598c255e524c87ac6fe75f5451263fe61445eb8cafaa35b23","anchor_positions":[246,493,740,987,1234,1482,1729,1976,2223,2470,2718,2965,3212,3459,3706,3954],"anchor_positions_sha256":"c09190aec7245025981cb99d1fb2f1b133b5cd4db4f1f1baa5e107f0f2266b39","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_single_1:length=4096:seed=12339:sample=0","config":"niah_single_1","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[246,493,740,987,1234,1481,1728,1975,2222,2469,2716,2963,3210,3457,3704,3952],"fisher_boundary_sha256":"5d449cbf648406313b9dd1abfb6d827d86ea00767627eae59437398324207334","horizon":1,"input_positions":[247,494,741,988,1235,1482,1729,1976,2223,2470,2717,2964,3211,3458,3705,3953],"input_token_ids_sha256":"bc6efe8e70dc84d76b6d63607de536e9dad39e3e692c1153b17d01d1f948f5b4","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[248,495,742,989,1236,1483,1730,1977,2224,2471,2718,2965,3212,3459,3706,3954],"target_token_ids_sha256":"8b9cd390ee896f79de83be50c222152af4aff55c3013e4e55bd32df875a3182e"},"formatted_content_sha256":"5044e6b8cb4a5c93592f73b69fc42041fad9c54b2ba7d53216fc486266f9f8ba","generator_receipt_sha256":"bd8f1387bc9aca3118ced5d9287c08f72c670bfbe6ef88729d36d6940cc2a22f","identity_record_sha256":"9209f85027779d40b0ad2a15080aef80e042f89c3c96760320f1648c2f18b461","prompt_token_ids_sha256":"786505e0d874cafdc0c5a41e3e0922fe96e048b0fb0ed0d4c098605d9bf54be8","ruler_category":"retrieval","seed":12339,"selection_rank":15,"selection_sha256":"ffbb6f30b667ab8eb83eff25cc9f46717b4a36965d1f9c15e08b6633fe46e13d","sequence_length":3955,"sequence_token_ids_sha256":"786505e0d874cafdc0c5a41e3e0922fe96e048b0fb0ed0d4c098605d9bf54be8","source_content_sha256":"bd8f1387bc9aca3118ced5d9287c08f72c670bfbe6ef88729d36d6940cc2a22f","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":3955,"cache_exposed_stop":3955,"prefill_start":0,"prefill_stop":3955,"scored_start":3955,"scored_stop":3955},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"}],"resolver_version":6,"schema_version":5,"selection":{"humaneval_plus_stage_a_b_namespace":"recurquant.experiment013.humaneval-plus.stage-a-b.v1\u0000","humaneval_plus_stage_c_namespace":"recurquant.experiment013.humaneval-plus.stage-c.v1\u0000","pg19_test_namespace":"recurquant.experiment013.pg19.test.v1\u0000","pg19_train_namespace":"recurquant.experiment013.pg19.train.v1\u0000","pg19_validation_namespace":"recurquant.experiment013.pg19.validation.v1\u0000"},"source_manifest_sha256":"e29e24e1f1873813f5cf9432f4fee5c88dcea1c8ae84c417167e0a02e0c2a847","status":"frozen","tokenizer":{"class":"Qwen2Tokenizer","file_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef","files":[{"name":"merges.txt","sha256":"a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d","size_bytes":3353259},{"name":"tokenizer.json","sha256":"fe000e3ed39ed12b8d2481d527d44f93c65d37e87645d2dcc80d1bf9d50d2927","size_bytes":12807196},{"name":"tokenizer_config.json","sha256":"e611fbccc7c29ef3b1cafb1cb7ea548d189968632901d678fd62be68c47885de","size_bytes":16712},{"name":"vocab.json","sha256":"ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003","size_bytes":6722759}],"revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68","source_id":"Qwen/Qwen3.5-0.8B-Base","transformers_version":"5.14.1"},"upstream_tool_contracts":{"evalplus_formatter":{"id":"evalplus/evalplus","revision":"26d6d00bb1fd0fa37f39c99d5290da67891d1c5e"},"ruler_generator":{"id":"NVIDIA/RULER","revision":"c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a"}}}} From 475659ac8a0a98aaf38814e89f0f95d31392ec8b Mon Sep 17 00:00:00 2001 From: HiddenTrojan Date: Sat, 15 Aug 2026 05:05:02 +0800 Subject: [PATCH 22/24] Repair Experiment 013 model staging preflight --- ...EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md | 196 ++++- scripts/run_static_q468_calibration.py | 336 +++++++- tests/test_run_static_q468_calibration.py | 760 +++++++++++++++++- 3 files changed, 1231 insertions(+), 61 deletions(-) diff --git a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md index 2ef21ef..398beda 100644 --- a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md +++ b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md @@ -1,7 +1,7 @@ # Experiment 013: static RHT-Q468 packed-native adoption protocol -> **Status: second replacement-H0 candidate after a preserved pre-model -> sealed-smoke contract failure; not yet re-preregistered.** +> **Status: third replacement-H0 candidate after a preserved post-authorization, +> pre-payload model-staging path failure; not yet re-preregistered.** > > This replacement working copy becomes the next frozen Experiment 013 > preregistration only when its exact bytes and dependencies are committed in a @@ -19,7 +19,7 @@ Protocol draft initiated: 2026-08-02 -Current second replacement-H0 candidate amended: 2026-08-15 +Current third replacement-H0 candidate amended: 2026-08-15 Pre-resolution audit amendment: 2026-08-02. The amendment corrects a cache-exposed-span off-by-one, binds the Stage-A calibration chain by exact @@ -379,6 +379,108 @@ pass unchanged. The old H1 model root may be treated only as preserved incident evidence, not as the official model root for the next H1; a fresh no-overwrite root must be published by a newly authorized `stage-model` execution. +Fifteenth pre-resolution model-staging-path amendment: 2026-08-15. Under +source commit `0f3ea5e86e5d2ec13d5c5836540ce105e41ad02b`, tag +`experiment013-h0-0f3ea5e`, and identity-only descendant +`dae5587adc8f9a2b16335dfdc501e7f0a3f5e6ab`, the required read-only +model-staging authorization succeeded. Its frozen identity file SHA-256 was +`65cd1ccd932db1aa4c8f2f06e4b7a88b67532734f611dc33bbba39fbfea1cdb7`. +The subsequent first `stage-model` invocation supplied repository-local Hub +cache root `.cache/exp013-identity`. Calibration-runner v4 fully +reauthenticated H1, H0 source, the frozen identity, and the public model- +metadata manifest, then deterministically rejected the normalized cache root +because an official Hub cache must be outside the repository. + +The attempted output root +`C:\tmp\recurquant-exp013-model-h1-dae5587a` remained absent. The failed +command inspected only staging-path metadata after authorization. It did not +import a Hub client, invoke a downloader, traverse or read cached payload +files, create a staging directory, publish an output root, construct an +adapter, load model configuration or weights, materialize calibration data, +execute CUDA or Fisher computation, or produce a score, policy, smoke marker, +calibration binding, stability value, or quality result. The Git worktree +remained clean. + +The immediate cause was an invalid command argument. The execution-control +defect was the absence of a non-consuming staging-path preflight and runner +v4's ordering of path validation after H1 authentication. This was neither an +experimental result nor an infrastructure interruption, so the existing +same-command infrastructure-retry exception does not apply and no retroactive +exception is introduced. The H0, tag, H1, identity, and failed command remain +unchanged as superseded incident evidence and authorize no further official +execution. + +Calibration-runner v5 introduces one pure staging-path validator shared by +`verify-model-staging-paths` and `stage-model`. The read-only verifier accepts +only repository root, Hub cache root, and prospective model output root. It +performs no Git operation or identity, source-manifest, model-manifest, Hub, +cache-payload, adapter, model, or dataset access; imports no Hub downloader; +creates no directory or artifact; and writes only deterministic canonical JSON +to stdout. It requires an existing regular non-link repository root, an +existing regular non-link external Hub cache, an existing regular non-link +output parent, an absent non-root output destination, and pairwise disjoint +repository, cache, and output roots in both nesting directions. Every existing +path component must be free of links and reparse points. The Hub-cache root and +output parent may not themselves be filesystem roots. The output leaf is 1 +through 128 characters, begins with an ASCII alphanumeric, contains only ASCII +alphanumerics, dot, underscore, or hyphen, and ends with an ASCII alphanumeric, +underscore, or hyphen. Reserved DOS names are also rejected. + +`stage-model` invokes the same pure validator before Git-executable +authentication or any H1 authentication and requires +`--expected-model-staging-path-contract-sha256` to equal the digest produced by +the prior verifier. A missing, malformed, or unequal digest fails at that pure +boundary and does not consume H1. After successful authentication, it repeats +the validator and requires the same normalized roots and directory-component +identities before Hub import, payload access, or staging creation. Immediately +before atomic publication it repeats both staging-path and identity +authorization and rejects any root, component-identity, destination-existence, +or authorization drift. A semantic or authentication mismatch still retires +H1; a documented infrastructure interruption after authentication begins but +before payload access permits only the existing exact same-command retry. + +Runner revision advances from v4 to v5. The new canonical stdout document uses +artifact kind `recurquant_experiment013_model_staging_paths_verification` and +schema version one. It publishes no raw local path. It contains SHA-256 digests +of the normalized absolute repository, Hub-cache, output-parent, and output- +root paths; SHA-256 digests of the ordered device/inode/mode identity chains for +the repository, cache, and output-parent components; explicit states +`existing_regular_non_link_directory` for those three existing roots and +`absent` for the output root; and `path_contract_sha256`. That contract digest +authenticates canonical newline-terminated JSON containing schema version one +plus exactly those path, component-identity, and state fields. A successful +`stage-model` result repeats the same digest as +`model_staging_path_contract_sha256`, coupling the standalone preflight to the +internally revalidated staging call. Identity schema v5; capture and resolver +procedure v6; adapter revision v2; RULER launcher v7, generation-manifest v2, +and runtime-manifest v3; calibration runtime-manifest v4; source-manifest +schema/profile v2; model-manifest v1; existing verification stdout schemas v1; +run-report v2; and Fisher-boundary and smoke-marker contracts v1 remain +unchanged. No record selection, dataset revision, calibration span, Fisher +boundary, quantization policy, metric, gate, model contract, or protected-stage +rule changes. + +A fresh clean H0 and source manifest, newly promoted calibration identity, +identity-only H1, successful staging-path preflight, model-staging +authorization, and fresh no-overwrite identity-bound model publication are +required. The exact RULER v7 batch, sealed runtime, public model metadata +manifest, Parquet manifest, and external shared Hub cache remain reusable only +after their existing point-of-use checks pass unchanged. The replacement +identity must retain all 160 records and content manifest +`ee72483a8f8b4370c9e667e4287747e5bc358aeb0265a58167140f4e780a7b29`; +relative to the retired +`65cd1ccd932db1aa4c8f2f06e4b7a88b67532734f611dc33bbba39fbfea1cdb7` +identity, only these five repository-source and promotion-hash cascade JSON +pointers may differ: + +```text +/canonical_evidence_sha256 +/evidence/execution_bindings/repository_source_manifest_file_sha256 +/evidence/promotion/candidate_canonical_evidence_sha256 +/evidence/promotion/candidate_file_sha256 +/evidence/source_manifest_sha256 +``` + ## Question Can a calibration-frozen, static Q4/Q6/Q8 recurrent-state layout satisfy the @@ -774,44 +876,74 @@ Before committing H1, the exact promoted identity bytes in their ignored, no-overwrite precommit location must pass `verify-frozen-identity-contract`. That read-only command authenticates H0 and its source manifest, loads the exact H0 resolver, and consumes the complete record inventory through calibration- -runner v4's identity view. It accepts no H1, model manifest, Hub, cache, or +runner v5's identity view. It accepts no H1, model manifest, Hub, cache, or output argument. Its non-persisted canonical JSON stdout document uses artifact kind `recurquant_experiment013_frozen_identity_contract_verification`, schema version one, and binds the H0/source contract, portable Git identity, all four execution bindings, complete identity/canonical/assignment hashes, public model and tokenizer contracts, and record count. The bytes that passed are then copied without modification as the sole H1 tree change; regeneration or hand editing -after that preflight is forbidden. After H1, the read-only command -`verify-model-staging-authorization` must invoke the same authorization path -used by `stage-model` and reauthenticate the H1, index, and -worktree identity bytes; H0 ancestry and unchanged source tree; the complete -frozen identity and execution bindings; and the exact public model-metadata -manifest. The command accepts no cache or output root, imports no Hub downloader, +after that preflight is forbidden. + +After H1 and before model-staging authorization, run the read-only +`verify-model-staging-paths` command twice against the exact intended +repository, external Hub cache, and absent output root. It accepts only those +three path arguments, uses the same pure validator as `stage-model`, performs no +Git or artifact authentication and no Hub, payload, adapter, model, or dataset +access, and creates no filesystem entry. Its non-persisted canonical JSON +stdout document uses artifact kind +`recurquant_experiment013_model_staging_paths_verification`, schema version one, +and binds status, runner revision, normalized absolute-path digests for the +repository, cache, output parent, and output root; component-identity-chain +digests for the three existing roots; their exact regular/non-link states; the +absent-output state; and the aggregate `path_contract_sha256`. The two stdout +byte strings must be identical. The exact digest must then be supplied to +`stage-model` as `--expected-model-staging-path-contract-sha256`; a missing, +malformed, or unequal value fails before Git or H1 authentication. A successful +`stage-model` result must echo that required digest as +`model_staging_path_contract_sha256`. A failure at this pre-authentication +boundary does not consume H1. + +Only then may `verify-model-staging-authorization` invoke the same authorization +path used by `stage-model` and reauthenticate the H1, index, and worktree +identity bytes; H0 ancestry and unchanged source tree; the complete frozen +identity and execution bindings; and the exact public model-metadata manifest. +The command accepts no cache or output root, imports no Hub downloader, downloads no file, and creates no directory or artifact. Its canonical JSON stdout document, which the command does not persist, uses artifact kind `recurquant_experiment013_model_staging_authorization`, schema version one, and binds status, runner revision, frozen-identity hash, H1, H0, repository-source- manifest hash, model-manifest hash, public model ID/revision, Hub-tree-manifest -hash, file count, and total bytes. Only a successful stdout document permits -`stage-model` to be attempted. A semantic or authentication mismatch retires -that H1; it may not be hand-edited or weakened. An argument-parse failure does -not consume H1 because authentication did not begin. A documented infrastructure -interruption before any model-payload access permits only an exact same-command -retry under that H1. +hash, file count, and total bytes. Only successful path and authorization +documents permit `stage-model` to be attempted with those same roots. A +semantic or authentication mismatch retires that H1; it may not be hand-edited +or weakened. An argument-parse or initial pure path-precondition failure does +not consume H1 because authentication did not begin. A documented +infrastructure interruption after authentication begins but before model- +payload access permits only an exact same-command retry under that H1. Model payload staging begins only after the frozen identity is tracked with identical H1, index, and worktree bytes. The identity-bound stager downloads only the exact sorted root files in the frozen model manifest at the exact -40-hex public Hub revision, using an external cache and no token. Returned cache -paths are untrusted: every source must resolve inside that cache, then be -stream-copied into a fresh sibling staging directory. Ordinary files are checked -by Git blob OID and size; LFS payloads are checked by payload SHA-256 and size. -The staged tree must have exact case-insensitive-unique inventory and contain no -links, reparse points, cache metadata, marker, or extra file. Reauthenticate the -identity, source, and manifest immediately before an atomic no-replace directory -rename, then independently authenticate the published model root. Failure may -clean only the owned staging directory; it never overwrites the output or -deletes the shared Hub cache. +40-hex public Hub revision, using an external cache and no token. Before Git or +H1 authentication, immediately after authorization, and immediately before +publication, the stager repeats the pure path validator and requires identical +normalized roots and existing-component identities. The cache and output +parent may not be filesystem roots, and the absent output leaf must be the same +canonical 1-through-128-character basename accepted by the preflight: an ASCII +alphanumeric first character, only ASCII alphanumeric/dot/underscore/hyphen +interior characters, an ASCII alphanumeric/underscore/hyphen final character, +and no reserved DOS name. Returned cache paths are untrusted: every source must +resolve inside that cache, then be stream-copied into a fresh sibling staging +directory. +Ordinary files are checked by Git blob OID and size; LFS payloads are checked by +payload SHA-256 and size. The staged tree must have exact case-insensitive- +unique inventory and contain no links, reparse points, cache metadata, marker, +or extra file. Reauthenticate the identity, source, and manifest immediately +before an atomic no-replace directory rename, then independently authenticate +the published model root. Failure may clean only the exact owned staging- +directory identity; it never overwrites the output or deletes the shared Hub +cache. Stage-A resolution additionally consumes one strictly decoded `experiment-013-stage-a-calibration-binding-v3` artifact. The resolved Stage-A @@ -1193,6 +1325,18 @@ accepted revision, manifest, and downloaded object is checked against its frozen identity before protected execution. Reports must not describe this as cryptographic attestation of the complete OS or toolchain. +The model-staging directory-component snapshots are honest-process race +hardening, not complete filesystem attestation. The runner snapshots every +component before and after resolution and repeats the complete path contract +before authentication, after authentication, and immediately before no-replace +publication; observed replacement or identity drift fails closed. It does not +retain kernel directory handles or perform every operation handle-relatively, +so a hostile local process or administrator could still race path-based I/O +between checks. The local OS, filesystem, and concurrently privileged processes +therefore remain in the trusted computing base. Stronger protection would +require held Windows directory handles and file IDs, or POSIX `openat`-style +no-follow operations. + A person with filesystem control can deliberately delete the lock and reflog, rewrite or remove refs, or start from a fresh clone. The pre-run seal also cannot authenticate a result that does not yet exist. Stronger public proof diff --git a/scripts/run_static_q468_calibration.py b/scripts/run_static_q468_calibration.py index 8f47b19..594b52f 100644 --- a/scripts/run_static_q468_calibration.py +++ b/scripts/run_static_q468_calibration.py @@ -56,7 +56,7 @@ CANONICAL_ADAPTER_MODULE: Final = "recurquant.experiment013_qwen35_adapter" CANONICAL_ADAPTER_PATH: Final = "src/recurquant/experiment013_qwen35_adapter.py" -RUNNER_REVISION: Final = "experiment-013-static-q468-calibration-runner-v4" +RUNNER_REVISION: Final = "experiment-013-static-q468-calibration-runner-v5" FROZEN_IDENTITY_SCHEMA_VERSION: Final = 5 FISHER_BOUNDARY_SCHEMA: Final = "recurquant.experiment013.fisher-boundary.v1" FISHER_BOUNDARY_NAMESPACE: Final = b"recurquant.experiment013.fisher-boundary.v1\0" @@ -81,6 +81,8 @@ FROZEN_IDENTITY_CONTRACT_SCHEMA: Final = 1 MODEL_STAGING_AUTHORIZATION_KIND: Final = "recurquant_experiment013_model_staging_authorization" MODEL_STAGING_AUTHORIZATION_SCHEMA: Final = 1 +MODEL_STAGING_PATHS_KIND: Final = "recurquant_experiment013_model_staging_paths_verification" +MODEL_STAGING_PATHS_SCHEMA: Final = 1 RUNTIME_MANIFEST_KIND: Final = "recurquant_experiment013_calibration_runtime_manifest" RUNTIME_MANIFEST_SCHEMA: Final = 4 RUN_REPORT_KIND: Final = "recurquant_experiment013_calibration_run" @@ -93,6 +95,19 @@ _SHA256_RE: Final = re.compile(r"[0-9a-f]{64}") _GIT_REVISION_RE: Final = re.compile(r"[0-9a-f]{40}") _SAFE_MODEL_FILE_RE: Final = re.compile(r"[A-Za-z0-9][A-Za-z0-9._/-]*") +_MODEL_STAGING_OUTPUT_ROOT_NAME_RE: Final = re.compile( + r"[A-Za-z0-9](?:[A-Za-z0-9._-]{0,126}[A-Za-z0-9_-])?" +) +_WINDOWS_RESERVED_BASENAMES: Final = frozenset( + { + "aux", + "con", + "nul", + "prn", + *(f"com{index}" for index in range(1, 10)), + *(f"lpt{index}" for index in range(1, 10)), + } +) _WEIGHT_FILE_RE: Final = re.compile( r"(?:^|/)(?:model(?:-[0-9]+-of-[0-9]+)?|" r"model\.safetensors-[0-9]+-of-[0-9]+)\.safetensors$" @@ -765,6 +780,23 @@ class ModelStagingAuthorization: source_commit: str +@dataclass(frozen=True, slots=True) +class DirectoryComponentIdentity: + device: int + inode: int + mode: int + + +@dataclass(frozen=True, slots=True) +class ModelStagingPaths: + repository_root: Path + repository_component_identities: tuple[DirectoryComponentIdentity, ...] + hub_cache_root: Path + hub_cache_component_identities: tuple[DirectoryComponentIdentity, ...] + output_root: Path + output_parent_component_identities: tuple[DirectoryComponentIdentity, ...] + + @dataclass(frozen=True, slots=True) class FrozenIdentitySourceAuthorization: identity: FrozenCalibrationIdentity @@ -1844,19 +1876,121 @@ def _path_is_within(path: Path, root: Path) -> bool: return True -def _ensure_regular_directory(path: Path, *, context: str) -> Path: - absolute = Path(os.path.abspath(path)) - if os.path.lexists(absolute): - if _is_link_or_reparse(absolute) or not absolute.is_dir(): - raise CalibrationRunError(f"{context} must be a regular non-link directory") - else: - absolute.mkdir(parents=True, exist_ok=False) - candidate = Path(absolute.anchor) +def _require_existing_regular_directory( + path: Path, + *, + context: str, +) -> tuple[Path, tuple[DirectoryComponentIdentity, ...]]: + """Normalize one existing directory without creating or following links.""" + + try: + absolute = Path(os.path.abspath(path)) + except (OSError, TypeError, ValueError) as exc: + raise CalibrationRunError(f"{context} is unavailable") from exc + if not absolute.anchor: + raise CalibrationRunError(f"{context} is not absolute after normalization") + component = Path(absolute.anchor) + components = [component] for part in absolute.parts[1:]: - candidate /= part - if _is_link_or_reparse(candidate) or not candidate.is_dir(): - raise CalibrationRunError(f"{context} traverses a link or non-directory") - return absolute.resolve(strict=True) + component /= part + components.append(component) + + def snapshot() -> tuple[DirectoryComponentIdentity, ...]: + identities: list[DirectoryComponentIdentity] = [] + for candidate in components: + if not os.path.lexists(candidate): + raise CalibrationRunError(f"{context} must already exist") + try: + status = candidate.lstat() + except OSError as exc: + raise CalibrationRunError(f"{context} is unavailable") from exc + if stat.S_ISLNK(status.st_mode) or bool( + getattr(status, "st_file_attributes", 0) & _WINDOWS_REPARSE_POINT + ): + raise CalibrationRunError(f"{context} traverses a link or non-directory") + if not stat.S_ISDIR(status.st_mode): + raise CalibrationRunError(f"{context} traverses a link or non-directory") + identities.append( + DirectoryComponentIdentity( + device=status.st_dev, + inode=status.st_ino, + mode=status.st_mode, + ) + ) + return tuple(identities) + + before = snapshot() + try: + resolved = absolute.resolve(strict=True) + except (OSError, RuntimeError) as exc: + raise CalibrationRunError(f"{context} is unavailable") from exc + after = snapshot() + if after != before: + raise CalibrationRunError(f"{context} changed while it was validated") + return resolved, after + + +def _normalized_absolute_path_sha256(path: Path) -> str: + if not path.is_absolute(): + raise ValueError("path digest input must be absolute") + normalized = os.path.normcase(os.path.normpath(str(path))) + return sha256_bytes(normalized.encode("utf-8")) + + +def _directory_component_identities_sha256( + identities: tuple[DirectoryComponentIdentity, ...], +) -> str: + return sha256_bytes( + canonical_json_bytes( + [ + { + "device": item.device, + "inode": item.inode, + "mode": item.mode, + } + for item in identities + ] + ) + ) + + +def _model_staging_path_contract(paths: ModelStagingPaths) -> dict[str, object]: + return { + "hub_cache_component_identities_sha256": _directory_component_identities_sha256( + paths.hub_cache_component_identities + ), + "hub_cache_root_absolute_path_sha256": _normalized_absolute_path_sha256( + paths.hub_cache_root + ), + "hub_cache_root_state": "existing_regular_non_link_directory", + "output_parent_absolute_path_sha256": _normalized_absolute_path_sha256( + paths.output_root.parent + ), + "output_parent_component_identities_sha256": _directory_component_identities_sha256( + paths.output_parent_component_identities + ), + "output_parent_state": "existing_regular_non_link_directory", + "output_root_absolute_path_sha256": _normalized_absolute_path_sha256(paths.output_root), + "output_root_state": "absent", + "repository_component_identities_sha256": _directory_component_identities_sha256( + paths.repository_component_identities + ), + "repository_root_absolute_path_sha256": _normalized_absolute_path_sha256( + paths.repository_root + ), + "repository_root_state": "existing_regular_non_link_directory", + } + + +def _model_staging_path_contract_sha256(paths: ModelStagingPaths) -> str: + return sha256_bytes( + canonical_json_bytes( + { + "schema_version": MODEL_STAGING_PATHS_SCHEMA, + **_model_staging_path_contract(paths), + } + ) + ) def _validate_model_staging_roots( @@ -1864,23 +1998,97 @@ def _validate_model_staging_roots( repository_root: Path, hub_cache_root: Path, output_root: Path, -) -> tuple[Path, Path]: - repository = Path(os.path.abspath(repository_root)).resolve(strict=True) - destination = Path(os.path.abspath(output_root)) - if not destination.name: +) -> ModelStagingPaths: + """Validate and snapshot the model-staging path boundary without writes.""" + + repository, repository_identities = _require_existing_regular_directory( + repository_root, + context="repository root", + ) + try: + lexical_destination = Path(output_root) + lexical_name = lexical_destination.name + destination = Path(os.path.abspath(lexical_destination)) + except (OSError, TypeError, ValueError) as exc: + raise CalibrationRunError("model output root is unavailable") from exc + if not lexical_name or not destination.name: raise CalibrationRunError("model output root cannot be a filesystem root") - if os.path.lexists(destination): - raise FileExistsError(f"refusing to overwrite staged model root: {destination}") - parent = _ensure_regular_directory(destination.parent, context="model output parent") - cache = _ensure_regular_directory(hub_cache_root, context="Hub cache root") + if ( + _MODEL_STAGING_OUTPUT_ROOT_NAME_RE.fullmatch(lexical_name) is None + or lexical_name.endswith((".", " ")) + or lexical_name.casefold().partition(".")[0] in _WINDOWS_RESERVED_BASENAMES + or destination.name != lexical_name + ): + raise CalibrationRunError( + "model output root name must be a canonical Windows-safe basename" + ) + parent, output_parent_identities = _require_existing_regular_directory( + destination.parent, + context="model output parent", + ) + cache, cache_identities = _require_existing_regular_directory( + hub_cache_root, + context="Hub cache root", + ) + if not parent.name: + raise CalibrationRunError("model output parent cannot be a filesystem root") + if not cache.name: + raise CalibrationRunError("Hub cache root cannot be a filesystem root") resolved_destination = parent / destination.name - roots = (("model output root", resolved_destination), ("Hub cache root", cache)) - for context, candidate in roots: - if _path_is_within(candidate, repository): - raise CalibrationRunError(f"{context} must be outside the repository") - if _path_is_within(resolved_destination, cache) or _path_is_within(cache, resolved_destination): + if os.path.lexists(resolved_destination): + raise FileExistsError(f"refusing to overwrite staged model root: {resolved_destination}") + repository_identity = repository_identities[-1] + cache_identity = cache_identities[-1] + if ( + _path_is_within(resolved_destination, repository) + or repository_identity in output_parent_identities + ): + raise CalibrationRunError("model output root must be outside the repository") + if ( + _path_is_within(cache, repository) + or _path_is_within(repository, cache) + or repository_identity in cache_identities + or cache_identity in repository_identities + ): + raise CalibrationRunError("Hub cache root must not overlap the repository") + if ( + _path_is_within(resolved_destination, cache) + or _path_is_within(cache, resolved_destination) + or cache_identity in output_parent_identities + ): raise CalibrationRunError("Hub cache and staged model roots must not be nested") - return cache, resolved_destination + return ModelStagingPaths( + repository_root=repository, + repository_component_identities=repository_identities, + hub_cache_root=cache, + hub_cache_component_identities=cache_identities, + output_root=resolved_destination, + output_parent_component_identities=output_parent_identities, + ) + + +def verify_model_staging_paths( + *, + repository_root: Path, + hub_cache_root: Path, + output_root: Path, +) -> dict[str, object]: + """Verify model-staging roots without authorization, imports, or writes.""" + + paths = _validate_model_staging_roots( + repository_root=repository_root, + hub_cache_root=hub_cache_root, + output_root=output_root, + ) + contract = _model_staging_path_contract(paths) + return { + "artifact_kind": MODEL_STAGING_PATHS_KIND, + **contract, + "path_contract_sha256": _model_staging_path_contract_sha256(paths), + "runner_revision": RUNNER_REVISION, + "schema_version": MODEL_STAGING_PATHS_SCHEMA, + "status": "verified_model_staging_paths", + } def _assert_regular_cache_payload(cache_root: Path, returned_path: object) -> Path: @@ -2029,6 +2237,7 @@ def stage_identity_bound_model( source_commit: str, model_file_manifest_path: Path, expected_model_file_manifest_sha256: str, + expected_model_staging_path_contract_sha256: str, hub_cache_root: Path, output_root: Path, local_files_only: bool = False, @@ -2038,23 +2247,39 @@ def stage_identity_bound_model( if type(local_files_only) is not bool: raise TypeError("local_files_only must be bool") + initial_paths = _validate_model_staging_roots( + repository_root=repository_root, + hub_cache_root=hub_cache_root, + output_root=output_root, + ) + expected_path_contract_sha256 = _sha256( + expected_model_staging_path_contract_sha256, + context="expected model-staging path contract SHA-256", + ) + actual_path_contract_sha256 = _model_staging_path_contract_sha256(initial_paths) + if actual_path_contract_sha256 != expected_path_contract_sha256: + raise CalibrationRunError("model-staging path contract differs from the CLI binding") git_executable = _authenticate_git_executable(git_executable_path) authorization = _authenticate_model_staging_authorization( git_executable=git_executable, frozen_identity_path=frozen_identity_path, expected_frozen_identity_sha256=expected_frozen_identity_sha256, identity_commit=identity_commit, - repository_root=repository_root, + repository_root=initial_paths.repository_root, repository_source_manifest_path=repository_source_manifest_path, source_commit=source_commit, model_file_manifest_path=model_file_manifest_path, expected_model_file_manifest_sha256=expected_model_file_manifest_sha256, ) - cache, destination = _validate_model_staging_roots( + confirmed_paths = _validate_model_staging_roots( repository_root=repository_root, hub_cache_root=hub_cache_root, output_root=output_root, ) + if confirmed_paths != initial_paths: + raise CalibrationRunError("model-staging roots changed during authorization") + cache = confirmed_paths.hub_cache_root + destination = confirmed_paths.output_root if downloader is None: from huggingface_hub import hf_hub_download @@ -2063,7 +2288,12 @@ def stage_identity_bound_model( prefix = f".{destination.name}.staging-" staging = Path(tempfile.mkdtemp(prefix=prefix, dir=destination.parent)) owned_staging = True + staging_component_identities: tuple[DirectoryComponentIdentity, ...] | None = None try: + staging, staging_component_identities = _require_existing_regular_directory( + staging, + context="owned model staging directory", + ) for record in authorization.model_manifest.files: returned = downloader( repo_id=authorization.model_manifest.model_id, @@ -2084,7 +2314,7 @@ def stage_identity_bound_model( frozen_identity_path=frozen_identity_path, expected_frozen_identity_sha256=expected_frozen_identity_sha256, identity_commit=identity_commit, - repository_root=repository_root, + repository_root=initial_paths.repository_root, repository_source_manifest_path=repository_source_manifest_path, source_commit=source_commit, model_file_manifest_path=model_file_manifest_path, @@ -2093,6 +2323,13 @@ def stage_identity_bound_model( if repeated != authorization: raise CalibrationRunError("model-staging authorization changed before publication") _verify_exact_local_model_tree(staging, repeated.model_manifest) + publication_paths = _validate_model_staging_roots( + repository_root=repository_root, + hub_cache_root=hub_cache_root, + output_root=output_root, + ) + if publication_paths != initial_paths: + raise CalibrationRunError("model-staging roots changed before publication") _atomic_rename_directory_no_overwrite(staging, destination) owned_staging = False finally: @@ -2103,6 +2340,19 @@ def stage_identity_bound_model( raise RuntimeError("owned model staging directory escaped its parent") from exc if not staging.name.startswith(prefix): raise RuntimeError("owned model staging directory name drifted") + if staging_component_identities is None: + raise RuntimeError("owned model staging directory identity was not captured") + try: + current_staging, current_identities = _require_existing_regular_directory( + staging, + context="owned model staging directory", + ) + except CalibrationRunError as exc: + raise RuntimeError( + "refusing to clean an unauthenticated model staging directory" + ) from exc + if current_staging != staging or current_identities != staging_component_identities: + raise RuntimeError("refusing to clean a replaced model staging directory") shutil.rmtree(staging, ignore_errors=False) _verify_exact_local_model_tree(destination, authorization.model_manifest) return { @@ -2112,6 +2362,7 @@ def stage_identity_bound_model( "model_id": authorization.model_manifest.model_id, "model_manifest_file_sha256": authorization.model_manifest.file_sha256, "model_root": str(destination), + "model_staging_path_contract_sha256": expected_path_contract_sha256, "revision": authorization.model_manifest.revision, "source_commit": authorization.source_commit, "status": "staged_authenticated_model", @@ -5618,10 +5869,14 @@ def _capture_manifest_mode(arguments: Sequence[str]) -> int | None: "stage-model", "verify-frozen-identity-contract", "verify-model-staging-authorization", + "verify-model-staging-paths", }: return None command = arguments[0] - parser = argparse.ArgumentParser(prog=f"{Path(__file__).name} {command}") + parser = argparse.ArgumentParser( + prog=f"{Path(__file__).name} {command}", + allow_abbrev=False, + ) if command == "prepare-runtime": parser.add_argument("--git-executable", required=True, type=Path) parser.add_argument("--source-python", required=True, type=Path) @@ -5638,6 +5893,10 @@ def _capture_manifest_mode(arguments: Sequence[str]) -> int | None: parser.add_argument("--repository-root", required=True, type=Path) parser.add_argument("--repository-source-manifest", required=True, type=Path) parser.add_argument("--source-commit", required=True) + elif command == "verify-model-staging-paths": + parser.add_argument("--repository-root", required=True, type=Path) + parser.add_argument("--hub-cache-root", required=True, type=Path) + parser.add_argument("--output-root", required=True, type=Path) elif command in {"stage-model", "verify-model-staging-authorization"}: parser.add_argument("--git-executable", required=True, type=Path) parser.add_argument("--frozen-identity", required=True, type=Path) @@ -5651,6 +5910,10 @@ def _capture_manifest_mode(arguments: Sequence[str]) -> int | None: if command == "stage-model": parser.add_argument("--hub-cache-root", required=True, type=Path) parser.add_argument("--output-root", required=True, type=Path) + parser.add_argument( + "--expected-model-staging-path-contract-sha256", + required=True, + ) parser.add_argument("--local-files-only", action="store_true") else: parser.add_argument("--output", required=True, type=Path) @@ -5689,12 +5952,23 @@ def _capture_manifest_mode(arguments: Sequence[str]) -> int | None: source_commit=args.source_commit, model_file_manifest_path=args.model_file_manifest, expected_model_file_manifest_sha256=args.expected_model_file_manifest_sha256, + expected_model_staging_path_contract_sha256=( + args.expected_model_staging_path_contract_sha256 + ), hub_cache_root=args.hub_cache_root, output_root=args.output_root, local_files_only=args.local_files_only, ) print(json.dumps(details, sort_keys=True)) return 0 + if command == "verify-model-staging-paths": + details = verify_model_staging_paths( + repository_root=args.repository_root, + hub_cache_root=args.hub_cache_root, + output_root=args.output_root, + ) + print(canonical_json_bytes(details).decode("utf-8"), end="") + return 0 if command == "verify-frozen-identity-contract": details = verify_frozen_identity_contract( git_executable_path=args.git_executable, diff --git a/tests/test_run_static_q468_calibration.py b/tests/test_run_static_q468_calibration.py index a5faf00..0acbd00 100644 --- a/tests/test_run_static_q468_calibration.py +++ b/tests/test_run_static_q468_calibration.py @@ -66,6 +66,20 @@ def authenticated_git_path() -> Path: return runner._authenticate_git_executable(None).path +def model_staging_path_contract_sha256( + repository_root: Path, + hub_cache_root: Path, + output_root: Path, +) -> str: + return runner._model_staging_path_contract_sha256( + runner._validate_model_staging_roots( + repository_root=repository_root, + hub_cache_root=hub_cache_root, + output_root=output_root, + ) + ) + + def fisher_boundary_contract( token_ids: tuple[int, ...] = (1, 2, 3), ) -> dict[str, object]: @@ -1794,7 +1808,7 @@ def test_model_authentication_rejects_extra_files(tmp_path: Path) -> None: runner.authenticate_local_model_files(root, manifest, calibration_api=api) -def test_stage_model_authenticates_before_touching_downloader_cache_or_output( +def test_stage_model_auth_failure_does_not_import_download_or_write( tmp_path: Path, ) -> None: candidate = runner.canonical_json_bytes( @@ -1812,6 +1826,9 @@ def test_stage_model_authenticates_before_touching_downloader_cache_or_output( identity_path = tmp_path / "candidate.json" identity_path.write_bytes(candidate) cache = tmp_path / "cache" + cache.mkdir() + cache_marker = cache / "preexisting.txt" + cache_marker.write_text("untouched", encoding="utf-8") output = tmp_path / "model" calls: list[dict[str, object]] = [] @@ -1825,13 +1842,17 @@ def test_stage_model_authenticates_before_touching_downloader_cache_or_output( source_commit="1" * 40, model_file_manifest_path=tmp_path / "missing-model.json", expected_model_file_manifest_sha256="2" * 64, + expected_model_staging_path_contract_sha256=( + model_staging_path_contract_sha256(SCRIPT.parents[1], cache, output) + ), hub_cache_root=cache, output_root=output, downloader=lambda **kwargs: calls.append(kwargs), ) assert calls == [] - assert not cache.exists() + assert cache_marker.read_text(encoding="utf-8") == "untouched" + assert list(cache.iterdir()) == [cache_marker] assert not output.exists() @@ -2476,6 +2497,497 @@ def test_committed_frozen_identity_requires_exact_head_index_and_worktree_blob( ) +def test_model_staging_path_preflight_is_bound_deterministic_and_read_only( + tmp_path: Path, +) -> None: + repository = tmp_path / "repository" + cache = tmp_path / "hub-cache" + repository.mkdir() + cache.mkdir() + marker = cache / "preexisting.txt" + marker.write_text("untouched", encoding="utf-8") + output = tmp_path / "published-model" + + first = runner.verify_model_staging_paths( + repository_root=repository, + hub_cache_root=cache, + output_root=output, + ) + second = runner.verify_model_staging_paths( + repository_root=repository, + hub_cache_root=cache, + output_root=output, + ) + paths = runner._validate_model_staging_roots( + repository_root=repository, + hub_cache_root=cache, + output_root=output, + ) + + assert set(first) == { + "artifact_kind", + "hub_cache_component_identities_sha256", + "hub_cache_root_absolute_path_sha256", + "hub_cache_root_state", + "output_parent_absolute_path_sha256", + "output_parent_component_identities_sha256", + "output_parent_state", + "output_root_absolute_path_sha256", + "output_root_state", + "path_contract_sha256", + "repository_component_identities_sha256", + "repository_root_absolute_path_sha256", + "repository_root_state", + "runner_revision", + "schema_version", + "status", + } + assert first == { + "artifact_kind": runner.MODEL_STAGING_PATHS_KIND, + "hub_cache_component_identities_sha256": ( + runner._directory_component_identities_sha256(paths.hub_cache_component_identities) + ), + "hub_cache_root_absolute_path_sha256": runner._normalized_absolute_path_sha256( + cache.resolve() + ), + "hub_cache_root_state": "existing_regular_non_link_directory", + "output_parent_absolute_path_sha256": runner._normalized_absolute_path_sha256( + output.parent.resolve() + ), + "output_parent_component_identities_sha256": ( + runner._directory_component_identities_sha256(paths.output_parent_component_identities) + ), + "output_parent_state": "existing_regular_non_link_directory", + "output_root_absolute_path_sha256": runner._normalized_absolute_path_sha256( + output.parent.resolve() / output.name + ), + "output_root_state": "absent", + "path_contract_sha256": runner._model_staging_path_contract_sha256(paths), + "repository_component_identities_sha256": ( + runner._directory_component_identities_sha256(paths.repository_component_identities) + ), + "repository_root_absolute_path_sha256": runner._normalized_absolute_path_sha256( + repository.resolve() + ), + "repository_root_state": "existing_regular_non_link_directory", + "runner_revision": runner.RUNNER_REVISION, + "schema_version": runner.MODEL_STAGING_PATHS_SCHEMA, + "status": "verified_model_staging_paths", + } + assert runner.canonical_json_bytes(first) == runner.canonical_json_bytes(second) + assert str(tmp_path).encode("utf-8") not in runner.canonical_json_bytes(first) + assert marker.read_text(encoding="utf-8") == "untouched" + assert list(cache.iterdir()) == [marker] + assert not output.exists() + + +@pytest.mark.parametrize( + ("case", "message"), + [ + ("missing-cache", "Hub cache root must already exist"), + ("cache-file", "Hub cache root traverses a link or non-directory"), + ("missing-output-parent", "model output parent must already exist"), + ("existing-output-file", "refusing to overwrite"), + ("existing-output-directory", "refusing to overwrite"), + ("repo-local-cache", "Hub cache root must not overlap the repository"), + ("cache-contains-repository", "Hub cache root must not overlap the repository"), + ("repo-local-output", "model output root must be outside the repository"), + ("output-inside-cache", "must not be nested"), + ("lexical-output-inside-cache", "must not be nested"), + ], +) +def test_model_staging_path_preflight_rejects_invalid_roots_without_writes( + case: str, + message: str, + tmp_path: Path, +) -> None: + repository = tmp_path / "repository" + outside = tmp_path / "outside" + output_parent = outside / "output-parent" + repository.mkdir() + output_parent.mkdir(parents=True) + cache = outside / "hub-cache" + cache.mkdir() + output = output_parent / "model" + + if case == "missing-cache": + cache = outside / "missing-cache" + elif case == "cache-file": + cache = outside / "cache-file" + cache.write_text("not a directory", encoding="utf-8") + elif case == "missing-output-parent": + output = outside / "missing-parent" / "model" + elif case == "existing-output-file": + output.write_text("occupied", encoding="utf-8") + elif case == "existing-output-directory": + output.mkdir() + elif case == "repo-local-cache": + cache = repository / "cache" + cache.mkdir() + elif case == "cache-contains-repository": + cache = tmp_path + elif case == "repo-local-output": + output = repository / "model" + elif case == "output-inside-cache": + output = cache / "model" + elif case == "lexical-output-inside-cache": + output = cache / "unused" / ".." / "model" + else: # pragma: no cover - guards the table itself + raise AssertionError(case) + + before = sorted(path.relative_to(tmp_path) for path in tmp_path.rglob("*")) + with pytest.raises((FileExistsError, runner.CalibrationRunError), match=message): + runner.verify_model_staging_paths( + repository_root=repository, + hub_cache_root=cache, + output_root=output, + ) + after = sorted(path.relative_to(tmp_path) for path in tmp_path.rglob("*")) + + assert after == before + + +def test_model_staging_path_preflight_rejects_filesystem_roots(tmp_path: Path) -> None: + repository = tmp_path / "repository" + cache = tmp_path / "cache" + repository.mkdir() + cache.mkdir() + root = Path(tmp_path.anchor) + + with pytest.raises(runner.CalibrationRunError, match="output parent.*filesystem root"): + runner.verify_model_staging_paths( + repository_root=repository, + hub_cache_root=cache, + output_root=root / f"recurquant-output-{tmp_path.name}", + ) + with pytest.raises(runner.CalibrationRunError, match="cache root.*filesystem root"): + runner.verify_model_staging_paths( + repository_root=repository, + hub_cache_root=root, + output_root=tmp_path / "model", + ) + + +@pytest.mark.parametrize( + "name", + ("model:stream", "CON", "aux.json", "model.", "model ", "bad name", "\x01model"), +) +def test_model_staging_path_preflight_requires_windows_safe_output_basename( + name: str, + tmp_path: Path, +) -> None: + repository = tmp_path / "repository" + cache = tmp_path / "cache" + repository.mkdir() + cache.mkdir() + + with pytest.raises(runner.CalibrationRunError, match="Windows-safe basename"): + runner.verify_model_staging_paths( + repository_root=repository, + hub_cache_root=cache, + output_root=tmp_path / name, + ) + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows case aliases only") +def test_model_staging_path_preflight_rejects_case_alias_repository_cache( + tmp_path: Path, +) -> None: + repository = tmp_path / "repository" + repository.mkdir() + case_alias = Path(str(repository).swapcase()) + + with pytest.raises(runner.CalibrationRunError, match="must not overlap"): + runner.verify_model_staging_paths( + repository_root=repository, + hub_cache_root=case_alias, + output_root=tmp_path / "model", + ) + + +def test_model_staging_path_preflight_rejects_links_and_dangling_output( + tmp_path: Path, +) -> None: + repository = tmp_path / "repository" + real_cache = tmp_path / "real-cache" + repository.mkdir() + real_cache.mkdir() + cache_link = tmp_path / "cache-link" + dangling_output = tmp_path / "dangling-output" + try: + cache_link.symlink_to(real_cache, target_is_directory=True) + dangling_output.symlink_to(tmp_path / "missing-target", target_is_directory=True) + except OSError as error: + pytest.skip(f"symlink creation is unavailable: {type(error).__name__}") + + with pytest.raises(runner.CalibrationRunError, match="link or non-directory"): + runner.verify_model_staging_paths( + repository_root=repository, + hub_cache_root=cache_link, + output_root=tmp_path / "model", + ) + with pytest.raises(FileExistsError, match="refusing to overwrite"): + runner.verify_model_staging_paths( + repository_root=repository, + hub_cache_root=real_cache, + output_root=dangling_output, + ) + + +def test_model_staging_path_snapshot_rejects_in_validation_component_replacement( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository = tmp_path / "repository" + cache = tmp_path / "cache" + displaced = tmp_path / "displaced-cache" + repository.mkdir() + cache.mkdir() + original_resolve = Path.resolve + replaced = False + + def replace_after_resolve(path: Path, strict: bool = False) -> Path: + nonlocal replaced + resolved = original_resolve(path, strict=strict) + if path == cache and not replaced: + replaced = True + cache.rename(displaced) + cache.mkdir() + return resolved + + monkeypatch.setattr(Path, "resolve", replace_after_resolve) + with pytest.raises(runner.CalibrationRunError, match="changed while it was validated"): + runner.verify_model_staging_paths( + repository_root=repository, + hub_cache_root=cache, + output_root=tmp_path / "model", + ) + + +def test_repo_local_cache_fails_before_git_h1_import_or_write( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository = tmp_path / "repository" + cache = repository / "cache" + output_parent = tmp_path / "outside" + cache.mkdir(parents=True) + output_parent.mkdir() + output = output_parent / "model" + authentication_calls: list[str] = [] + original_import = builtins.__import__ + + def guarded_import(name: str, *args: object, **kwargs: object) -> object: + if name == "huggingface_hub" or name.startswith("huggingface_hub."): + pytest.fail("invalid staging roots imported a Hugging Face client") + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", guarded_import) + monkeypatch.setattr( + runner, + "_authenticate_git_executable", + lambda _path: authentication_calls.append("git"), + ) + monkeypatch.setattr( + runner, + "_authenticate_model_staging_authorization", + lambda **_kwargs: authentication_calls.append("h1"), + ) + before = sorted(path.relative_to(tmp_path) for path in tmp_path.rglob("*")) + + with pytest.raises(runner.CalibrationRunError, match="must not overlap"): + runner.stage_identity_bound_model( + frozen_identity_path=tmp_path / "missing-identity.json", + expected_frozen_identity_sha256="d" * 64, + identity_commit="3" * 40, + repository_root=repository, + repository_source_manifest_path=tmp_path / "missing-source.json", + source_commit="1" * 40, + model_file_manifest_path=tmp_path / "missing-model.json", + expected_model_file_manifest_sha256="2" * 64, + expected_model_staging_path_contract_sha256="0" * 64, + hub_cache_root=cache, + output_root=output, + ) + + assert authentication_calls == [] + assert sorted(path.relative_to(tmp_path) for path in tmp_path.rglob("*")) == before + assert not output.exists() + + +def test_stage_model_path_contract_mismatch_fails_before_git_h1_hub_or_staging( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository = tmp_path / "repository" + cache = tmp_path / "cache" + repository.mkdir() + cache.mkdir() + marker = cache / "preexisting.txt" + marker.write_text("untouched", encoding="utf-8") + output = tmp_path / "model" + authentication_calls: list[str] = [] + original_import = builtins.__import__ + + def guarded_import(name: str, *args: object, **kwargs: object) -> object: + if name == "huggingface_hub" or name.startswith("huggingface_hub."): + pytest.fail("path-contract mismatch imported a Hugging Face client") + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", guarded_import) + monkeypatch.setattr( + runner, + "_authenticate_git_executable", + lambda _path: authentication_calls.append("git"), + ) + monkeypatch.setattr( + runner, + "_authenticate_model_staging_authorization", + lambda **_kwargs: authentication_calls.append("h1"), + ) + + with pytest.raises(runner.CalibrationRunError, match="differs from the CLI binding"): + runner.stage_identity_bound_model( + frozen_identity_path=tmp_path / "missing-identity.json", + expected_frozen_identity_sha256="d" * 64, + identity_commit="3" * 40, + repository_root=repository, + repository_source_manifest_path=tmp_path / "missing-source.json", + source_commit="1" * 40, + model_file_manifest_path=tmp_path / "missing-model.json", + expected_model_file_manifest_sha256="2" * 64, + expected_model_staging_path_contract_sha256="0" * 64, + hub_cache_root=cache, + output_root=output, + ) + + assert authentication_calls == [] + assert marker.read_text(encoding="utf-8") == "untouched" + assert list(cache.iterdir()) == [marker] + assert not output.exists() + assert not list(tmp_path.glob(".model.staging-*")) + + +@pytest.mark.parametrize( + "invalid_sha256", + ("A" * 64, "0" * 63, "g" * 64), + ids=("uppercase", "short", "non-hex"), +) +def test_stage_model_requires_exact_lowercase_path_contract_sha256_before_auth( + invalid_sha256: str, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository = tmp_path / "repository" + cache = tmp_path / "cache" + repository.mkdir() + cache.mkdir() + output = tmp_path / "model" + authentication_calls: list[str] = [] + monkeypatch.setattr( + runner, + "_authenticate_git_executable", + lambda _path: authentication_calls.append("git"), + ) + monkeypatch.setattr( + runner, + "_authenticate_model_staging_authorization", + lambda **_kwargs: authentication_calls.append("h1"), + ) + + with pytest.raises(ValueError, match="lowercase SHA-256"): + runner.stage_identity_bound_model( + frozen_identity_path=tmp_path / "identity.json", + expected_frozen_identity_sha256="d" * 64, + identity_commit="3" * 40, + repository_root=repository, + repository_source_manifest_path=tmp_path / "source.json", + source_commit="1" * 40, + model_file_manifest_path=tmp_path / "model.json", + expected_model_file_manifest_sha256="2" * 64, + expected_model_staging_path_contract_sha256=invalid_sha256, + hub_cache_root=cache, + output_root=output, + ) + + assert authentication_calls == [] + assert not output.exists() + + +def test_stage_model_rejects_path_snapshot_mismatch_before_hub_or_staging( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository = tmp_path / "repository" + cache = tmp_path / "cache" + other_cache = tmp_path / "other-cache" + repository.mkdir() + cache.mkdir() + other_cache.mkdir() + output = tmp_path / "model" + original_validate = runner._validate_model_staging_roots + expected_path_contract_sha256 = model_staging_path_contract_sha256( + repository, + cache, + output, + ) + validations = 0 + authorizations = 0 + original_import = builtins.__import__ + + def validate(**kwargs: object) -> runner.ModelStagingPaths: + nonlocal validations + validations += 1 + result = original_validate(**kwargs) + if validations == 2: + return replace(result, hub_cache_root=other_cache.resolve()) + return result + + def authenticate(**_kwargs: object) -> runner.ModelStagingAuthorization: + nonlocal authorizations + authorizations += 1 + return model_staging_authorization() + + def guarded_import(name: str, *args: object, **kwargs: object) -> object: + if name == "huggingface_hub" or name.startswith("huggingface_hub."): + pytest.fail("path mismatch imported a Hugging Face client") + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(runner, "_validate_model_staging_roots", validate) + monkeypatch.setattr( + runner, + "_authenticate_git_executable", + lambda _path: runner.AuthenticatedGitExecutable( + path=tmp_path / "git.exe", + absolute_path_sha256="a" * 64, + sha256="b" * 64, + size_bytes=1, + ), + ) + monkeypatch.setattr(runner, "_authenticate_model_staging_authorization", authenticate) + monkeypatch.setattr(builtins, "__import__", guarded_import) + + with pytest.raises(runner.CalibrationRunError, match="changed during authorization"): + runner.stage_identity_bound_model( + frozen_identity_path=tmp_path / "identity.json", + expected_frozen_identity_sha256="d" * 64, + identity_commit="3" * 40, + repository_root=repository, + repository_source_manifest_path=tmp_path / "source.json", + source_commit="1" * 40, + model_file_manifest_path=tmp_path / "model.json", + expected_model_file_manifest_sha256="2" * 64, + expected_model_staging_path_contract_sha256=expected_path_contract_sha256, + hub_cache_root=cache, + output_root=output, + ) + + assert validations == 2 + assert authorizations == 1 + assert not output.exists() + assert not list(tmp_path.glob(".model.staging-*")) + + def test_stage_model_downloads_only_exact_bound_files_and_publishes_atomically( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -2502,7 +3014,15 @@ def download(**kwargs: object) -> str: monkeypatch.setattr(runner, "_authenticate_model_staging_authorization", authenticate) cache = tmp_path / "hub-cache" + cache.mkdir() output = tmp_path / "published-model" + expected_path_contract_sha256 = runner._model_staging_path_contract_sha256( + runner._validate_model_staging_roots( + repository_root=SCRIPT.parents[1], + hub_cache_root=cache, + output_root=output, + ) + ) result = runner.stage_identity_bound_model( frozen_identity_path=tmp_path / "identity.json", expected_frozen_identity_sha256="d" * 64, @@ -2512,6 +3032,7 @@ def download(**kwargs: object) -> str: source_commit="1" * 40, model_file_manifest_path=tmp_path / "model-manifest.json", expected_model_file_manifest_sha256=authorization.model_manifest.file_sha256, + expected_model_staging_path_contract_sha256=expected_path_contract_sha256, hub_cache_root=cache, output_root=output, local_files_only=True, @@ -2539,6 +3060,65 @@ def download(**kwargs: object) -> str: assert not list(tmp_path.glob(".published-model.staging-*")) assert result["status"] == "staged_authenticated_model" assert result["source_commit"] == "1" * 40 + assert result["model_staging_path_contract_sha256"] == expected_path_contract_sha256 + + +def test_stage_model_revalidates_root_identities_immediately_before_publication( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + files = staged_model_files() + authorization = model_staging_authorization(files) + repository = tmp_path / "repository" + cache = tmp_path / "cache" + displaced_cache = tmp_path / "displaced-cache" + repository.mkdir() + cache.mkdir() + output = tmp_path / "published-model" + expected_path_contract_sha256 = model_staging_path_contract_sha256( + repository, + cache, + output, + ) + authorization_calls = 0 + + def authenticate(**_kwargs: object) -> runner.ModelStagingAuthorization: + nonlocal authorization_calls + authorization_calls += 1 + if authorization_calls == 2: + cache.rename(displaced_cache) + cache.mkdir() + return authorization + + def download(**kwargs: object) -> str: + name = str(kwargs["filename"]) + target = Path(kwargs["cache_dir"]) / "snapshot" / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(files[name]) + return str(target) + + monkeypatch.setattr(runner, "_authenticate_model_staging_authorization", authenticate) + + with pytest.raises(runner.CalibrationRunError, match="changed before publication"): + runner.stage_identity_bound_model( + frozen_identity_path=tmp_path / "identity.json", + expected_frozen_identity_sha256="d" * 64, + identity_commit="3" * 40, + repository_root=repository, + repository_source_manifest_path=tmp_path / "source.json", + source_commit="1" * 40, + model_file_manifest_path=tmp_path / "manifest.json", + expected_model_file_manifest_sha256=authorization.model_manifest.file_sha256, + expected_model_staging_path_contract_sha256=expected_path_contract_sha256, + hub_cache_root=cache, + output_root=output, + downloader=download, + ) + + assert authorization_calls == 2 + assert not output.exists() + assert not list(tmp_path.glob(".published-model.staging-*")) + assert displaced_cache.is_dir() def test_stage_model_failure_cleans_owned_staging_and_never_exposes_final_root( @@ -2565,6 +3145,8 @@ def download(**kwargs: object) -> str: "_authenticate_model_staging_authorization", lambda **_kwargs: authorization, ) + cache = tmp_path / "cache" + cache.mkdir() output = tmp_path / "published-model" with pytest.raises(OSError, match="injected"): runner.stage_identity_bound_model( @@ -2576,7 +3158,10 @@ def download(**kwargs: object) -> str: source_commit="1" * 40, model_file_manifest_path=tmp_path / "manifest.json", expected_model_file_manifest_sha256=authorization.model_manifest.file_sha256, - hub_cache_root=tmp_path / "cache", + expected_model_staging_path_contract_sha256=( + model_staging_path_contract_sha256(SCRIPT.parents[1], cache, output) + ), + hub_cache_root=cache, output_root=output, downloader=download, ) @@ -2585,6 +3170,58 @@ def download(**kwargs: object) -> str: assert not list(tmp_path.glob(".published-model.staging-*")) +def test_stage_model_refuses_to_clean_replaced_owned_staging_directory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + authorization = model_staging_authorization() + repository = tmp_path / "repository" + cache = tmp_path / "cache" + repository.mkdir() + cache.mkdir() + output = tmp_path / "published-model" + displaced = tmp_path / "displaced-owned-staging" + replacement: Path | None = None + + def download(**_kwargs: object) -> str: + nonlocal replacement + [owned] = list(tmp_path.glob(".published-model.staging-*")) + owned.rename(displaced) + owned.mkdir() + (owned / "replacement-owner.txt").write_text("preserve", encoding="utf-8") + replacement = owned + raise OSError("injected after staging replacement") + + monkeypatch.setattr( + runner, + "_authenticate_model_staging_authorization", + lambda **_kwargs: authorization, + ) + + with pytest.raises(RuntimeError, match="refusing to clean a replaced"): + runner.stage_identity_bound_model( + frozen_identity_path=tmp_path / "identity.json", + expected_frozen_identity_sha256="d" * 64, + identity_commit="3" * 40, + repository_root=repository, + repository_source_manifest_path=tmp_path / "source.json", + source_commit="1" * 40, + model_file_manifest_path=tmp_path / "manifest.json", + expected_model_file_manifest_sha256=authorization.model_manifest.file_sha256, + expected_model_staging_path_contract_sha256=( + model_staging_path_contract_sha256(repository, cache, output) + ), + hub_cache_root=cache, + output_root=output, + downloader=download, + ) + + assert replacement is not None + assert (replacement / "replacement-owner.txt").read_text(encoding="utf-8") == "preserve" + assert displaced.is_dir() + assert not output.exists() + + @pytest.mark.parametrize("outside_kind", ("outside", "wrong-content")) def test_stage_model_rejects_untrusted_cache_payload_without_publication( tmp_path: Path, @@ -2598,6 +3235,8 @@ def test_stage_model_rejects_untrusted_cache_payload_without_publication( "_authenticate_model_staging_authorization", lambda **_kwargs: authorization, ) + cache = tmp_path / "cache" + cache.mkdir() def download(**kwargs: object) -> str: name = str(kwargs["filename"]) @@ -2623,7 +3262,10 @@ def download(**kwargs: object) -> str: source_commit="1" * 40, model_file_manifest_path=tmp_path / "manifest.json", expected_model_file_manifest_sha256=authorization.model_manifest.file_sha256, - hub_cache_root=tmp_path / "cache", + expected_model_staging_path_contract_sha256=( + model_staging_path_contract_sha256(SCRIPT.parents[1], cache, output) + ), + hub_cache_root=cache, output_root=output, downloader=download, ) @@ -3167,6 +3809,8 @@ def stage(**kwargs: object) -> dict[str, object]: str(tmp_path / "cache"), "--output-root", str(tmp_path / "output"), + "--expected-model-staging-path-contract-sha256", + "5" * 64, "--local-files-only", ] @@ -3175,6 +3819,7 @@ def stage(**kwargs: object) -> dict[str, object]: { "expected_frozen_identity_sha256": "1" * 64, "expected_model_file_manifest_sha256": "4" * 64, + "expected_model_staging_path_contract_sha256": "5" * 64, "frozen_identity_path": tmp_path / "identity.json", "git_executable_path": authenticated_git_path(), "hub_cache_root": tmp_path / "cache", @@ -3188,6 +3833,110 @@ def stage(**kwargs: object) -> dict[str, object]: } ] + missing_contract_arguments = arguments[:-3] + arguments[-1:] + with pytest.raises(SystemExit): + runner.main(missing_contract_arguments) + assert len(calls) == 1 + + +def test_verify_model_staging_paths_cli_is_canonical_and_exactly_forwarded( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + calls: list[dict[str, object]] = [] + receipt = { + "artifact_kind": runner.MODEL_STAGING_PATHS_KIND, + "schema_version": runner.MODEL_STAGING_PATHS_SCHEMA, + "status": "verified_model_staging_paths", + } + + def verify(**kwargs: object) -> dict[str, object]: + calls.append(dict(kwargs)) + return receipt + + monkeypatch.setattr(runner, "verify_model_staging_paths", verify) + arguments = [ + "verify-model-staging-paths", + "--repository-root", + str(tmp_path / "repository"), + "--hub-cache-root", + str(tmp_path / "cache"), + "--output-root", + str(tmp_path / "output"), + ] + + assert runner.main(arguments) == 0 + assert capsys.readouterr().out == runner.canonical_json_bytes(receipt).decode("utf-8") + assert calls == [ + { + "hub_cache_root": tmp_path / "cache", + "output_root": tmp_path / "output", + "repository_root": tmp_path / "repository", + } + ] + + +@pytest.mark.parametrize( + "forbidden", + [ + ("--git-executable", "git"), + ("--frozen-identity", "identity.json"), + ("--expected-frozen-identity-sha256", "1" * 64), + ("--identity-commit", "2" * 40), + ("--repository-source-manifest", "source.json"), + ("--source-commit", "3" * 40), + ("--model-file-manifest", "model.json"), + ("--expected-model-file-manifest-sha256", "4" * 64), + ("--expected-model-staging-path-contract-sha256", "5" * 64), + ("--local-files-only",), + ("--model-root", "model"), + ("--cache-root", "cache"), + ("--output", "receipt.json"), + ("--output-dir", "output"), + ], + ids=( + "git", + "identity", + "identity-hash", + "h1", + "source-manifest", + "h0", + "model-manifest", + "model-manifest-hash", + "staging-path-contract-hash", + "local-hub-mode", + "model-root", + "generic-cache", + "persisted-output", + "calibration-output", + ), +) +def test_verify_model_staging_paths_cli_rejects_every_non_path_surface( + forbidden: tuple[str, ...], + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[dict[str, object]] = [] + monkeypatch.setattr( + runner, + "verify_model_staging_paths", + lambda **kwargs: calls.append(dict(kwargs)), + ) + arguments = [ + "verify-model-staging-paths", + "--repository-root", + str(tmp_path / "repository"), + "--hub-cache-root", + str(tmp_path / "cache"), + "--output-root", + str(tmp_path / "output"), + ] + + with pytest.raises(SystemExit): + runner.main([*arguments, *forbidden]) + assert calls == [] + def test_verify_frozen_identity_contract_cli_is_canonical_and_exactly_forwarded( tmp_path: Path, @@ -3242,6 +3991,7 @@ def verify(**kwargs: object) -> dict[str, object]: ("--identity-commit", "2" * 40), ("--model-file-manifest", "model.json"), ("--expected-model-file-manifest-sha256", "4" * 64), + ("--expected-model-staging-path-contract-sha256", "5" * 64), ("--hub-cache-root", "cache"), ("--output-root", "output"), ("--output", "output.json"), @@ -3252,6 +4002,7 @@ def verify(**kwargs: object) -> dict[str, object]: "h1", "model-manifest", "model-manifest-hash", + "staging-path-contract-hash", "hub-cache", "output-root", "output", @@ -3348,6 +4099,7 @@ def verify(**kwargs: object) -> dict[str, object]: for forbidden in ( ("--hub-cache-root", str(tmp_path / "forbidden-cache")), ("--output-root", str(tmp_path / "forbidden-output")), + ("--expected-model-staging-path-contract-sha256", "5" * 64), ("--local-files-only",), ): with pytest.raises(SystemExit): From fd67384944dc92abac0422960ea53fc973b64736 Mon Sep 17 00:00:00 2001 From: HiddenTrojan Date: Sat, 15 Aug 2026 05:42:10 +0800 Subject: [PATCH 23/24] Record Experiment 013 runner-v5 calibration identity --- evidence/experiment013-calibration-identity-v5-h0-475659a.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 evidence/experiment013-calibration-identity-v5-h0-475659a.json diff --git a/evidence/experiment013-calibration-identity-v5-h0-475659a.json b/evidence/experiment013-calibration-identity-v5-h0-475659a.json new file mode 100644 index 0000000..b900781 --- /dev/null +++ b/evidence/experiment013-calibration-identity-v5-h0-475659a.json @@ -0,0 +1 @@ +{"canonical_evidence_sha256":"c8da4c13b59edb2eef92ec888e008061be08da7fa0ee2d7df8ae1fdee01b629b","evidence":{"artifact_kind":"recurquant_static_rht_q468_identity","calibration_binding":null,"calibration_split_half":{"assignment":[{"canonical_id":"872","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":0,"rank_sha256":"01e970ad8f49c615f8994f4fc293d373d517f753cf5cef3ee60e04b0f70e91de","ruler_category":null,"seed":null,"sequence_length":205},{"canonical_id":"907","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":1,"rank_sha256":"0302f20255cec68e850f67a965f94ac9c6e975fdbeb3654b213c03aaf97749f8","ruler_category":null,"seed":null,"sequence_length":195},{"canonical_id":"602","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":2,"rank_sha256":"0533693c5eafac0f2bffd8ecb5f531f37e3220e7347ef75eab142ed8a013bcf5","ruler_category":null,"seed":null,"sequence_length":131},{"canonical_id":"884","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":3,"rank_sha256":"08317423d6ed9f655783d9a55d4773b1ff31c0a8355dfd9186fb319920e3d84b","ruler_category":null,"seed":null,"sequence_length":180},{"canonical_id":"771","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":4,"rank_sha256":"0aa65ef4aee19c821f354944ea63d3a5573594cbf98328585a2bf06c2de91309","ruler_category":null,"seed":null,"sequence_length":210},{"canonical_id":"885","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":5,"rank_sha256":"0b8df4c9f6062430c6c4ccd1cb93ed972689c2181d647b3425c8ac35779c5f9c","ruler_category":null,"seed":null,"sequence_length":193},{"canonical_id":"672","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":6,"rank_sha256":"0d45859c2052e7483b7caae8e52246e30d4af1be980cbcb175164817f5ed511d","ruler_category":null,"seed":null,"sequence_length":164},{"canonical_id":"955","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":7,"rank_sha256":"15743574d0995e07e15392053b3cafd1cef3998b668e87041851ac95861883ca","ruler_category":null,"seed":null,"sequence_length":116},{"canonical_id":"794","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":8,"rank_sha256":"19ee9c8929dc155f23f27a50530b329f876367ad3aea0bf4ad48ce4621328517","ruler_category":null,"seed":null,"sequence_length":154},{"canonical_id":"641","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":9,"rank_sha256":"19fd16656926305e85d6a90a8abfc8688cde21fff9383ac96aa4a507279d79f9","ruler_category":null,"seed":null,"sequence_length":104},{"canonical_id":"744","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":10,"rank_sha256":"1afe31e9cfc41a6df7f6609d1e2119ad55693730c75b4bd46f7c9235522eafa0","ruler_category":null,"seed":null,"sequence_length":137},{"canonical_id":"647","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":11,"rank_sha256":"1b76da1efe10f0a39d6b61952eb3fee319e653390381bd4a8efb060e10161d77","ruler_category":null,"seed":null,"sequence_length":107},{"canonical_id":"763","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":12,"rank_sha256":"1c7722a133b7999ee3e6ff0c4c7d9286371137589ac7b013d5d9a1615c2eca43","ruler_category":null,"seed":null,"sequence_length":181},{"canonical_id":"835","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":13,"rank_sha256":"27e79049077f4f70e44f882154a065bb8fe957aa57c51a5f95d025e7f6b69e7e","ruler_category":null,"seed":null,"sequence_length":113},{"canonical_id":"720","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":14,"rank_sha256":"2c2ba3d77f6374f2c035569f8be3b970f76c25bc587f0f2bd165939b49af1c27","ruler_category":null,"seed":null,"sequence_length":292},{"canonical_id":"677","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":15,"rank_sha256":"3015295069fd5535022e22aa7d447fdfd3b893623d25fbe8b2279d94c7d99ee5","ruler_category":null,"seed":null,"sequence_length":120},{"canonical_id":"819","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":16,"rank_sha256":"3355f161793b19251d4281f96e19b37a970a709a1b417d52e6dfbb2a4812d792","ruler_category":null,"seed":null,"sequence_length":385},{"canonical_id":"950","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":17,"rank_sha256":"339900553b726e1b1256a75b102fbeed3f547bfae8d96f80e8fec0f8dec6c09b","ruler_category":null,"seed":null,"sequence_length":399},{"canonical_id":"622","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":18,"rank_sha256":"363cce01da9a0aa22049e9b3a8e04a2dc931468e02d0086bb7af971621cfc711","ruler_category":null,"seed":null,"sequence_length":373},{"canonical_id":"708","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":19,"rank_sha256":"372acd4106f75fd22c1fdb5d8e13b74c64a9364db2f490f50daa482c0962112d","ruler_category":null,"seed":null,"sequence_length":97},{"canonical_id":"878","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":20,"rank_sha256":"38728fc433a299c9ff5e0f49471dda51ee67c5a980c966c00c715bd76e7f1c88","ruler_category":null,"seed":null,"sequence_length":168},{"canonical_id":"648","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":21,"rank_sha256":"388ca5f79823a878cd3ce1e033b61e5cab8bf668fe57d751138244feef74a2f4","ruler_category":null,"seed":null,"sequence_length":225},{"canonical_id":"937","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":22,"rank_sha256":"3ad5eee8852be5065365af9c25ff78ec2974ee89b2c6fa6759e27b7de255e202","ruler_category":null,"seed":null,"sequence_length":110},{"canonical_id":"894","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":23,"rank_sha256":"3c7238b06352863f75aab36214380eedfa02c7c388385142283f0980d21c1f7a","ruler_category":null,"seed":null,"sequence_length":226},{"canonical_id":"653","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":24,"rank_sha256":"3dd1d2a846dba10b3ffa7e554091ac65fe9284c15f2d83dc44c705155130a763","ruler_category":null,"seed":null,"sequence_length":287},{"canonical_id":"679","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":25,"rank_sha256":"3e59d101b564d5e3344a2e764584a194a5f663f3b20f7e939d7f719f84dacba4","ruler_category":null,"seed":null,"sequence_length":136},{"canonical_id":"657","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":26,"rank_sha256":"3e6b96b2be683d5d193895bbc05beb5a13bfc3b1db01872788e0ae8133c72b36","ruler_category":null,"seed":null,"sequence_length":171},{"canonical_id":"924","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":27,"rank_sha256":"45104189ef24ab437b12b531013325cf0a41819cf135255a4ca993f29f74dfd2","ruler_category":null,"seed":null,"sequence_length":103},{"canonical_id":"686","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":28,"rank_sha256":"47010c76b0b792ee5452a5eb55e8ba40d386f4a17d895b443eb1a66556c90596","ruler_category":null,"seed":null,"sequence_length":285},{"canonical_id":"860","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":29,"rank_sha256":"471b89737df3331c676e99243030a77140d460cde893f819d2c7916e0a16f453","ruler_category":null,"seed":null,"sequence_length":141},{"canonical_id":"698","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":30,"rank_sha256":"4a9f3f47f58cfaabd9afda89457088ca35397d7acc80568d8d218a723451116e","ruler_category":null,"seed":null,"sequence_length":359},{"canonical_id":"827","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":31,"rank_sha256":"4b073dd1bc9f2a79d60985c8d50aae572f2030e62a55477c64702d735d849023","ruler_category":null,"seed":null,"sequence_length":174},{"canonical_id":"651","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":32,"rank_sha256":"4bf2b2787c95b7b8a2f8ff089c36ea2662b4987dbf230b02b81c424382853b7b","ruler_category":null,"seed":null,"sequence_length":155},{"canonical_id":"930","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":33,"rank_sha256":"4ffef951fb9708b54468eaa75c7b42b9f58c198c366cfacd6b4f665c9d77984f","ruler_category":null,"seed":null,"sequence_length":138},{"canonical_id":"962","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":34,"rank_sha256":"51a6c027909f096835086346110e63a131c1462156020bb129058b320f27516a","ruler_category":null,"seed":null,"sequence_length":149},{"canonical_id":"853","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":35,"rank_sha256":"525322d1199e29de4998af2c95211cd981e642e9d602fe16554a8a63fc02d729","ruler_category":null,"seed":null,"sequence_length":219},{"canonical_id":"722","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":36,"rank_sha256":"52acb8744119319aaa1a68b3e192415c937a6822ceb2963069d4ede3e9b75cc4","ruler_category":null,"seed":null,"sequence_length":388},{"canonical_id":"705","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":37,"rank_sha256":"547e7ebc86a8992205db2eda5a7117796f15c3042e90200d556582e58f881c17","ruler_category":null,"seed":null,"sequence_length":258},{"canonical_id":"944","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":38,"rank_sha256":"577f992e266f3ffe7684e9483310b96df9b338bd836e00af79e0116821fddaa5","ruler_category":null,"seed":null,"sequence_length":117},{"canonical_id":"849","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":39,"rank_sha256":"59660871dc9200a74f124b924d6e85f743e541a39ae93af501968a7ed5d0d751","ruler_category":null,"seed":null,"sequence_length":161},{"canonical_id":"919","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":40,"rank_sha256":"5a21a04a8534d8dff43c2d02da1415ffaabed4a09662059b05e916108da7665e","ruler_category":null,"seed":null,"sequence_length":113},{"canonical_id":"629","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":41,"rank_sha256":"5aeb994e6b892f1741b894b66750ea3690b71585304601a66a17d599e3e96bc5","ruler_category":null,"seed":null,"sequence_length":150},{"canonical_id":"820","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":42,"rank_sha256":"6063af147dd771453be5c51054092d7aa85f751eab9d697cc1b77be9259ed7ff","ruler_category":null,"seed":null,"sequence_length":105},{"canonical_id":"627","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":43,"rank_sha256":"608304ac3176b904f89e46ddf06a578afe6f80161ceb1adb2b5049c5b31a8c5a","ruler_category":null,"seed":null,"sequence_length":198},{"canonical_id":"755","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":44,"rank_sha256":"60b435df2cb81d92c2de2d1edc7b06a50a523b8d40047dd521cf241475d6b370","ruler_category":null,"seed":null,"sequence_length":209},{"canonical_id":"911","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":45,"rank_sha256":"614cef397364dc767d15426e06989a10147811417996eb4fa91721038ecac80c","ruler_category":null,"seed":null,"sequence_length":236},{"canonical_id":"874","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":46,"rank_sha256":"62fc38937c372269a01f02df68b088b1fdc004f7ae4590a50cb3673f385737d0","ruler_category":null,"seed":null,"sequence_length":153},{"canonical_id":"918","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":47,"rank_sha256":"6326299b9eaf024dfc7efa6b26bb898a5b869898127d546067800b05ba8c5932","ruler_category":null,"seed":null,"sequence_length":229},{"canonical_id":"803","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":48,"rank_sha256":"65df1000f86d2eacc5ecb7d9b91504d3c6074a9e9c5e6cc8abebfc45c0f60974","ruler_category":null,"seed":null,"sequence_length":142},{"canonical_id":"868","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":49,"rank_sha256":"69fda125d9dfbbac0a44eaf6db26953f3f644a70389add149dd1c47e3ca90bf4","ruler_category":null,"seed":null,"sequence_length":137},{"canonical_id":"666","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":50,"rank_sha256":"6c8674e41d1ec38f03b2d176e08aef538902ff0c35234be50164674f4e52a341","ruler_category":null,"seed":null,"sequence_length":108},{"canonical_id":"768","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":51,"rank_sha256":"71cb4040b63cf195e0e702fd08f6b6e9eeef03a3de4a3c29ca0a715a42e2fbf6","ruler_category":null,"seed":null,"sequence_length":144},{"canonical_id":"636","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":52,"rank_sha256":"7357df8332514f1e676d0104bd545961a71f432580a835738fb62ad7dbc89065","ruler_category":null,"seed":null,"sequence_length":127},{"canonical_id":"671","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":53,"rank_sha256":"7a7a9eda4414dd86e6dbd5ebf5e1ac84df7df6e12e3242cb2574817453c497b8","ruler_category":null,"seed":null,"sequence_length":193},{"canonical_id":"783","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":54,"rank_sha256":"7aabfc7187ec019772ef6f0f0ae5203e0193909146a86ecdc6a1f789f86ff6c3","ruler_category":null,"seed":null,"sequence_length":428},{"canonical_id":"695","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":55,"rank_sha256":"7ce9ea381ac2a57a8373c1253b0dfbedb3b2e838c4dfd92bb0730f83c972434c","ruler_category":null,"seed":null,"sequence_length":168},{"canonical_id":"851","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":56,"rank_sha256":"7df3d4757b59a5be15e6e4b9208bfbdbf189f4590d7b2e93ca0f84047f2a97b4","ruler_category":null,"seed":null,"sequence_length":131},{"canonical_id":"945","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":57,"rank_sha256":"7fdc834d39787ed9f97e73645f7df6692b711d910c52482f0bbab1d41512f720","ruler_category":null,"seed":null,"sequence_length":132},{"canonical_id":"687","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":58,"rank_sha256":"80a7b335ef9b11db152007e896afdd070710cbee60d4766ef829826eacbbf56b","ruler_category":null,"seed":null,"sequence_length":153},{"canonical_id":"908","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":59,"rank_sha256":"8267d3b36347390e9dbe928d4f2af728975e565b8c89232c8fe011b762fb9e75","ruler_category":null,"seed":null,"sequence_length":175},{"canonical_id":"973","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":60,"rank_sha256":"85f2e19defa2f2194e4bd627c43470d94dfce10775f216d0eee0f5b7c89a8113","ruler_category":null,"seed":null,"sequence_length":107},{"canonical_id":"795","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":61,"rank_sha256":"874c549c5d365887eba08a0eac638427d2825d7bedc3da4b4aa6c213de86162a","ruler_category":null,"seed":null,"sequence_length":325},{"canonical_id":"822","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":62,"rank_sha256":"87ee9e94899a058de1bc21331e5ffecb56f11ce924750fc73a6a928a2501d281","ruler_category":null,"seed":null,"sequence_length":203},{"canonical_id":"728","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":63,"rank_sha256":"884e088f56e16b57342c6b5d6b1021b0f7e756e9d8d1720ce65c243eb1c95acf","ruler_category":null,"seed":null,"sequence_length":165},{"canonical_id":"895","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":64,"rank_sha256":"888165087ffe22467a3cefc2c892a649a93905d8f22cd2389a8c7ba0be70f967","ruler_category":null,"seed":null,"sequence_length":293},{"canonical_id":"764","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":65,"rank_sha256":"8981728482ffcdb96388f812ddfe70ae76010f928900d65b3f4a8c8490ed266b","ruler_category":null,"seed":null,"sequence_length":125},{"canonical_id":"804","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":66,"rank_sha256":"8996abd03167db364eb3cf521940939970ca3b49310060c371c0e447dca9f4d4","ruler_category":null,"seed":null,"sequence_length":134},{"canonical_id":"732","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":67,"rank_sha256":"89d9cbbf3458bcc72f432dd40166313978822085bc657d2cc2ff5fe6919c27a8","ruler_category":null,"seed":null,"sequence_length":134},{"canonical_id":"839","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":68,"rank_sha256":"8c48b54163b5a2b5d91d303abe4d2f5990fd2296dd47e1960208c1d0286d9450","ruler_category":null,"seed":null,"sequence_length":365},{"canonical_id":"823","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":69,"rank_sha256":"8df1ac0f53587cc228a0d68d4959d98a79d5c3046a25094f36d1698d013a1a7a","ruler_category":null,"seed":null,"sequence_length":205},{"canonical_id":"713","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":70,"rank_sha256":"8eb162011c59bc92c4728ac7fe2fd64ac0f7f78f7aef1ba048088ed9743e16d2","ruler_category":null,"seed":null,"sequence_length":121},{"canonical_id":"772","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":71,"rank_sha256":"8f08bc003b61873c2e9f27eceb15bf7a3a451a8f4877c759cc73b5f437a170af","ruler_category":null,"seed":null,"sequence_length":166},{"canonical_id":"727","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":72,"rank_sha256":"925293e3739b4a2498fb5d6c9cbf9fbd57520f22f99a7b3468fad6290361cf36","ruler_category":null,"seed":null,"sequence_length":161},{"canonical_id":"858","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":73,"rank_sha256":"935132e4b17a56013a0312175973c9a6b2b092902d5bf1d474b03a34da4953f6","ruler_category":null,"seed":null,"sequence_length":177},{"canonical_id":"800","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":74,"rank_sha256":"9400093971582810aed9b43834286c1bbcdbe15e12fd1f749b9e3a7abbefa8f4","ruler_category":null,"seed":null,"sequence_length":104},{"canonical_id":"751","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":75,"rank_sha256":"96679ba0da36bb95058654f8fea49ff5305709d80cd95a1017d570f0d7997ddf","ruler_category":null,"seed":null,"sequence_length":243},{"canonical_id":"674","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":76,"rank_sha256":"96c806e69e6b65095699a2192e225cc22ad2b21211dbb8eae36893932ef358e1","ruler_category":null,"seed":null,"sequence_length":128},{"canonical_id":"960","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":77,"rank_sha256":"97692bbfd642b241043c3eb82fd110341e33df034bcc65a60b79b26933a15b3e","ruler_category":null,"seed":null,"sequence_length":124},{"canonical_id":"867","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":78,"rank_sha256":"979109f8d629072c0302399d65fa7263972fa9ed22cefff82f4c11028507238f","ruler_category":null,"seed":null,"sequence_length":170},{"canonical_id":"669","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":79,"rank_sha256":"98ddb8c36da9cd7adc3852209dcda61e9da6d3acd0d1fcf1211f25ae04c07cc5","ruler_category":null,"seed":null,"sequence_length":296},{"canonical_id":"656","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":80,"rank_sha256":"9979d6795117639a4d1f8afae4ddfa11dfeeefbf2ca8687a7e67b84643a1744f","ruler_category":null,"seed":null,"sequence_length":169},{"canonical_id":"862","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":81,"rank_sha256":"99e8fb364eec07a688b1d0dbc74ba9372c47719c0193b304c88bc2659ecd75c4","ruler_category":null,"seed":null,"sequence_length":177},{"canonical_id":"635","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":82,"rank_sha256":"9cf09b9e19663d0a700e97d8d627776519ef82d0d8608b62a5645b11309ec58f","ruler_category":null,"seed":null,"sequence_length":266},{"canonical_id":"756","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":83,"rank_sha256":"9d6ace56a87e526ec18af2d10e07c6dd1a3287a2d8271415ac4a0c3a3f642035","ruler_category":null,"seed":null,"sequence_length":139},{"canonical_id":"869","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":84,"rank_sha256":"9d975c8aee95cb7e1e54deeee4073d57a63712f67247e5fb61dcf791dbb16564","ruler_category":null,"seed":null,"sequence_length":352},{"canonical_id":"857","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":85,"rank_sha256":"9e00a569dd1098d4912c32f57e2351709bf3ffce9a75edf00179d5f023c41d41","ruler_category":null,"seed":null,"sequence_length":293},{"canonical_id":"607","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":86,"rank_sha256":"a18910fa12a223a0ecb7d6707500711e3b376af22c72b99625792f523b548d89","ruler_category":null,"seed":null,"sequence_length":207},{"canonical_id":"929","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":87,"rank_sha256":"a39d16127e92e6eb192ad63a45fa7db3125509f3a86fcc647d0a72ffdfc25483","ruler_category":null,"seed":null,"sequence_length":168},{"canonical_id":"606","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":88,"rank_sha256":"a4d325b0ce1088bd97fbaac9299e2d0dbb32b002026f2528825225d44f1e1e60","ruler_category":null,"seed":null,"sequence_length":144},{"canonical_id":"662","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":89,"rank_sha256":"a572f1763ed2dacfc5b6f8bafb04903bbc369519be95c8ecfd7d26a2792dc619","ruler_category":null,"seed":null,"sequence_length":350},{"canonical_id":"877","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":90,"rank_sha256":"a7576eaa33a3fb7087c29ae14949e5bfc2e1ef4449b997384e88a2d5dd631591","ruler_category":null,"seed":null,"sequence_length":93},{"canonical_id":"946","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":91,"rank_sha256":"a884dbdb72cb7d11613027848d08702a496c419bc5c987e9e963e2184c774bd8","ruler_category":null,"seed":null,"sequence_length":209},{"canonical_id":"785","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":92,"rank_sha256":"ab824ea47e78d00ae4aec6a45480e122e3703f1da3037053af0aa7a891b637ee","ruler_category":null,"seed":null,"sequence_length":152},{"canonical_id":"920","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":93,"rank_sha256":"ad357f573b8f19c5b89f84aa9097d23ace8d1488e9cb2c1322577137d0d53c2a","ruler_category":null,"seed":null,"sequence_length":251},{"canonical_id":"616","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":94,"rank_sha256":"b09c89c6a2366a5a51caa203e1f363d01e8340abcd1ca162839843b86eef1161","ruler_category":null,"seed":null,"sequence_length":221},{"canonical_id":"840","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":95,"rank_sha256":"b2be594c39afb592547a351b8235b8061e103c0c75d777ad1549d2e015a87989","ruler_category":null,"seed":null,"sequence_length":131},{"canonical_id":"644","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":96,"rank_sha256":"b6557529f8ec5764b47cd34b0b579eb20371c1c581831a55709bb7d1b27348af","ruler_category":null,"seed":null,"sequence_length":186},{"canonical_id":"704","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":97,"rank_sha256":"ba157c706c3d65835a4022b531b73e3f1b65c1660d0699fd7293821d2ebfe350","ruler_category":null,"seed":null,"sequence_length":153},{"canonical_id":"712","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":98,"rank_sha256":"bb6cd04968145b1904af1e96e70b74699cb7138a72c4c62e0b82ee87a3375fc0","ruler_category":null,"seed":null,"sequence_length":219},{"canonical_id":"882","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":99,"rank_sha256":"bf3e15adb4c326aab0f479085d1e9971db834476e9ecaf1a9c3d7764fec4378b","ruler_category":null,"seed":null,"sequence_length":112},{"canonical_id":"710","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":100,"rank_sha256":"bf5d27c709dca8c30c14453c4e20b7c75f1e93bd5952da6bbc741e43a68f0575","ruler_category":null,"seed":null,"sequence_length":163},{"canonical_id":"821","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":101,"rank_sha256":"c0ab0a4bb636c134ba2233ecc3d1925c81df2a89060c09a2908c9dddbc14b591","ruler_category":null,"seed":null,"sequence_length":282},{"canonical_id":"707","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":102,"rank_sha256":"c43bbdda480f2ebabe97769f2958f17a19febca5a53140948914ad1bdd0a6e3e","ruler_category":null,"seed":null,"sequence_length":200},{"canonical_id":"614","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":103,"rank_sha256":"c5450b1ef19a8eaf7e4710ad9115e76118a4634fb769d3924067bdec342e2e70","ruler_category":null,"seed":null,"sequence_length":165},{"canonical_id":"741","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":104,"rank_sha256":"c7c6fbada9b7a2d3d38750bc16cb99ea97efb71c0413563d3f399cc0722c6bcd","ruler_category":null,"seed":null,"sequence_length":124},{"canonical_id":"702","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":105,"rank_sha256":"c8a56538eebae04a29a21ac3c638e71a9ea53a1fdc12603034b61d9ef4359cc9","ruler_category":null,"seed":null,"sequence_length":332},{"canonical_id":"824","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":106,"rank_sha256":"c9106dec19a3fa31e33d340927464d15b6150cbc251a6aff64336706d0400b98","ruler_category":null,"seed":null,"sequence_length":129},{"canonical_id":"809","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":107,"rank_sha256":"ca59973992bc52f67689974ae41022e87d598c8c31de31955f30bfaf21889c3f","ruler_category":null,"seed":null,"sequence_length":173},{"canonical_id":"886","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":108,"rank_sha256":"cc1cdebfc08022bfe9fae59cf187db791a655ca5b48463b8c2725e74349fa708","ruler_category":null,"seed":null,"sequence_length":152},{"canonical_id":"789","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":109,"rank_sha256":"d71db894fab22f0f681f7e02a1ec80d922272c111bb9462b9d066866ab601c18","ruler_category":null,"seed":null,"sequence_length":98},{"canonical_id":"870","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":110,"rank_sha256":"dacc01a92b59eb20074bd1a8255fdbbc3298f37030e34e3d4e074c0bb66134ba","ruler_category":null,"seed":null,"sequence_length":208},{"canonical_id":"854","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":111,"rank_sha256":"dd5f39a3a28901c30d626c95276606c2a6ad00b0f64384c782406f9a13ef8d1e","ruler_category":null,"seed":null,"sequence_length":235},{"canonical_id":"646","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":112,"rank_sha256":"deb0492a1dc5a716095faeaf6d418cf2bf74040a83940a2690a6ea83442404de","ruler_category":null,"seed":null,"sequence_length":129},{"canonical_id":"605","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":113,"rank_sha256":"e161ff3b2e28c4f02462643895bc6e95b48655efc7ac0ace9eec5c4226576cb0","ruler_category":null,"seed":null,"sequence_length":125},{"canonical_id":"902","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":114,"rank_sha256":"e1ebf3f6386478a8625ad8c53858193b08d3e9a3ec267789cd98e85c340f314a","ruler_category":null,"seed":null,"sequence_length":310},{"canonical_id":"848","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":115,"rank_sha256":"e2bc9c9a9b23e33f1abf22dce38357e1e5c10acff55d2a208423270bc02a7289","ruler_category":null,"seed":null,"sequence_length":129},{"canonical_id":"847","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":116,"rank_sha256":"e351b9058c9e5c2e9e58444e1e4f2346c4886c6705652807fc2baab4f1806bd0","ruler_category":null,"seed":null,"sequence_length":143},{"canonical_id":"630","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":117,"rank_sha256":"e553afed5e6fb0d6843dfe2f22bdb6aa65d8f483b406e1b83ee5a2bf2e87fba2","ruler_category":null,"seed":null,"sequence_length":327},{"canonical_id":"793","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":118,"rank_sha256":"e76c7363df6aa1bca2a659469266d4228a66b4a2e10bf91c06346d0eaf0da89e","ruler_category":null,"seed":null,"sequence_length":209},{"canonical_id":"725","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":119,"rank_sha256":"ebb93bbe303910a142ceeda734226a9701884eaee00b68852b557cfc23fc231b","ruler_category":null,"seed":null,"sequence_length":164},{"canonical_id":"903","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":120,"rank_sha256":"ed33b084d9852e0df6989759582108080423613a5014d3f1150c3cbbc0d60573","ruler_category":null,"seed":null,"sequence_length":161},{"canonical_id":"797","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":121,"rank_sha256":"edb3a9f5d11a94af9db154ef9fee83d36402a77de27eb0ba830a3902dca5536c","ruler_category":null,"seed":null,"sequence_length":147},{"canonical_id":"792","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":122,"rank_sha256":"ee24c9054290bd8ccfeccee4993ac8dc6e0e20c72cdf4fbe5d28593c947a2246","ruler_category":null,"seed":null,"sequence_length":130},{"canonical_id":"615","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":123,"rank_sha256":"eed8e28b1d920cec5525bff2f0f861ca6e43429ed75ae657e5754ce34b81d71a","ruler_category":null,"seed":null,"sequence_length":338},{"canonical_id":"640","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":124,"rank_sha256":"f08156d84d873d2ae9184c8db61828a6c9a15f94b5a43b4ef8fe5c1867c2f16d","ruler_category":null,"seed":null,"sequence_length":118},{"canonical_id":"709","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":125,"rank_sha256":"f94bcd46daf0c1bd69bc60844bac9acd167ddc7981f10db96385b6d755574ec2","ruler_category":null,"seed":null,"sequence_length":390},{"canonical_id":"846","config":"full","configured_length":null,"group":"mbpp","half":"a","rank":126,"rank_sha256":"fb81297eb57060d653be5ab5aa61ab169ac4bb236168fbda838bc333f61d4bc7","ruler_category":null,"seed":null,"sequence_length":305},{"canonical_id":"936","config":"full","configured_length":null,"group":"mbpp","half":"b","rank":127,"rank_sha256":"fefd4316fc5fd98157422b169451622b064b3709b67d87d78e00278992813ea2","ruler_category":null,"seed":null,"sequence_length":297},{"canonical_id":"http://www.gutenberg.org/ebooks/1219","config":"default","configured_length":null,"group":"pg19","half":"a","rank":0,"rank_sha256":"061889c1d506cd1d5fb9d49ee4d63712d7e12e6056ffcc25a15ed400766bb5b8","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/38160","config":"default","configured_length":null,"group":"pg19","half":"b","rank":1,"rank_sha256":"10e82e9edc65238326568f332ff5556646b86707e652e24b051769eb8ad7fb02","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/39447","config":"default","configured_length":null,"group":"pg19","half":"a","rank":2,"rank_sha256":"14037bdad952830661d418776aaf4616905c877bfdb291977c357381de5c229c","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/29413","config":"default","configured_length":null,"group":"pg19","half":"b","rank":3,"rank_sha256":"217a13c587e9ae2e779aae14a77fa4a49dbbcbea34bfd717de53c2ac37ea9675","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/1253","config":"default","configured_length":null,"group":"pg19","half":"a","rank":4,"rank_sha256":"2a24d338d8456351047da3a9c12b7feb6c9e39c040ea62275c9284d4bc3b02f6","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/35786","config":"default","configured_length":null,"group":"pg19","half":"b","rank":5,"rank_sha256":"5b4d0ebce948cb01a07e9166f2db8c4d3163c3ce8780be96639120dc13fb26e2","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/17774","config":"default","configured_length":null,"group":"pg19","half":"a","rank":6,"rank_sha256":"5cfbeaf96d6b8bf5cbe8754a74cd22b63225e5c1775594c88f7ff24fc3be1d90","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/10609","config":"default","configured_length":null,"group":"pg19","half":"b","rank":7,"rank_sha256":"73a499a3165431d2fcf4fe5184ef91447b8184d5bf8f0683adb49a4bf058e0fb","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/20279","config":"default","configured_length":null,"group":"pg19","half":"a","rank":8,"rank_sha256":"7da541ed6b56ea9ddd7749aef82c4f66b36beb7ca9ebd5b3ab2623c60996cac9","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/28065","config":"default","configured_length":null,"group":"pg19","half":"b","rank":9,"rank_sha256":"beee6f7ab98cda2f3595fe22f5221f16fd4ce8717a7fe77344ea7e851c85287f","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/24903","config":"default","configured_length":null,"group":"pg19","half":"a","rank":10,"rank_sha256":"c4b007d21ff91cbeb1959618d1f954e864dc7b874ae3986598c860a30b425cc0","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/33622","config":"default","configured_length":null,"group":"pg19","half":"b","rank":11,"rank_sha256":"dcca53b6159ae6063a9e62f780ecd4130f39bf20f2342fe194d39248032cb43d","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/26008","config":"default","configured_length":null,"group":"pg19","half":"a","rank":12,"rank_sha256":"e5734f2587660eccf1f1cae256502a6d7f22fa8d8898f4dee591a3e3a931de27","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/36808","config":"default","configured_length":null,"group":"pg19","half":"b","rank":13,"rank_sha256":"e9713427a3a331bb20abf5840a4a40e37e72569131f89b7ab0691653bc679901","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/2773","config":"default","configured_length":null,"group":"pg19","half":"a","rank":14,"rank_sha256":"f0e4622ba87c3913d530c0f8f7f20c419b2375ff1ca073e6062914dbabe1a4f5","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"http://www.gutenberg.org/ebooks/32024","config":"default","configured_length":null,"group":"pg19","half":"b","rank":15,"rank_sha256":"f8ecebbe43befef18b00a5780b7383b2a5e3971aca73ed9caf82dd87b7be9b58","ruler_category":null,"seed":null,"sequence_length":2304},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:fwe:length=2048:seed=12339:sample=0","config":"fwe","configured_length":2048,"group":"ruler:aggregation","half":"a","rank":0,"rank_sha256":"407456144cdf7e7f879eeb8c590fc8039a5de9c5d2a99e094f58c354dc7a2e14","ruler_category":"aggregation","seed":12339,"sequence_length":1951},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:cwe:length=4096:seed=12340:sample=0","config":"cwe","configured_length":4096,"group":"ruler:aggregation","half":"b","rank":1,"rank_sha256":"6eed2f87558224dfd68001e835c96568c05e5f827c533e157752f375f4500e19","ruler_category":"aggregation","seed":12340,"sequence_length":3576},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:fwe:length=4096:seed=12339:sample=0","config":"fwe","configured_length":4096,"group":"ruler:aggregation","half":"a","rank":2,"rank_sha256":"9ba9392d1dc6140c8a464c6ab01eae04fd883d8520b997db8d8e7a81ec69412a","ruler_category":"aggregation","seed":12339,"sequence_length":3987},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:cwe:length=2048:seed=12340:sample=0","config":"cwe","configured_length":2048,"group":"ruler:aggregation","half":"b","rank":3,"rank_sha256":"acfea4ad2d1ba514b26bd0835657efe402f76ae3d519c669004fe8f3398c41ba","ruler_category":"aggregation","seed":12340,"sequence_length":1862},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=2048:seed=12339:sample=0","config":"vt","configured_length":2048,"group":"ruler:multi_hop_tracing","half":"a","rank":0,"rank_sha256":"089158a73f7f141394a585c20b3642a57e30b83534a971bffe8d8cc58d1d88a0","ruler_category":"multi_hop_tracing","seed":12339,"sequence_length":2011},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=4096:seed=12339:sample=0","config":"vt","configured_length":4096,"group":"ruler:multi_hop_tracing","half":"b","rank":1,"rank_sha256":"0c5867a34d4c0109a91512496cb821eeda3eacfff858dc77cce30e9fdd0dec20","ruler_category":"multi_hop_tracing","seed":12339,"sequence_length":4056},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=2048:seed=12340:sample=0","config":"vt","configured_length":2048,"group":"ruler:multi_hop_tracing","half":"a","rank":2,"rank_sha256":"47daa45868dba636bc69249fc76653ed16cc4674127f6229ffb7b5c6e0dc816b","ruler_category":"multi_hop_tracing","seed":12340,"sequence_length":2009},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=4096:seed=12340:sample=0","config":"vt","configured_length":4096,"group":"ruler:multi_hop_tracing","half":"b","rank":3,"rank_sha256":"cea1514b80ab90b346af0afd72d4e596f64be107a08c70781a292c2824193d97","ruler_category":"multi_hop_tracing","seed":12340,"sequence_length":4057},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_2:length=2048:seed=12340:sample=0","config":"qa_2","configured_length":2048,"group":"ruler:question_answering","half":"a","rank":0,"rank_sha256":"362ee5e975e3602b3afddead8d433435ec23dbad4defa23337e192b538091cbc","ruler_category":"question_answering","seed":12340,"sequence_length":1893},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_2:length=4096:seed=12340:sample=0","config":"qa_2","configured_length":4096,"group":"ruler:question_answering","half":"b","rank":1,"rank_sha256":"4e8e5919e5dec4b8c1e7aed66e68abf8414441353d6669352f7bb5eb6f3560ba","ruler_category":"question_answering","seed":12340,"sequence_length":2281},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_1:length=2048:seed=12339:sample=0","config":"qa_1","configured_length":2048,"group":"ruler:question_answering","half":"a","rank":2,"rank_sha256":"743c29a00e4cbe2d6bb62a01300b447fc3a78ded5878e9c03e7d80686d62e6c3","ruler_category":"question_answering","seed":12339,"sequence_length":1368},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_1:length=4096:seed=12339:sample=0","config":"qa_1","configured_length":4096,"group":"ruler:question_answering","half":"b","rank":3,"rank_sha256":"ec18351546c72bc1cad5f950ced4793e70ee2fa618a341924278809a62774186","ruler_category":"question_answering","seed":12339,"sequence_length":4040},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_single_1:length=4096:seed=12339:sample=0","config":"niah_single_1","configured_length":4096,"group":"ruler:retrieval","half":"a","rank":0,"rank_sha256":"4a756a6ea055639254af12c8568b7afc07bb626029548efa51a12d72a0109efb","ruler_category":"retrieval","seed":12339,"sequence_length":3955},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multiquery:length=2048:seed=12339:sample=0","config":"niah_multiquery","configured_length":2048,"group":"ruler:retrieval","half":"b","rank":1,"rank_sha256":"5ac13727e634d2432f98b2670f23761bdef39b61ebfadcecff6aa6e1dc1966a9","ruler_category":"retrieval","seed":12339,"sequence_length":1289},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multikey_2:length=2048:seed=12340:sample=0","config":"niah_multikey_2","configured_length":2048,"group":"ruler:retrieval","half":"a","rank":2,"rank_sha256":"8deff28bb1380b3aa1be3b678ea1793381004620e9b4a67fea888e2a859cb53c","ruler_category":"retrieval","seed":12340,"sequence_length":1828},{"canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multivalue:length=4096:seed=12340:sample=0","config":"niah_multivalue","configured_length":4096,"group":"ruler:retrieval","half":"b","rank":3,"rank_sha256":"b13ed20ccd6d9f0502b4d16468a25828441ad0e8b8e8063b1dd3f5d674a1815f","ruler_category":"retrieval","seed":12340,"sequence_length":3964}],"assignment_sha256":"a42cf4b332cc8cf58b27709d7d261fc03a356b27ec1c9ccd56914d99e60c1797","namespace":"recurquant.experiment013.calibration-split.v1\u0000"},"claim_boundary":"This artifact freezes Experiment 013 data, tokenizer, source, runtime, and metadata-only model-file identity. It is not quality, latency, novelty, state-of-the-art, deployment, or breakthrough evidence.","content_manifest_sha256":"ee72483a8f8b4370c9e667e4287747e5bc358aeb0265a58167140f4e780a7b29","datasets":[{"canonical_id_field":"task_id","canonical_id_manifest_sha256":"f46a3c772f1416c75bdd8668c69f4886a32a243c1737b13bbddfdc23b9605c70","config":"full","dataset_id":"google-research-datasets/mbpp","formatter_id":"recurquant.mbpp-prompt-code.v1","formatter_sha256":"882e20ec9f5cbcb7e6f1310cbf46d19153721beac41ecc7ee308c39be17532ff","key":"mbpp","revision":"4bb6404fdc6cacfda99d4ac4205087b89d32030c","split":"train"},{"canonical_id_field":"url","canonical_id_manifest_sha256":"8d7ed934210f5e78f13559cdf48518b922d3c203df9258d0de3c4d62a18bdaa7","config":"default","dataset_id":"emozilla/pg19","formatter_id":"recurquant.pg19-token-slice.v1","formatter_sha256":"faea2480bf85adcd34339cae88a0e9b631b705eb547020572db74402bf525730","key":"pg19","revision":"c021754c8e01c5b1cc83a1f549c1f97fbbb756b8","split":"train"},{"canonical_id_field":"configuration_id","canonical_id_manifest_sha256":"d7fec5f2a57409fb842dcb967de5ed1efce00253723f2d891b28681b8d57952d","config":"official-generator","dataset_id":"NVIDIA/RULER","formatter_id":"recurquant.ruler-official-generated-record.v1","formatter_sha256":"4f797cbca4dfbe802c47dc5b238111b629f1497e46c84ea116d06319012016b6","key":"ruler","revision":"c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a","split":"generated"},{"canonical_id_field":"task_id","canonical_id_manifest_sha256":"c53bc1db6c7965cb13d4238321ef09c0c7b3dbfeb4f4736bfd6e46e2914449da","config":"default","dataset_id":"evalplus/humanevalplus","formatter_id":"recurquant.humaneval-plus-prompt-solution.v1","formatter_sha256":"12204389715ddc210e5a8b1b291f4fbcaf2b64fde94c22699873de80d682204c","key":"humaneval_plus","revision":"d32357cf319e50e9c8d8dab5ea876c72b0fd321b","split":"test"}],"execution_bindings":{"calibration_runtime_manifest_file_sha256":"80ca233a29af4facbb334fd4fb51a4f6e9a3d6815465cb79b9f3db63ef668d6a","model_file_manifest_file_sha256":"586d9c7e520f3bbd99ecef30663bf07d283eb14622475c58891becd8e033b05c","parquet_materialization_manifest_file_sha256":"ee5628e50e5d3516fd79077542d355fd915455ac0e53128d372f4177ad63d39c","repository_source_manifest_file_sha256":"c07e867ed912f903d839683ca62b09265e9b6d237e5b1024476fd36fc3b211f6"},"identity_only":true,"identity_schema":"recurquant.experiment013.identity-frozen.v5","model_contracts":{"conditional_scale_check":{"cold_start_peak_hbm_limit_bytes":8053063680,"id":"Qwen/Qwen3.5-2B-Base","revision":"b1485b2fa6dfa1287294f269f5fb618e03d52d7c"},"primary":{"id":"Qwen/Qwen3.5-0.8B-Base","revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68"},"weights_loaded":false},"phase":"calibration","promotion":{"candidate_canonical_evidence_sha256":"0fe46f9a193d5c2c68895a1c69598c9fadc4e7cafa1d676fda06b3c95a44a129","candidate_file_sha256":"2cec5fb9868b4b61550d10fce3a869ad45e0b4b2d56df5f6c2472cb7d17c3794","explicit":true},"promotion_required":false,"protected_identity":{"ordinary_tests_may_read_protected_content":false,"stage_b_read":false,"stage_c_read":false},"record_count":160,"records":[{"anchor_manifest_sha256":"9727dad06649abe3aad4a15772fd49b9a0ae5c0ef55e61bba6ccfd0445ab8f73","anchor_positions":[7,15,23,32,40,48,56,65,73,81,89,98,106,114,122,131],"anchor_positions_sha256":"24769466c0c30738992bfaea31b322a83686427b8c8c2f42d5977ce98da1e86f","canonical_id":"945","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,31,39,47,55,64,72,80,88,96,104,112,120,129],"fisher_boundary_sha256":"98d4c6b32acad28c83342cab67b79f665cc5144d363b8e6653d190aedaddc45f","horizon":1,"input_positions":[8,16,24,32,40,48,56,65,73,81,89,97,105,113,121,130],"input_token_ids_sha256":"5d42f41e2436cb702ed1f8eb971b20537d123d7e1f41409244fd8dd8129656fc","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,33,41,49,57,66,74,82,90,98,106,114,122,131],"target_token_ids_sha256":"9a4cf735c9ab0560b97058fccd7a2d08e1a7cc3d126701d11007e641f736719b"},"formatted_content_sha256":"aa6feb35c931bc49027c89402eb6fe154ef150b92030a5f3e730756026205a4c","generator_receipt_sha256":null,"identity_record_sha256":"3c49ec4e6136f525892d68d4cf7267b90d1836a9011dc55e73ece42bc882546e","prompt_token_ids_sha256":"5766712b5c83365901b89092414948a16c2a894641c0580978a6720d74c7bb4b","ruler_category":null,"seed":null,"selection_rank":0,"selection_sha256":"006481602de32e6484f069e30640315b4902f198bc2b977318b0a90084e2503d","sequence_length":132,"sequence_token_ids_sha256":"91c8172e893e28a3e7c965916084be5fd76619bd9dc7881f155bfceacf710738","source_content_sha256":"dbd241865f0bf98d18ba09369665294382ef86e1be0440c9b18c083d5cb8529d","target_token_ids_sha256":"16acd83d1b28de14a0fcfcbad7f9a2538e0df11843ba645c1a55bcfeb285cecb","token_span":{"cache_exposed_start":132,"cache_exposed_stop":132,"prefill_start":0,"prefill_stop":112,"scored_start":112,"scored_stop":132},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2efa60ce513767dd377aebb493099ff3ebdc8a0362f30593887e54997aaa3ffc","anchor_positions":[8,18,27,37,47,56,66,76,85,95,104,114,124,133,143,153],"anchor_positions_sha256":"e227da918c51f69b24ec0ca4e8d85817a5330fb6e593d88068dce4e00d5d6d1c","canonical_id":"794","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,18,27,37,46,56,65,75,84,94,103,113,122,132,141,151],"fisher_boundary_sha256":"f690f6f75e34bb40e62a6b7fc6b3bb68573fc9ae30bacc4e60a641b17ec90391","horizon":1,"input_positions":[9,19,28,38,47,57,66,76,85,95,104,114,123,133,142,152],"input_token_ids_sha256":"bf4865a5ba31ed871b7bd8e3db64daa50d06ca08cbecab0fe5005d6b1a8347ad","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,20,29,39,48,58,67,77,86,96,105,115,124,134,143,153],"target_token_ids_sha256":"68567590e4faf2baa82a41a27758544c2001b1f8b881cbca709b0b85b1bb0ad4"},"formatted_content_sha256":"d8535b9c9deb650f87372133fd68f93792a2190a2ef9fe56ca55a611be6956e0","generator_receipt_sha256":null,"identity_record_sha256":"19a48bcd8ab87c826d9cf5e9160f127b2f1f1be8e45a068c8e44863d1648ee1e","prompt_token_ids_sha256":"b21de4f99224333b5f102dbb403f23e8367304e199e27380c91e192867b7cc1b","ruler_category":null,"seed":null,"selection_rank":1,"selection_sha256":"0172341ebe7bf74bb15297781b17837005ed9a9efbcb89d7bac34d2b25000aba","sequence_length":154,"sequence_token_ids_sha256":"b4da2809a0dead622eddd5200a89a8d4ffcc43b2c108c805e70f4449f813d222","source_content_sha256":"27e38eda1acb40e360eceaac200a88d7b2850555fa05a1efc123ec8c4a915f21","target_token_ids_sha256":"348ffb18662c2da5694709279e9caf22da5687654401cf9aa365e6382bba0ba8","token_span":{"cache_exposed_start":154,"cache_exposed_stop":154,"prefill_start":0,"prefill_stop":101,"scored_start":101,"scored_stop":154},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e0ef0af8afe4b6c4a81f3cabbd81a8ee7f213579fe41ab99f6103d4b0219b7e6","anchor_positions":[9,20,31,41,52,63,73,84,95,105,116,127,137,148,159,170],"anchor_positions_sha256":"fa5efe05ea2fd4bb11defcbff4d3edc28a1c8dc11911a2450816825e972052d2","canonical_id":"657","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,30,41,51,62,72,83,94,104,115,125,136,146,157,168],"fisher_boundary_sha256":"68596f206280b567de8e4d47c206f7bf68a9bb74b4435cf78f400f9f1ab7d29d","horizon":1,"input_positions":[10,21,31,42,52,63,73,84,95,105,116,126,137,147,158,169],"input_token_ids_sha256":"e3067d1464515f4ac7220f562d34c1c90289997d9fb5fefc653bd53f5f81f18e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,32,43,53,64,74,85,96,106,117,127,138,148,159,170],"target_token_ids_sha256":"a85ac1c61455daf925b47063d1ff1f66b29773ceec176e8263f5e6f25a1b3ea1"},"formatted_content_sha256":"058b7f8c5600efa023e5b2daf39dfaf3c3ed7a333e538e1af40e7b0b66c74221","generator_receipt_sha256":null,"identity_record_sha256":"53d315ac0c1c29a162fde0f0fcec7febb3700ec10fef28e6ed4749c118c383e8","prompt_token_ids_sha256":"d229c54edd42db7f254ae246f41c7e3e4207ee238c734a8f628e2b15ef6d2ea8","ruler_category":null,"seed":null,"selection_rank":2,"selection_sha256":"01c4ac54c87a838a046990fc691e469914fa8fffc3bd5230e6a5954f963ce09b","sequence_length":171,"sequence_token_ids_sha256":"2cc570399bb13417211fedf4c35131117f7d22b47aa4bd94e603f6303bbecf3d","source_content_sha256":"87238f15d4968d72a1caf68b3a7d6d09014ca1cd79cd8a42053ba1d0df51fe15","target_token_ids_sha256":"470c291e9349adfcd9620fddf232ec6756cdb6300d975612d2a94ad69e7deca4","token_span":{"cache_exposed_start":171,"cache_exposed_stop":171,"prefill_start":0,"prefill_stop":76,"scored_start":76,"scored_stop":171},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1a950a1c62dc0a9cadc655d3670aa54a1e6cb647d9a1892093d06959656b3aa9","anchor_positions":[19,40,61,82,102,123,144,165,185,206,227,248,268,289,310,331],"anchor_positions_sha256":"42314f586870267235ff8a21de13271c7bc12dbde0efe06732f5f12b3dd32240","canonical_id":"702","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[19,40,60,81,102,122,143,164,184,205,225,246,267,287,308,329],"fisher_boundary_sha256":"1c176e9d1164044fc7633185803af174aed6b6dde3205b6a93344da5dde70a67","horizon":1,"input_positions":[20,41,61,82,103,123,144,165,185,206,226,247,268,288,309,330],"input_token_ids_sha256":"6439978ab1e3336355d0e247c88f80b5fe217a10e6f08dbf7d49232dc4526902","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[21,42,62,83,104,124,145,166,186,207,227,248,269,289,310,331],"target_token_ids_sha256":"1e85d0fddb9002f2699f987057a43eb9c49e9fa5ef1f342f1e03060dd826b7df"},"formatted_content_sha256":"154dfb8b73bb69a8b67592d334c059c4ce3fa2a8dfe9716538e93896567d0906","generator_receipt_sha256":null,"identity_record_sha256":"08e19720136cb0b9e54d61f609f904ecacefee69429d33d7a9794d8b46085bf5","prompt_token_ids_sha256":"bddadf4e931724e7fd559f3f6b3d7128e8ca27b120efc2124cf05a63e7b2f6c4","ruler_category":null,"seed":null,"selection_rank":3,"selection_sha256":"01d2ae83b6bab88a4805a43141ed140fa1eb40a0d24f5579edffc9db44efe0e9","sequence_length":332,"sequence_token_ids_sha256":"91c517b2851f293a1c925d44556ecdd49cf6fc1874d095521f4dba829c5d5237","source_content_sha256":"97404d0bd18d3e30402ad3b46bee420986959afd67ef272b725a56c88634eec4","target_token_ids_sha256":"c94cc439fcffac9c155039ed067b4cf6d4dccca08e579f728aaf004b72be2d8e","token_span":{"cache_exposed_start":332,"cache_exposed_stop":332,"prefill_start":0,"prefill_stop":151,"scored_start":151,"scored_stop":332},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3ed130f8f695b8d6eb298a7742bd1582483350b3fd739da368175309b21ef0b1","anchor_positions":[8,18,28,37,47,57,66,76,86,95,105,115,124,134,144,154],"anchor_positions_sha256":"dda22fb55d9a1168e0db21fb1e43033ab66ab6d1c06af8fc27c118e4671afeeb","canonical_id":"651","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,18,27,37,46,56,65,75,85,94,104,113,123,132,142,152],"fisher_boundary_sha256":"9efbc09f1ef4234ec391fd7fee3a2a47a03dcf28666c0aa550bfc04ea6bcde0c","horizon":1,"input_positions":[9,19,28,38,47,57,66,76,86,95,105,114,124,133,143,153],"input_token_ids_sha256":"b5b770374e871c6af58aaed89d7fdf049c3423bc742e960e5f9a6f4e991faf81","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,20,29,39,48,58,67,77,87,96,106,115,125,134,144,154],"target_token_ids_sha256":"8271372b97b8f4205c2218769701468573205610c8d62b3474ad0d357b160840"},"formatted_content_sha256":"1ccb4c308ca379db3003d12f2a08486a59d19d9a104a1208f6f91af5104409dd","generator_receipt_sha256":null,"identity_record_sha256":"aa519005f371474709ebcac572071735625e58adcc2ea5484d987a610bd03f36","prompt_token_ids_sha256":"d3dc05d28fe582fdfab921a7ad3bac6d80d1d9368804e5955561df1f18f29e1d","ruler_category":null,"seed":null,"selection_rank":4,"selection_sha256":"030870e9654f434f272f49b61535aad38a9c0eacc093ccedb4b1328a163d00f7","sequence_length":155,"sequence_token_ids_sha256":"b4fc5ea0199d736cffaba0bfb4e5db058abd8417d844063350138206ca085d62","source_content_sha256":"c093103b7e2b0fc0cff523616e4dd14264349d2add37f8cc2e248dbd630be4f3","target_token_ids_sha256":"39e1f51bdd6ad42a474c0dde935c2ac6a6696b58b1ccc288e77f6bb849d587ca","token_span":{"cache_exposed_start":155,"cache_exposed_stop":155,"prefill_start":0,"prefill_stop":117,"scored_start":117,"scored_stop":155},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"30f57c28b1d21a916283d0e281b04bbb0087858a98279771835d257c5187478a","anchor_positions":[17,35,53,72,90,108,126,145,163,181,199,218,236,254,272,291],"anchor_positions_sha256":"8d9aa0aedd74a7302174328c9e98110efb9351fb25a7e9f001ca338898298fc3","canonical_id":"720","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,35,53,71,89,107,125,144,162,180,198,216,234,252,270,289],"fisher_boundary_sha256":"c47ca1eef3f1024a8ccaaa6a3156efd81373c0981d1b7794d0e74a6a4596fa0e","horizon":1,"input_positions":[18,36,54,72,90,108,126,145,163,181,199,217,235,253,271,290],"input_token_ids_sha256":"699513d81a38caf1a3166032551a6daa9d12868c8369d97465ec8d642a321141","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,37,55,73,91,109,127,146,164,182,200,218,236,254,272,291],"target_token_ids_sha256":"12845035b9a29a7b08b584abc1f28701bd58dc87b1360013b830d1863deed34d"},"formatted_content_sha256":"08569657d1fd1af97815b4671f162ad3cb646b9db6f4ade1b53b237267f3360c","generator_receipt_sha256":null,"identity_record_sha256":"4c68c0c7fd3b4f5ef68188ca20cfd016414d07212968996c98d64d097d36f013","prompt_token_ids_sha256":"41dcd0cbf8f176b6eabaa2189a2cc9dad19e1d0e1b7466a57fcee4fcd8695223","ruler_category":null,"seed":null,"selection_rank":5,"selection_sha256":"0331ac5ad147dee4ef983f58ff124f388c082a848eb8f7a1e2df5c58e3096b83","sequence_length":292,"sequence_token_ids_sha256":"f0361fd70f4b1f9d2a2cbb95c296333e1e9f146ad837ece7cb669c2993a27bab","source_content_sha256":"e916cd640a7665ee439bd983ca0769129de74af589e7e5eba3b559f7ea146c60","target_token_ids_sha256":"bf0037e9004b38848b545334c18c95d9234edfda314a7fc61ac5ba9fd36b25af","token_span":{"cache_exposed_start":292,"cache_exposed_stop":292,"prefill_start":0,"prefill_stop":240,"scored_start":240,"scored_stop":292},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"076f7a7d27bd0edfad63c949645fb3eea8f9d8d32c238b4af19053c5407a8b2b","anchor_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,160],"anchor_positions_sha256":"cea919870992245499fb8d8a858d848e1e76f02a4e2230ada935442533c03198","canonical_id":"903","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,18,28,38,48,58,68,78,88,98,108,118,128,138,148,158],"fisher_boundary_sha256":"dfb2af41b7aacbbc1cb03c5a460d70edf42a188290ce3b01ee741cbb663a7aef","horizon":1,"input_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,159],"input_token_ids_sha256":"8e0cb14d6334e3493c75e96c63848e991fa05b6bc027f04b99b09dd2cbddaf7b","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,160],"target_token_ids_sha256":"6800fb22eb04b0dcd2b08692779155cb065925a066e2f97f52600c51c38ed71b"},"formatted_content_sha256":"f296f132da28a9fa5eeea0a099d9ad97955044e13b9fe30f6d9300ff9e041c78","generator_receipt_sha256":null,"identity_record_sha256":"a64089c9832bde6f4f730dea47c3c93a3ac71dd538c45b2169607180571708d9","prompt_token_ids_sha256":"a454237da9066ead36df97766389f9b28ac1b5df54bd0d9073baa9459aae696d","ruler_category":null,"seed":null,"selection_rank":6,"selection_sha256":"03a537026beb3ae778851b82f172083dc21eb07f0ce7d1683756019ad86e6d98","sequence_length":161,"sequence_token_ids_sha256":"86cdcf0d01cc99b913a8495e4074583c23687d2a77e68c6b0cda5700932a8937","source_content_sha256":"edfb4574ba6b7cb41c0f0d1dc6452e8ac7fea922b120c3ad057a2e77b15fb36e","target_token_ids_sha256":"5fb56e9b4dfe8c41e4ff0043461c0b44248cf165cac01b9f42beb4f0e756f516","token_span":{"cache_exposed_start":161,"cache_exposed_stop":161,"prefill_start":0,"prefill_stop":83,"scored_start":83,"scored_stop":161},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"6ed1f32d878fa277d86681ed3b0a9e7b9b172c3cf90d4213020217eb3296cf68","anchor_positions":[13,27,41,56,70,84,99,113,127,142,156,170,185,199,213,228],"anchor_positions_sha256":"e3fc0847eae8451ba50c7fae586b6479de5eec6dd1bcdf44504db22f0d700d62","canonical_id":"918","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[13,27,41,55,69,84,98,112,126,140,155,169,183,197,211,226],"fisher_boundary_sha256":"53c76b32efa019cb22ef480e27bc5b9dc7c12c0683b0d1b8ad250ac6571380a9","horizon":1,"input_positions":[14,28,42,56,70,85,99,113,127,141,156,170,184,198,212,227],"input_token_ids_sha256":"14e214e41d049733a9ea7e8c02fce42e7aba76edb16178e6d3405bad4c23055c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[15,29,43,57,71,86,100,114,128,142,157,171,185,199,213,228],"target_token_ids_sha256":"24028e01155b1e01eca8ccfc7c2080a6a56b4a3e7c15839043e13b8a00a8ee8b"},"formatted_content_sha256":"f4ed4188dc61ab3e90526a061246e459b1d5fe2fc68d01226a124501eec0be2b","generator_receipt_sha256":null,"identity_record_sha256":"4f00e2535ec2aa52070ed18120472b9d45845790809ac1fb5a7a26a8f6b51d20","prompt_token_ids_sha256":"d8f70964c5c840cc4c74b02a5c432adb08de3a38f706cca7ff9f50f513f616e7","ruler_category":null,"seed":null,"selection_rank":7,"selection_sha256":"03f2b70cb41aa2df2ca06599a3740907c1bd89976be4f7d79613cd3428eb75d7","sequence_length":229,"sequence_token_ids_sha256":"b42ae62c8481ac868fb8f833b03b35ab7f84490d87329022e4548df9b8a49299","source_content_sha256":"1c74c6e9767910ee576993d40b0be7e4f8a3653223ce943b2b48da881f043551","target_token_ids_sha256":"c0130989066437b78efb8eb68c747a3645ecb631cb508775b1f7b001deb4e1e0","token_span":{"cache_exposed_start":229,"cache_exposed_stop":229,"prefill_start":0,"prefill_stop":96,"scored_start":96,"scored_stop":229},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"32d8375f975ffd5157bb123bebf58d926b2fb6e715d79b1c29711866e8ae1d7a","anchor_positions":[11,24,37,50,63,75,88,101,114,127,139,152,165,178,191,204],"anchor_positions_sha256":"48082390de232d2e911509462169a4b3ebe8a03329ea579a66fe38623c52b3bd","canonical_id":"823","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,49,62,75,87,100,113,125,138,151,163,176,189,202],"fisher_boundary_sha256":"ca339ac21962b12f69ea2e43cd5efc5fc9f045e43e311f236cff0f3fc7a9532c","horizon":1,"input_positions":[12,25,38,50,63,76,88,101,114,126,139,152,164,177,190,203],"input_token_ids_sha256":"9060d0dc9e82be8da8cc47571caa8e03245ea38a15689377bf15f3403304608c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,51,64,77,89,102,115,127,140,153,165,178,191,204],"target_token_ids_sha256":"9c64639807a2aba57e44b252f7c9204c7994ff505668345c293c352b795673ab"},"formatted_content_sha256":"72e3934a85cf2d4cb4b035ef066f9a69edbe304668167f633c705ac5a7c3ee74","generator_receipt_sha256":null,"identity_record_sha256":"6775751ca115709c8dd0187f2d2309b90237ea599613ee56420cc077c5a86d67","prompt_token_ids_sha256":"a7322a6a594060b8983da2adc969063889ad65c9524cfc9d9ace5e4856362612","ruler_category":null,"seed":null,"selection_rank":8,"selection_sha256":"04318f2e0a32de485412bc0aa95414984aabdb5973974b77667909ca919923cf","sequence_length":205,"sequence_token_ids_sha256":"c39424dfee203bd58c46a3bbaafc51b57eb8c76c9f8ad175fd4355d227510f19","source_content_sha256":"7be1174fc06994281445e57b76099586c2214066f9358928e22281d793715d3f","target_token_ids_sha256":"c0ca3637b8b1da8af549e87df7bf8e67da2eb444227889016f2678ec5d8b420b","token_span":{"cache_exposed_start":205,"cache_exposed_stop":205,"prefill_start":0,"prefill_stop":120,"scored_start":120,"scored_stop":205},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"bb690c1557693c9a1151a55a056b413abe207a0ede6130a1c4eee76cd7832910","anchor_positions":[8,17,26,36,45,54,64,73,82,92,101,110,120,129,138,148],"anchor_positions_sha256":"34299c685b72af2a076ee41a9cc49b613c0e3c8039202f3412e50762385cd13f","canonical_id":"962","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,26,35,44,54,63,72,81,90,100,109,118,127,136,146],"fisher_boundary_sha256":"f483b9a7b54d8e98c4fb81ac8e8c508a85638477f08243915cc53d3ee584a92e","horizon":1,"input_positions":[9,18,27,36,45,55,64,73,82,91,101,110,119,128,137,147],"input_token_ids_sha256":"562a3beb47bbf20a72e84b88e739d67bf0a6fd83ea3ec4a06ce7d8527bc32c9d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,28,37,46,56,65,74,83,92,102,111,120,129,138,148],"target_token_ids_sha256":"8d91bcd0f3417c6e6ff0e30b8ef2ad4a5200ab497081407b0de689fac826fe32"},"formatted_content_sha256":"35f9c6bf72383681061847ec15dc2a4d69a179546f872d72042b1dbeb7eb3c94","generator_receipt_sha256":null,"identity_record_sha256":"7f528a9f6b6f25735216e2ddd0cf3bc970efee42f4ddc18607d7615c585b5da5","prompt_token_ids_sha256":"ad2690a82eb727187859d0918f78622936f9220218fa708cc305b79f9e3129b2","ruler_category":null,"seed":null,"selection_rank":9,"selection_sha256":"053a8089119cb6c0a1699960c729b5d0bbf904347c2611f2bdbc20d68ac98896","sequence_length":149,"sequence_token_ids_sha256":"39192da46d67ba8a6a1aecc11d2517cb72d4adbd21944f31d854f4aebde4f950","source_content_sha256":"6d510f0a092393c876a9d3827db9cccd4b422742e08dfee9a9094f3786b67fbf","target_token_ids_sha256":"24d4ddf4dc36fb254790754f2826967503025402abfe4dc1c81a57b608e358c1","token_span":{"cache_exposed_start":149,"cache_exposed_stop":149,"prefill_start":0,"prefill_stop":87,"scored_start":87,"scored_stop":149},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"510bbf0acc641074d0ed15a7484a726b97a35b663fe88ff076ca0808ea574c8f","anchor_positions":[5,12,18,25,31,38,44,51,57,64,70,77,83,90,96,103],"anchor_positions_sha256":"307c6a1944f2d94c59465c6af96e251aaf0ea665f8f1a3e0e55e3eb0568d8551","canonical_id":"641","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,11,18,24,30,37,43,50,56,62,69,75,81,88,94,101],"fisher_boundary_sha256":"74d1a793166b47319560030b1ea4243c84cca2ef770c4bd539a7069d57a6516c","horizon":1,"input_positions":[6,12,19,25,31,38,44,51,57,63,70,76,82,89,95,102],"input_token_ids_sha256":"693ad260a9464069c62f03bdc9f97044efaadaf73fba426e3993f726d373516e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,13,20,26,32,39,45,52,58,64,71,77,83,90,96,103],"target_token_ids_sha256":"24fc6cf6c761d9d2d39f6a0e2d1988fd8c0568a28c602b83a298a2efe14f5b75"},"formatted_content_sha256":"0d5d8389772326b9a23b37c5d64023b87ff76167c738f0f04a73f350d75146a1","generator_receipt_sha256":null,"identity_record_sha256":"d257e6e664de7884c47c6c8efd30d9049f908be0c1f0f15604c5b1e680278db7","prompt_token_ids_sha256":"8b73365de43af24e2b250e80fbb6c87c821ce915961cb60bb5f92dd492aa4e25","ruler_category":null,"seed":null,"selection_rank":10,"selection_sha256":"0561b3053aedb5bf75595d41c4c1d56dffc9d7d22d4f594c77acd661612fbbda","sequence_length":104,"sequence_token_ids_sha256":"edaafb16efe7861b6641b4284c1b03cabf4553b33c20400911590227e2717873","source_content_sha256":"b4af7687b1fdeeba11f23fbb43c11d07ffdb4918405f0574dba17f6932087d51","target_token_ids_sha256":"10c4c8e1ab1664e434fe26b8093fe1929571437707a4a4c5082c71bb6879f108","token_span":{"cache_exposed_start":104,"cache_exposed_stop":104,"prefill_start":0,"prefill_stop":80,"scored_start":80,"scored_stop":104},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"77e62d656a470f85ca90c98dfc91b52de62fec0d714ab54d9d78b6111588449a","anchor_positions":[16,34,51,69,87,104,122,140,157,175,192,210,228,245,263,281],"anchor_positions_sha256":"d3c1ef41c3fc59649a4da8be56fc51bf8a16621374c8218a7236e1c502a9ed29","canonical_id":"821","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[16,34,51,69,86,104,121,139,156,174,191,209,226,244,261,279],"fisher_boundary_sha256":"6ce814085aa258c14254daa52b6c87e5c35bb95bb9ce43819236089ed670a938","horizon":1,"input_positions":[17,35,52,70,87,105,122,140,157,175,192,210,227,245,262,280],"input_token_ids_sha256":"52e2a2814003d5a0c9558d2807c349140c1daee27300a41a386f98243a887244","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[18,36,53,71,88,106,123,141,158,176,193,211,228,246,263,281],"target_token_ids_sha256":"a841d33aff758b3642820a6e5c0b714add3840501814ab31d90f6dc5b41336dd"},"formatted_content_sha256":"9b4c39a7f47e7c5485594bfc1f0bc255be70f74e8e73be2bfab32e77bd11cb37","generator_receipt_sha256":null,"identity_record_sha256":"bbe20a7f23dfdd53afdd97893a491105ab3d0d770f28da4cafd4d0f760a10e07","prompt_token_ids_sha256":"8438715e3ee27d1c3e5e158afeb22e85f9543635b4c24c7151828b62262082b1","ruler_category":null,"seed":null,"selection_rank":11,"selection_sha256":"05ca9ce71ec3ccbff2dc9051e6519ff621aeaf051f13006ef2a903db53cfa322","sequence_length":282,"sequence_token_ids_sha256":"d4a708e94f01dc74ebdbaa4a0850da22f19f8c7616df125de0fa93a03cd941d1","source_content_sha256":"6cdab710688743670db0377d5ff8806298470ee86aa94a2f4fd430f2542e26b4","target_token_ids_sha256":"52fee9ad2633a9725f62006b3a2a598f494e32fd6e12dcf43893a911e7032a91","token_span":{"cache_exposed_start":282,"cache_exposed_stop":282,"prefill_start":0,"prefill_stop":246,"scored_start":246,"scored_stop":282},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"88e7f4c26333024cf786b4c1d640f43c25aadabb8055c4006984b0b773283a63","anchor_positions":[6,13,20,27,34,41,48,55,62,69,76,83,90,97,104,111],"anchor_positions_sha256":"70a9c6d084ac7d3e411adc69adf277b706895b48989c28a96164561595a44b13","canonical_id":"882","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,19,26,33,40,47,54,60,67,74,81,88,95,102,109],"fisher_boundary_sha256":"af33587dbbed535e25e0f9ff381dc426652e98f76482f907c3a1b91a43afe65b","horizon":1,"input_positions":[6,13,20,27,34,41,48,55,61,68,75,82,89,96,103,110],"input_token_ids_sha256":"2e82b13f62352679ed5dae9b03694e1c2eeb56e00f762a0373672ed48440f408","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,21,28,35,42,49,56,62,69,76,83,90,97,104,111],"target_token_ids_sha256":"d9c7534eb6dbff16502f2748b6ab6576ab5d76645faba1e8bc35a67323be8c39"},"formatted_content_sha256":"8164f8fb3728db418776886f3233778e01a1c5b7e54b4f350ff9c98d6f4c3018","generator_receipt_sha256":null,"identity_record_sha256":"79d822ab45c87b51b8aa2c0ecb29bfbbd85453f82f6a3e764bec8805d2f9708b","prompt_token_ids_sha256":"4c61f175c320b5f738697b92438e1bb158f00bd6a061e49a529ad3158d4f02d5","ruler_category":null,"seed":null,"selection_rank":12,"selection_sha256":"06182bf7b5db58b26f203c4f99a7fb97d3252f5d92dd5f4333d47e49043b0f9d","sequence_length":112,"sequence_token_ids_sha256":"a69b8ee713bb9e4fdf0cdec61d49a10f148243a2cf8fe92a4420823b82bb3839","source_content_sha256":"f9edf8d912223ba1c68389438b5a930e3ff8116a79ef7e9e4ab5b7f48cad2e1e","target_token_ids_sha256":"b9330c3608f52265a4d54352f0d4d3821513a3adbea5b89bf9f5146143a012fd","token_span":{"cache_exposed_start":112,"cache_exposed_stop":112,"prefill_start":0,"prefill_stop":90,"scored_start":90,"scored_stop":112},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"026a11f123c6c02b349d154973681d1484316586c77571635cb75396928e8801","anchor_positions":[8,18,27,37,46,56,65,75,85,94,104,113,123,132,142,152],"anchor_positions_sha256":"75c1fd0cbacbbfa75bfedc1647ca1a15436ad446c2021b146a8d7af4567321f4","canonical_id":"874","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,27,36,46,55,65,74,83,93,102,112,121,131,140,150],"fisher_boundary_sha256":"69de8bf124376d38cfd76f8801bc74e0e352d8cd33f8dc6b34cb10e4c6ff73d3","horizon":1,"input_positions":[9,18,28,37,47,56,66,75,84,94,103,113,122,132,141,151],"input_token_ids_sha256":"75967a14501363b155570c3d4d66959cde5cce50c53c8fa1fadf71f39e4218de","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,29,38,48,57,67,76,85,95,104,114,123,133,142,152],"target_token_ids_sha256":"38400510781c2bec0042957302068be57d0438f8b57f3a4a775204a7ae26b03b"},"formatted_content_sha256":"8ee29791043cd7726bdd815083a307f0d693443f77d0fc9af6c9249422f82a60","generator_receipt_sha256":null,"identity_record_sha256":"5f505f1a07e986bb3a69f547553e68edda687e89817230981e3d65a20f3b5d61","prompt_token_ids_sha256":"af780c9a24c687de5d7bb86c8de2e3a05d07e1e769e2bed9aef9bc82e0c2b3e0","ruler_category":null,"seed":null,"selection_rank":13,"selection_sha256":"06e6c099b79a31351c8aa115e8c35681452320d265ba3e7c9deb647189b5aedf","sequence_length":153,"sequence_token_ids_sha256":"059517b1e6703c7276b532d66d5dc53b137178142a0fbcebbf7a85ed806bb2df","source_content_sha256":"7be81d34f54ea71fa53a94438dc656b1e14cf83c1f0cb8b85a03aeec675a95cb","target_token_ids_sha256":"a00664d6a993b427df9b77218f9f6ab241e98e255c2754127a6aeb30818bf82b","token_span":{"cache_exposed_start":153,"cache_exposed_stop":153,"prefill_start":0,"prefill_stop":82,"scored_start":82,"scored_stop":153},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7e8e992a57a2c58864e789d43316ec4ca5557685022402a123eb7bec86e0a31d","anchor_positions":[8,17,26,35,44,53,62,71,80,89,98,107,116,125,134,143],"anchor_positions_sha256":"486e7f47d72e2b3d4a862156a2b41e9dbb05a70d78349fdcac0b01e09667750a","canonical_id":"768","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,25,34,43,52,61,70,78,87,96,105,114,123,132,141],"fisher_boundary_sha256":"2f57f4c3f66cd0d31d7220f66c2563c2f42ed9d860ccd032cab629a4a62b2d7d","horizon":1,"input_positions":[8,17,26,35,44,53,62,71,79,88,97,106,115,124,133,142],"input_token_ids_sha256":"951070a8ec9f2842c988081eae148ff3e650f6e041bad423eb8cf80577aa09ca","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,27,36,45,54,63,72,80,89,98,107,116,125,134,143],"target_token_ids_sha256":"be7a411acd1f8c53e98dbbe661a8f0e08925a5287a3a082d567b4348f5daea4a"},"formatted_content_sha256":"19fb2a15b1650f22847079c098721ed0b4ae36db1faae49cb10e7f8b7f8a951c","generator_receipt_sha256":null,"identity_record_sha256":"4f37dd32d3554ba3f0e3842355c0812d1532bf5eeb4d09edf28e778ec744907b","prompt_token_ids_sha256":"5abd62e58720f9b71da73a2f319d99d1dd3f0fe89c148c1e335674d33e0db6bc","ruler_category":null,"seed":null,"selection_rank":14,"selection_sha256":"08d405d25f1c1656f2532ca7d15875b63c46516c4e46a564326b3696f5943347","sequence_length":144,"sequence_token_ids_sha256":"d924aa1344bedd7c5de7beb28914b5f02d01c9242e93916c3ebf5e13a1b777e2","source_content_sha256":"96a4ff323c23dc1eba72d4b800a65cd77f46bad4c7b017e7716f69b066cf251e","target_token_ids_sha256":"2bb90e5895fb397d1b3239e2f50aab932ff08813656890b44cefc5327ce9f0fc","token_span":{"cache_exposed_start":144,"cache_exposed_stop":144,"prefill_start":0,"prefill_stop":79,"scored_start":79,"scored_stop":144},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2a32d45497a46d8765c2bef47acddb45c96e41bdb29282abb99a946dd86db58b","anchor_positions":[15,32,48,65,82,98,115,132,148,165,181,198,215,231,248,265],"anchor_positions_sha256":"0348f2e9e936ebff365da2919e86c0d7b23f030deacd2557e2975ff2c1bb6238","canonical_id":"635","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[15,32,48,65,81,98,114,131,147,164,180,197,213,230,246,263],"fisher_boundary_sha256":"bbb2193a791665a372d6f0871c8b972270ed33e5afe4d627b3f13c69caa5172e","horizon":1,"input_positions":[16,33,49,66,82,99,115,132,148,165,181,198,214,231,247,264],"input_token_ids_sha256":"54ae9eda3529af89c8491dfa14c002e2bbbddacc5f7fc9bd8f98dfd841106b17","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[17,34,50,67,83,100,116,133,149,166,182,199,215,232,248,265],"target_token_ids_sha256":"8e9dff8e5415e27d61633f2fbae7b6f5a404922b80ae0dc2d994004e0c00ff77"},"formatted_content_sha256":"252812434def51f2af7fe0f324a5922cddb2523bd81d2dfc65720c67ba9d9871","generator_receipt_sha256":null,"identity_record_sha256":"e2908b95f86b0b763a5d77e597f5d8cc0e9ff3b96e4129645e8ee178160dc9e8","prompt_token_ids_sha256":"99adc1c15eb492246a3ad45eb9cd6dc0a5f949c95b99aa332c2c8fe742dac882","ruler_category":null,"seed":null,"selection_rank":15,"selection_sha256":"09bdeb42fd2df571b9c13e3aecc461eec43355320ee267a53066bcbb23769161","sequence_length":266,"sequence_token_ids_sha256":"2ca3fac260d1b58347c86dbb21f794efef0df7c42a0de4d942d549481c583b2d","source_content_sha256":"b3bbd3ae7ff06e057b52c17af8b75ac83d20e8b436ca1c82f6f7ccf94d0a9214","target_token_ids_sha256":"b44b18d625486bf3a519bc0a87d4dbfbfb4851798e7e50bacb07a66cab69939c","token_span":{"cache_exposed_start":266,"cache_exposed_stop":266,"prefill_start":0,"prefill_stop":214,"scored_start":214,"scored_stop":266},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e6416dbb61ce7ce16cf3fff7ce25a1bacfecbb3dbf2fde6f06d8fc6699998cd6","anchor_positions":[5,12,19,26,32,39,46,53,59,66,73,80,86,93,100,107],"anchor_positions_sha256":"b9647c4baef8fee686d8a882e4d530418b6f7cc8044a0a83370fa394c3bc5ef2","canonical_id":"666","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,18,25,32,38,45,52,58,65,71,78,85,91,98,105],"fisher_boundary_sha256":"1a644132ebeecd04edf2c98217977f7fcbdedbec84e64b6b50a88a6151232772","horizon":1,"input_positions":[6,13,19,26,33,39,46,53,59,66,72,79,86,92,99,106],"input_token_ids_sha256":"32f5ac39d606f1ddf6a4e4346735931cd6faad7feec35e950f17b8edbcb045a2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,20,27,34,40,47,54,60,67,73,80,87,93,100,107],"target_token_ids_sha256":"eed792902f0c9e240b65aaacd0467e2ff2f4a7eba83a333214fd67d942feac55"},"formatted_content_sha256":"e3a3c1e1ed1b292d4877ff78da3ca92b8112f6b28b41a2eb63f18c8d72057088","generator_receipt_sha256":null,"identity_record_sha256":"ec5fb87c296ab7d9fc8671afd53e5bc11fd05c7ee0bf20f88d758453cb95da2d","prompt_token_ids_sha256":"92348f208743eebc810c3eeffd27e797bfbebe693b75b29192878c2766c17cb5","ruler_category":null,"seed":null,"selection_rank":16,"selection_sha256":"0ad2bb04e64cf7bafe43963d9c975c7597c00f97a065f172225e36b8de1be401","sequence_length":108,"sequence_token_ids_sha256":"05aa7077f2114bfc39c337134a54171570e1bf7f5e40e875dfd4ccc76cd3c5bd","source_content_sha256":"c5657a0fc079e76b566db9dad6a041aa51a885e91a3ce60f4b56e268d4f2283e","target_token_ids_sha256":"34eea4ea001c4d78cc33cc3368d1d71873c37b7e910507eceb9597f35255e3ef","token_span":{"cache_exposed_start":108,"cache_exposed_stop":108,"prefill_start":0,"prefill_stop":69,"scored_start":69,"scored_stop":108},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b641a12066174c147da77baa5576307cddc4326d490da447977a9880c63d2be5","anchor_positions":[19,39,59,80,100,120,141,161,181,202,222,242,263,283,303,324],"anchor_positions_sha256":"229481c1cae947d75547887104b565fe100ef01406aaa1ae33b6680fa4942d03","canonical_id":"795","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[19,39,59,79,99,120,140,160,180,200,221,241,261,281,301,322],"fisher_boundary_sha256":"075b4606ad628cb7bea3693f59ccdb0efa6c6599cce58a6552534c47baf39ab1","horizon":1,"input_positions":[20,40,60,80,100,121,141,161,181,201,222,242,262,282,302,323],"input_token_ids_sha256":"23b5ebefaddbeeafb86642c66656119a5122ac4590adfc291caa30b36d9aa0f0","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[21,41,61,81,101,122,142,162,182,202,223,243,263,283,303,324],"target_token_ids_sha256":"67fe7ce1a7b687bd4b008457e44ac3e68602c818d9241e2c029c21d035fb039c"},"formatted_content_sha256":"ba441afe1323472e606fbc265ed2df392c0910aa7b69c72034eb237f6f1eac12","generator_receipt_sha256":null,"identity_record_sha256":"462faa7c85aef3e98a366b53fff20c26ce10eaccece6d0cf5e990a957d47bab0","prompt_token_ids_sha256":"276d4bcbbf513e902b3e32cea17826db3f7c89340cf3d4a5944f77e4fe398378","ruler_category":null,"seed":null,"selection_rank":17,"selection_sha256":"0b64b061a95be11408788bd7d48178a22fe67b04451aacb2824273de604613c6","sequence_length":325,"sequence_token_ids_sha256":"3bed81ccd0aff452ebe2792b7538c8dab5d74a0ddf5fc9de5e67201d559dd9f4","source_content_sha256":"0c82e760835310eb8ee865c643c6bbd2fe9a92e58f43e818f9fe857fa8bb2b34","target_token_ids_sha256":"07898ad18dc9917a62d1d7d2c673264e506ffa3cd0296f61f5355d73ff74e2a1","token_span":{"cache_exposed_start":325,"cache_exposed_stop":325,"prefill_start":0,"prefill_stop":290,"scored_start":290,"scored_stop":325},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"73863421843464e12bf37b9bdf19118b2a0edae3ea6b28b7bb83ae34025fdb99","anchor_positions":[6,13,20,28,35,42,50,57,64,72,79,86,94,101,108,116],"anchor_positions_sha256":"65c4371e0db2357f72608fb79361ad62ba950063eae1f1df95567c3178cfceb4","canonical_id":"944","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,13,20,27,34,42,49,56,63,70,78,85,92,99,106,114],"fisher_boundary_sha256":"8f181fb4592d87100a01b9b5022fe28dec60a6c1a630f0253a95ef609f751477","horizon":1,"input_positions":[7,14,21,28,35,43,50,57,64,71,79,86,93,100,107,115],"input_token_ids_sha256":"bb992854b746a2d0322b0c73747a5b6e481b4ac55013af85dda8fc8c0bc4e324","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,15,22,29,36,44,51,58,65,72,80,87,94,101,108,116],"target_token_ids_sha256":"efc4327e8bc745b94eb74df47cb9d34c605ce69f88ab01258f192e2678c5c5c8"},"formatted_content_sha256":"588cb6ded28fd05719867c9a3c2ef078440822880f6fbe41587f8ab604fc41cf","generator_receipt_sha256":null,"identity_record_sha256":"a019ad696f9c4fe577d6efe7e08f14e983f53e6a023062d550dc4ffd16fef99d","prompt_token_ids_sha256":"6430dca518b59b80172b9552c64bfa32b6116fc5855d53c075e496d174d33f72","ruler_category":null,"seed":null,"selection_rank":18,"selection_sha256":"0b9a44c3022b9f3f3616c768c1535452329656aeea799dcb45a719751b3126e4","sequence_length":117,"sequence_token_ids_sha256":"a3b8d96a49a5559eb5401f9e9f9b40c0785ee84f73cf189a884618ba0cc77865","source_content_sha256":"2099f30aac30b18f4a8fa27a7204c64734c587cf50057d835eae933211f12c7c","target_token_ids_sha256":"73123ac6d971eeb25450ba23964d4d95870b3421781c2b721b8e3f5217fe1ffd","token_span":{"cache_exposed_start":117,"cache_exposed_stop":117,"prefill_start":0,"prefill_stop":91,"scored_start":91,"scored_stop":117},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"a3fb6da730653e3034c80d7d7da56cd1a27244994198f932d4b653175e07528d","anchor_positions":[16,34,52,70,88,106,124,142,160,178,196,214,232,250,268,286],"anchor_positions_sha256":"88ca5e0c73dcbbad356df2a5c320d1e65852f5459c97030e41b86ba39ef13c38","canonical_id":"653","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[16,34,52,70,88,105,123,141,159,177,194,212,230,248,266,284],"fisher_boundary_sha256":"f3b0c45ebf98940a7a54cb33ffec71487e06a0482e824289ffd032de00327ea1","horizon":1,"input_positions":[17,35,53,71,89,106,124,142,160,178,195,213,231,249,267,285],"input_token_ids_sha256":"d7dccbe6c436987756ad6166a8d563fa686494edff0e090f19a94996077cdfc1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[18,36,54,72,90,107,125,143,161,179,196,214,232,250,268,286],"target_token_ids_sha256":"8493cc8e2ef026b2ca43108aa6e7db63ec761ce211452609a1bb2760a0f66f2b"},"formatted_content_sha256":"157b49e7f32965e28ef9bb626baf1d0356b6d5f38d74f4c673a15e0198ce6751","generator_receipt_sha256":null,"identity_record_sha256":"31193a44e28b9b97aab66d0eff836463790dd908e352239fd1a3dee6e1ad0a3a","prompt_token_ids_sha256":"a3e2cc82d1e964fda655ec40dc4f4058a3f6f4adaa8b6def330531e94950c8e6","ruler_category":null,"seed":null,"selection_rank":19,"selection_sha256":"0c32b016c1fcca024f19127827b8ece87776b1df963f55614f9a15b354218018","sequence_length":287,"sequence_token_ids_sha256":"1d7f3ec55af99be19b85291e5b80c9d9555e9360cd4af99ded037154beaaad61","source_content_sha256":"e8146f7d1066326e2120f2f49e89fe13318dde45bc338d1b6a350b475840face","target_token_ids_sha256":"f4ea61620cf4a9d1cd26e8136680d5f09ad7d79a197bd08433ab22b67e683560","token_span":{"cache_exposed_start":287,"cache_exposed_stop":287,"prefill_start":0,"prefill_stop":249,"scored_start":249,"scored_stop":287},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"adeda0b376d342263631c70ee440fab7010e921ef7e9bc2997f75798d31a644d","anchor_positions":[17,35,53,72,90,108,127,145,163,182,200,218,237,255,273,292],"anchor_positions_sha256":"c72256a78f816fa7bf8e540cc3aaf3fc1795369a3859efe48a9c4a8cce0d2489","canonical_id":"857","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,35,53,71,89,108,126,144,162,180,199,217,235,253,271,290],"fisher_boundary_sha256":"f5b74e46bd51ccc196ee08a6967a1597fb09f1c1b2ba66309ed0bcb627f70190","horizon":1,"input_positions":[18,36,54,72,90,109,127,145,163,181,200,218,236,254,272,291],"input_token_ids_sha256":"555cbfa7f8498b7200297ecda4d0d95381d1e0e366a25c75a729a308b819e751","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,37,55,73,91,110,128,146,164,182,201,219,237,255,273,292],"target_token_ids_sha256":"ded8ec9b50bd3a7f2b508c6b34745e277cf32245748a23e733afeb28ab5be811"},"formatted_content_sha256":"4156119aeb73684201921a09e3d2fef7e8426ebd10babb52788c0f7e8e9813be","generator_receipt_sha256":null,"identity_record_sha256":"670b8008445ce0955e14e6aa12268d4f91c343a57d01f2618609843306761743","prompt_token_ids_sha256":"5e5ccf81697344dfc507d09f593651afc2e0d360fb85f31936aa419b069701de","ruler_category":null,"seed":null,"selection_rank":20,"selection_sha256":"0f3520a5dd05d707070cfc5723f57c992db4df4d8d98965890fa13a516a29c32","sequence_length":293,"sequence_token_ids_sha256":"ddd8c7e579f53e5c52e946378d772b93a5ed813a063da195e831d38abb0cc958","source_content_sha256":"22096f4ec39e2e444074f7f2cc3873ad56b3e27bb407587437fb05693324c674","target_token_ids_sha256":"66d83ee4ec91a16b9bd4d5c77a1b45e1850fd3594f689ead1e55c781506cfb6f","token_span":{"cache_exposed_start":293,"cache_exposed_stop":293,"prefill_start":0,"prefill_stop":271,"scored_start":271,"scored_stop":293},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"fa5b57e6457aed64b16e39b74ac6aa26eccd3fc3019570266249d9c77b045ed2","anchor_positions":[10,21,32,44,55,66,77,89,100,111,122,134,145,156,167,179],"anchor_positions_sha256":"fe99fb9f0105e4d1413d20b0bbf51aa70554cf263494d9944696f0b9ca84c89e","canonical_id":"884","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[10,21,32,43,54,65,76,88,99,110,121,132,143,154,165,177],"fisher_boundary_sha256":"7a8012c30b1d696314102eab8bd4402dcf733de71d67674552a2c672d81c2cc8","horizon":1,"input_positions":[11,22,33,44,55,66,77,89,100,111,122,133,144,155,166,178],"input_token_ids_sha256":"391312b446f027fbfa057f16da7c25100505a8356e160bc745be4b022c43ebba","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[12,23,34,45,56,67,78,90,101,112,123,134,145,156,167,179],"target_token_ids_sha256":"4f09e35ae3b70a36f49caa3032145ff047fcc34b2e0c9615bdfa4ec5effcacf4"},"formatted_content_sha256":"e2917eaf992f7bb16e2280d5b40f59553c293a9c2ff814a21cdbf4193246972d","generator_receipt_sha256":null,"identity_record_sha256":"170955cbe700df9f79cb32d7fd38d8902e1a304a3a7a3844b606bfcee8b8366b","prompt_token_ids_sha256":"676841a61aa3135e7a3ec64d03888de021cc34698ee1b25aebfff640fc502cf1","ruler_category":null,"seed":null,"selection_rank":21,"selection_sha256":"0f7355686d92107703dcf05c4240667b8c62bf7fc1f0c45fe1704ffc1686d685","sequence_length":180,"sequence_token_ids_sha256":"fc621b262cdee5eeb2686915a8d202aba1e489894daef49f381679515598da1f","source_content_sha256":"9cea206d7dc3308fcc17678f64598526990d06b799c3a6060837e47d5e0ac2ea","target_token_ids_sha256":"18f70329692e7510f6bc5c35dd7288681cb2d7fc3a45398cb46fe8c4064fc5b4","token_span":{"cache_exposed_start":180,"cache_exposed_stop":180,"prefill_start":0,"prefill_stop":112,"scored_start":112,"scored_stop":180},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"c56fd94d27ac497a874a2872a9378539bce23e6f40c0c88da8548530343fcee1","anchor_positions":[9,20,30,41,51,62,72,83,93,104,114,125,135,146,156,167],"anchor_positions_sha256":"eaf3a0e814f88177633695c30e9a05b6b492aa64c90bf98da98cb9be3917c812","canonical_id":"878","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,30,40,50,61,71,82,92,102,113,123,133,144,154,165],"fisher_boundary_sha256":"45cdd8b5f6368af3ea5312cd27a1db7a47955c6f64da0b88e27faf49b3c816e1","horizon":1,"input_positions":[10,20,31,41,51,62,72,83,93,103,114,124,134,145,155,166],"input_token_ids_sha256":"800e65a128ae4172d15f3bf9d35062d514957f43f113ef8390515f7bad4eb648","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,32,42,52,63,73,84,94,104,115,125,135,146,156,167],"target_token_ids_sha256":"cf7c2293e05be3baaf208adda2a202c2dac0599b886a00e133b951f71671b17f"},"formatted_content_sha256":"9fa1e6f58327152b9fd6caf18d856e0920f7b962af65f17c0b516a39a15e889c","generator_receipt_sha256":null,"identity_record_sha256":"9c45e96b568ec62ab8d97f8c4b13adc325eb92bf34bc8cb045cbf49deb8d226a","prompt_token_ids_sha256":"280373fa3a2da39e89a082829b753432d7f22d6ef6366e83598df36823d26b55","ruler_category":null,"seed":null,"selection_rank":22,"selection_sha256":"10b149086df38d9067b558f5886d640b68f4204287a8886f65495246268313d4","sequence_length":168,"sequence_token_ids_sha256":"0bd5f59a58f0f806de221fe3c637774e5c9e47f72c11f0b19b9f2cb06d977dfe","source_content_sha256":"44ecb08d42ef8de88bcb0e909c2e9befe4a195daaebf6c8c10e1baaf0438944c","target_token_ids_sha256":"5b3eced18f97c0b17b291bb1e4d2e732f1ef7b1b304ac11beb67409c8f2d3bc6","token_span":{"cache_exposed_start":168,"cache_exposed_stop":168,"prefill_start":0,"prefill_stop":139,"scored_start":139,"scored_stop":168},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2d6e6a2df8f80c87c48f19e05e8d274114304b958532988537c2cfd6c7ba447a","anchor_positions":[11,24,37,49,62,75,87,100,113,125,138,151,163,176,189,202],"anchor_positions_sha256":"0e1a219b40ec9118f3f3cab251874074a9782d1da2d399b19337c5a52ff6788a","canonical_id":"822","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,36,49,61,74,86,99,112,124,137,149,162,174,187,200],"fisher_boundary_sha256":"6667326d144985716907297f9fcad0d67d899e5951cfd594d7c70e6f0b3e7ec4","horizon":1,"input_positions":[12,25,37,50,62,75,87,100,113,125,138,150,163,175,188,201],"input_token_ids_sha256":"70f7035d5ef6a263a7316e7d5fe9a1d64361df51aca8f8b97408bc66942765a2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,38,51,63,76,88,101,114,126,139,151,164,176,189,202],"target_token_ids_sha256":"b72dca6074d7b5b12b92fb70cfe49900c4d0b2869f86c323e978902c8635fa80"},"formatted_content_sha256":"e207600ca200fa35202626097b7ce35aabbc9a909f74a7f4a9634823421b3f85","generator_receipt_sha256":null,"identity_record_sha256":"6f1cb4fcebbe64bdece660cf8e72d4b0d933e1d5e76363e0fc65bc84df091ccf","prompt_token_ids_sha256":"a45819695adc36f87315b6277308921c06ea8a973dd4acd584c641b42a489685","ruler_category":null,"seed":null,"selection_rank":23,"selection_sha256":"10e329cfe9c44c390c0748888b664bc73f711559a095943ffb716a5c49a43bfe","sequence_length":203,"sequence_token_ids_sha256":"7ca67bbfaf0cf4d58cd3fbf7999f0bdea010a29908aa1277c433c3640ccc93fb","source_content_sha256":"c34011770653c990605bfd0878f3caa642227458bbd94408174d49073634373c","target_token_ids_sha256":"d355095fa08c0462073247a017dce85a1e1739895f047802a3f3fba9380d16ac","token_span":{"cache_exposed_start":203,"cache_exposed_stop":203,"prefill_start":0,"prefill_stop":71,"scored_start":71,"scored_stop":203},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"89cd198269ee29a7bf3a758e494f86cdd7b4c8eb634e87418ec24fa6977ffb90","anchor_positions":[8,18,27,37,46,56,65,75,85,94,104,113,123,132,142,152],"anchor_positions_sha256":"75c1fd0cbacbbfa75bfedc1647ca1a15436ad446c2021b146a8d7af4567321f4","canonical_id":"687","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,27,36,46,55,65,74,83,93,102,112,121,131,140,150],"fisher_boundary_sha256":"ce22804019d01ac4a4b1ecc051fc015d0fc2edfdc083fe9b46238115a22d2681","horizon":1,"input_positions":[9,18,28,37,47,56,66,75,84,94,103,113,122,132,141,151],"input_token_ids_sha256":"fc68589c14fc3c7f27c207419457fdea2a3ca0ea5556ab837c7e77422872e50d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,29,38,48,57,67,76,85,95,104,114,123,133,142,152],"target_token_ids_sha256":"2aafef74e6bdf1473faf98f2e2f51258d9876ed6aa14fd3c8d901e57a6a67670"},"formatted_content_sha256":"34caea8f7845c77c8c7c5cb04e37b6b1594df9805f377e4ad243d7b6bd787df4","generator_receipt_sha256":null,"identity_record_sha256":"5ead1d0afb62e76c6d447e766cb0e770b7b3122bc85e14984b945f91559084eb","prompt_token_ids_sha256":"7231469d89e6746cffb3134204ab6225bd06db47e1a21a1b29978270a92818f3","ruler_category":null,"seed":null,"selection_rank":24,"selection_sha256":"1241cc732d851bc4d6e31425c413658747a68ac5bf1df371701b6bac3339de38","sequence_length":153,"sequence_token_ids_sha256":"0ff8fac034540d5fae83061b024846460ae750e94227132be74a1f8b13d0c5f7","source_content_sha256":"570e823f5d18678380cee81c7f8171a7dccbbe1956f576e5bef831ef0417fbfc","target_token_ids_sha256":"9dd750e2b89888cecb138a1c442f682c640b1ce4caf2a92c920fc630aff9b969","token_span":{"cache_exposed_start":153,"cache_exposed_stop":153,"prefill_start":0,"prefill_stop":89,"scored_start":89,"scored_stop":153},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4c2c007c3a069505a8984ea48151216aceab5ff41919fef5caa62a9c95724294","anchor_positions":[5,12,18,25,31,38,44,51,58,64,71,77,84,90,97,104],"anchor_positions_sha256":"3d14c7e60b70361fe3f48971fff7c78a0e3a89500325f79e8590e96f876c6d84","canonical_id":"820","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,11,18,24,31,37,44,50,56,63,69,76,82,89,95,102],"fisher_boundary_sha256":"a5d52b2d880ccd9e8c57da79f29d3e8871447c9521ac6f1afce731259780a7b4","horizon":1,"input_positions":[6,12,19,25,32,38,45,51,57,64,70,77,83,90,96,103],"input_token_ids_sha256":"f6d753d6a779f0b43d8eb24d7213ecc4ab0ee16118bcbc17c3cb053d5730670b","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,13,20,26,33,39,46,52,58,65,71,78,84,91,97,104],"target_token_ids_sha256":"75597ca42c1c682f0e8f425a875b8483f8c45067ec1d096e0fd49b964402322c"},"formatted_content_sha256":"c5473e872de3d1d10c5f923d4246d5e6e41987ce6e60afff50ad39c665b75d2b","generator_receipt_sha256":null,"identity_record_sha256":"bfa5a8c5d9a1e111c3b629cdc0f09fc8a0dada73721cc93beb5ce531328d1f35","prompt_token_ids_sha256":"9edc0e91d3b033f1f21804597df8021ebb7cede1211331d0898ea29ce2062107","ruler_category":null,"seed":null,"selection_rank":25,"selection_sha256":"1269c5ac64a35cc30e3aab0138aa3e2e5cf40fa1a583bd687f3718d0b78632b5","sequence_length":105,"sequence_token_ids_sha256":"d9f20cb4b5b15d6acba099a4e472ad67b130db359488d35bb838bd17655f28b3","source_content_sha256":"1cda14e8c3edcbccfd8ccbee369285b27f5c9203af7f1d0171339a6fc431ddbd","target_token_ids_sha256":"845201cf3c1564028c1f00ea5c73aa8c447fdf960863b76d2f859ed548bcd8a1","token_span":{"cache_exposed_start":105,"cache_exposed_stop":105,"prefill_start":0,"prefill_stop":74,"scored_start":74,"scored_stop":105},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"f689b6853781e9f787183224c1fd3cee86b9e7cc33bd38b263db4768f634cf37","anchor_positions":[14,30,46,61,77,93,108,124,140,155,171,187,202,218,234,250],"anchor_positions_sha256":"ee1858c2e927a82713536b65cb17e66e4c3e2dba47ef6753229f8cbe7def8d65","canonical_id":"920","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[14,30,45,61,76,92,107,123,139,154,170,185,201,216,232,248],"fisher_boundary_sha256":"0d18b72689a6cc158337087ae0bf96353aba3ba691e2c3ad3383f821d707d7a0","horizon":1,"input_positions":[15,31,46,62,77,93,108,124,140,155,171,186,202,217,233,249],"input_token_ids_sha256":"af45fdc8e583f1a48a51520447e6ae51e48ff97bf7106924892afcd5b445f380","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[16,32,47,63,78,94,109,125,141,156,172,187,203,218,234,250],"target_token_ids_sha256":"9fb5b7afdc77178b1ad18dc9c00bf26c57c1c7f7283fef7c2915df9d37953a04"},"formatted_content_sha256":"ca28cff2764b8d05657413d50f09faa64113177e180abfde5843c1dd2e08ff63","generator_receipt_sha256":null,"identity_record_sha256":"af618c65e0f65410c7799e66f057af21d4ef73c2941c226d581f4f95c21ec761","prompt_token_ids_sha256":"5b15d5261297dc4ac22a17ffede7ef05db15e3f11f0c33e02fb3a3b0e0716a02","ruler_category":null,"seed":null,"selection_rank":26,"selection_sha256":"131778dc6e39fd27c980789805b135c6f9d71c73da87e2cf5d8f360898a2dfd2","sequence_length":251,"sequence_token_ids_sha256":"c20ed19761b4f9b066a1329b36e80f89c62a064c75eaf0dc86e28e961252022c","source_content_sha256":"a36a2af1cadc864f5b43b1a1232f93e477ab83df4d2faa13fb766e215ba061ce","target_token_ids_sha256":"3f83098fcd8caa7e8285f24d6317b53da289468363aa33dd7c3ad415fa2f3ee7","token_span":{"cache_exposed_start":251,"cache_exposed_stop":251,"prefill_start":0,"prefill_stop":215,"scored_start":215,"scored_stop":251},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1df227dd0db9ee9876c5933450c5e9d7dad0c0b569bfe3c3c9999cb603466a66","anchor_positions":[12,25,38,51,64,77,90,104,117,130,143,156,169,182,195,209],"anchor_positions_sha256":"33a74989d1780ad7a110110bf9c662c1607a2b0974227d3d4808405cc10473b3","canonical_id":"771","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,207],"fisher_boundary_sha256":"ca40d8ca4200a96f42fdd96325a52572dfd4c03985d05bc5122da59ebb5411f5","horizon":1,"input_positions":[13,26,39,52,65,78,91,104,117,130,143,156,169,182,195,208],"input_token_ids_sha256":"4a16859fc5dbcd936377e8ff10cf7ee24881b00e341b233f5d9eb6bcea0473b2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[14,27,40,53,66,79,92,105,118,131,144,157,170,183,196,209],"target_token_ids_sha256":"5ca4ff0a723a1b4d28096b092591f8877fbcca9586d49c5ca5cec329031ee210"},"formatted_content_sha256":"12a4cc60de825c6e0b4030d980e6efa58ef1883c0dc4e19c9247b74a5cf96589","generator_receipt_sha256":null,"identity_record_sha256":"b0f0620e81f02da6395b8984ee9a7ea9030dc636bfe539b7bdc6cada58f330c8","prompt_token_ids_sha256":"f7dad551ce5d1c884e8df0a56f12ba6a9086a14bf88ab24c0585392f67253369","ruler_category":null,"seed":null,"selection_rank":27,"selection_sha256":"13342bc1fc998ac6effb2746ad6dd189bd6e08647608c26dfca8ba6e4aa9bc7d","sequence_length":210,"sequence_token_ids_sha256":"880adca8dd7b7e2a5ad8761db452ab4f71a3a2ddb89a2fa3d3f5a58203aefb0b","source_content_sha256":"f9bc388a30026b9c50d24c57c12c983ad57c610e3b1805c0f31da6113e7f0655","target_token_ids_sha256":"946d1f65de1ab807e8d9cdfe1ab226186db882b34df45f0fb02c5dc1c917ec65","token_span":{"cache_exposed_start":210,"cache_exposed_stop":210,"prefill_start":0,"prefill_stop":75,"scored_start":75,"scored_stop":210},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"ba5d2d0908e27069abed969317f8767e700f0653a11ffa619097bf013257832e","anchor_positions":[21,43,65,87,109,131,153,175,197,219,241,263,285,307,329,351],"anchor_positions_sha256":"cac63358bd3a7b31df1079c37c0d5ebcf2a9d198f8fbdbfabcbd20682cead52a","canonical_id":"869","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[20,42,64,86,108,130,152,174,195,217,239,261,283,305,327,349],"fisher_boundary_sha256":"2f61fd72d21168bcf628f22bfa5c23cc9ae7adad9c324ddc9436931a5fc1078a","horizon":1,"input_positions":[21,43,65,87,109,131,153,175,196,218,240,262,284,306,328,350],"input_token_ids_sha256":"4681d4fb7732b42586111bbc62eecd3d429bea617e31837702eba8e398a2cdec","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[22,44,66,88,110,132,154,176,197,219,241,263,285,307,329,351],"target_token_ids_sha256":"4703a2598c12a6054f3c45ce108e141b4a06b5c95d21417d8999f4cbef5fb5b9"},"formatted_content_sha256":"a0eb7c677531c1708849fe663f5664ca25a2bb7202ed10517fbd94dbde91daa8","generator_receipt_sha256":null,"identity_record_sha256":"f65800d83e860fc9520ba9f8b4f750dbd13615d735ec53de0c7592e72d90220a","prompt_token_ids_sha256":"b39968cf98321965c22a9c256c9561e9cc6e27d9a79af006cc89227ec7120efd","ruler_category":null,"seed":null,"selection_rank":28,"selection_sha256":"146f84886fd8ea499b01ff6f18e9470af408d245c5e61abbaa90c362cb26c801","sequence_length":352,"sequence_token_ids_sha256":"1b54a34c4d7ffc13d96aefb4e29514f90f431dde94601897a2dbb235edc52720","source_content_sha256":"aaffc31c45370a56d2068d7903cea3700521215fda95a64f3df629df4b8eeb38","target_token_ids_sha256":"604ca3456eb59e105b48ea010ab6e7f9f2dcfb395b4632901098473fb57640a5","token_span":{"cache_exposed_start":352,"cache_exposed_stop":352,"prefill_start":0,"prefill_stop":305,"scored_start":305,"scored_stop":352},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2bb8e9815739f4c21c430d24915060fe6bd9dba2fc037408c960a316957378f8","anchor_positions":[7,15,23,31,39,48,56,64,72,80,89,97,105,113,121,130],"anchor_positions_sha256":"4b59ffbfdc6171d05db7e4ce13fd81c13fb1c2704ef54cbfaede26eafe2b1382","canonical_id":"851","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"fisher_boundary_sha256":"0b7b6d1d65846d9e17cf0ca2a441e8df727713097fccdd92a3cd8bdb20be9263","horizon":1,"input_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,129],"input_token_ids_sha256":"08877abaf9a7a17c7f7f67ebc2737c37d6453391c6d7b8ec1fbcd14c32c4e1fa","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,33,41,49,57,65,73,81,89,97,105,113,121,130],"target_token_ids_sha256":"f8b2c9f022f43056408bb17bc5c2c95096dde7d6dfeeade0d28c5bf51c33888e"},"formatted_content_sha256":"571149c507816476043bbda2c1fdd724f3ecb9950bd16943ff2ad7eb4ea3f436","generator_receipt_sha256":null,"identity_record_sha256":"7f27dea0fefa17b65ffacb46cfdb9b6d59002d7ca0b9361a0895371d7110d026","prompt_token_ids_sha256":"286d768c4aa541e076ba087112772071c0ed8c5cf796ee399892f394bcbf99e9","ruler_category":null,"seed":null,"selection_rank":29,"selection_sha256":"148757e7e40834cbdd797fc8415b8ff9af1ad4582e23345b11d6f4df20524e54","sequence_length":131,"sequence_token_ids_sha256":"8d6714078aaa6f0fbd4d5dffc75dbd163c2b9867713c0dffd9d5fd8f7017a68b","source_content_sha256":"a2daebc00878723580b55d11ecc7c733a3b30dc6ac7d2162e0a5ddb14516f888","target_token_ids_sha256":"2007499fc1bb16a5e15e13b3ebe6e9c56b039a7ed23a50d2c653934a2788085c","token_span":{"cache_exposed_start":131,"cache_exposed_stop":131,"prefill_start":0,"prefill_stop":95,"scored_start":95,"scored_stop":131},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"348a971192939b69d748687327be2a9dfe6f51af36995ecaa6f6c112cad31b71","anchor_positions":[9,19,29,40,50,60,71,81,91,102,112,122,133,143,153,164],"anchor_positions_sha256":"e70afdac88bd21ff5ffd057462f0868a1aaa408a52296b58ecd4900f7f13759c","canonical_id":"728","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,39,49,60,70,80,90,100,111,121,131,141,151,162],"fisher_boundary_sha256":"de5b5407dcf2dbe46dcd273dbcb9112fd1e97d1b3786ce35fa8124c57a372da9","horizon":1,"input_positions":[10,20,30,40,50,61,71,81,91,101,112,122,132,142,152,163],"input_token_ids_sha256":"164cace43db8303a6bad35155786734ad14df0e29d82329399b4eb77c2ab2a51","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,41,51,62,72,82,92,102,113,123,133,143,153,164],"target_token_ids_sha256":"cf736f31488418c09e969cb79604c6e24ee27824d7b99f2cf3615788cd99df11"},"formatted_content_sha256":"876263584044b675e092dfa08f39f07f3de5267b9c2cc8ebc0f0dcd3f7e69bed","generator_receipt_sha256":null,"identity_record_sha256":"07b155ad0595cc23e7b6f3ba638e79b47f8790d682a99a574040b9b316eb1770","prompt_token_ids_sha256":"7673bf381655d73528d4ffb834dd0461fec79f874ac5706819d772a4fd92f274","ruler_category":null,"seed":null,"selection_rank":30,"selection_sha256":"1488999c83a1784fe27b3d4c152a98b782b1ce9c84a2375a4dba8bc8a3845802","sequence_length":165,"sequence_token_ids_sha256":"cbf68229cee850133e8ccd5fc251f1c822ecfb6aa1c9466194360609b2644aa8","source_content_sha256":"1e63d65d4f23efcd1ebde304ca66d531e9fdfa44f73cf04c72637826af3e7c94","target_token_ids_sha256":"519538cb0a590f8242c78a833368585a5650cdbbaffd4159acb21311b25be8bb","token_span":{"cache_exposed_start":165,"cache_exposed_stop":165,"prefill_start":0,"prefill_stop":128,"scored_start":128,"scored_stop":165},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"fd0a190e30ef1d0dac28f65cd40bb5a6bcf3b1813b383cf8c3595fc399c3cf68","anchor_positions":[8,18,27,37,46,56,65,75,85,94,104,113,123,132,142,152],"anchor_positions_sha256":"75c1fd0cbacbbfa75bfedc1647ca1a15436ad446c2021b146a8d7af4567321f4","canonical_id":"704","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,27,36,46,55,65,74,83,93,102,112,121,131,140,150],"fisher_boundary_sha256":"3f642dc0ec35708247c2582e0525688edfba25c8e20f052b50fddeb5a552f23a","horizon":1,"input_positions":[9,18,28,37,47,56,66,75,84,94,103,113,122,132,141,151],"input_token_ids_sha256":"c7f609f9659eee383626ea3792b82efa4d88eed93f90d975eb30c098b4b461a1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,29,38,48,57,67,76,85,95,104,114,123,133,142,152],"target_token_ids_sha256":"04c201a63e6b0f0929e297c5714da1234f09af3ca5cb45bc1b660e160389b369"},"formatted_content_sha256":"cae02a47f70b35282c8bb71d3b57044d50c57479d6d860ab99959bd7d6a92fc2","generator_receipt_sha256":null,"identity_record_sha256":"5ee66dd333ec6248d85a5f68ed91d44953b5193938257c58e4a4d12487288565","prompt_token_ids_sha256":"049fdd2c0e15e77a6301829c690e8a3c665ce6861ab91933d27aa964bb51de01","ruler_category":null,"seed":null,"selection_rank":31,"selection_sha256":"149396b0bf9de8c384e257ecabf9d186fe87150d089c3ff0d04e2adce108c249","sequence_length":153,"sequence_token_ids_sha256":"d06f1e44e281b3cfff0ab509061ec63103257ce65b9382636808622420a3847b","source_content_sha256":"9acfae6a9b6bf5138640d082e7cd35c1728c6aef929c1163bdc349fd4539d647","target_token_ids_sha256":"413994788ec19a2c46be1007e6767b52c27d228534ace24859c56b31248a539d","token_span":{"cache_exposed_start":153,"cache_exposed_stop":153,"prefill_start":0,"prefill_stop":114,"scored_start":114,"scored_stop":153},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"30b1518b021112b74a55061d787eb229a1e0d9413d6dacc4728a4dab232a67b7","anchor_positions":[9,19,29,40,50,60,70,81,91,101,111,122,132,142,152,163],"anchor_positions_sha256":"868d4ebf721b69ebc32ed80fb23287a93509d3cda6e8df7dfacdd2b56b0be622","canonical_id":"725","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,39,49,59,69,80,90,100,110,120,130,140,150,161],"fisher_boundary_sha256":"9143ea3e85262814307335f186e336430d15da7ab0b7b3d21349bf2995db95b8","horizon":1,"input_positions":[10,20,30,40,50,60,70,81,91,101,111,121,131,141,151,162],"input_token_ids_sha256":"596886915afefd5186de7498848bbc83ca47716f8e4d7009a44f03810ca08ca5","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,41,51,61,71,82,92,102,112,122,132,142,152,163],"target_token_ids_sha256":"3039436dc9e159ba1ad775ff006ed4075d724a7118b474bb9fb8a1f2e8adf4fa"},"formatted_content_sha256":"ba9dc9a284681815182d925a5298e2f7da2b4d824b73ef346adbda8f64016575","generator_receipt_sha256":null,"identity_record_sha256":"e596da18d1c37026ef17f3e29e6e8ff1d0a69c227e1cbbad9c536f121a0a3fa1","prompt_token_ids_sha256":"9d320289cf51463daa5b252b98ca89089d036d231b6c861dc2405bc54707a135","ruler_category":null,"seed":null,"selection_rank":32,"selection_sha256":"14ed1df91ab46cf4aa753625f81c4925778f6febc4e611062de8ad6126487c18","sequence_length":164,"sequence_token_ids_sha256":"b6e36e161450f26908c1d62afda5bc88c353a091bbe87f9de4dd698d1df5afe4","source_content_sha256":"d222c883a2e980af7460497236d6865d722ce738a2d6303dcb39cefe0ea11658","target_token_ids_sha256":"da7b35aeba5cd5e4fc9de97bd08c7c8e930056923fea9d072dfd25b57ef54703","token_span":{"cache_exposed_start":164,"cache_exposed_stop":164,"prefill_start":0,"prefill_stop":138,"scored_start":138,"scored_stop":164},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"33a12c7dff5a4e3f2b0e78cb96a2e15ffca6d15f728512f48ad3d2a521547859","anchor_positions":[12,26,40,54,68,81,95,109,123,137,150,164,178,192,206,220],"anchor_positions_sha256":"9cd7cad8fc6a3e6b59742658c86bba1f77992157cd1a23eb295add08927945d5","canonical_id":"616","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[12,26,40,53,67,81,94,108,122,135,149,163,176,190,204,218],"fisher_boundary_sha256":"7fb746d0e7bad7bf9d3e8daeb30cb4fb5e6d8058e6e2f60f208e45d9d3074ef8","horizon":1,"input_positions":[13,27,41,54,68,82,95,109,123,136,150,164,177,191,205,219],"input_token_ids_sha256":"2984e5c39ef7c7a9017c275ad13366ddae1e4a944113a0d7634a8743fe7d838e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[14,28,42,55,69,83,96,110,124,137,151,165,178,192,206,220],"target_token_ids_sha256":"f8ac64591035ce55623fdb95289f20ce8ce2512016905cd8c55cdbd1eaf0892e"},"formatted_content_sha256":"4c31755567c8f3a5c363166b6b45afe0e9756596cab355453c6d1bede973f1cb","generator_receipt_sha256":null,"identity_record_sha256":"2d8025f16c114b643173084fd6a55004c4cc8d89c19ba719eaf77c614f3de5bb","prompt_token_ids_sha256":"e3e1a17d69f2fe4102ab51a9bb887707430d5bd517f752ff79cb5ba3bc19372e","ruler_category":null,"seed":null,"selection_rank":33,"selection_sha256":"15619d70a9d1e84b42b7fe6f67fb25251ed4e033a22a1f6b44afdb5190e0d9e4","sequence_length":221,"sequence_token_ids_sha256":"c14c8b8afb98ee4e0db1a7d15b4a5ad8f4ceed0533d70f5c7f3e3069aa019030","source_content_sha256":"1754fb38b7c558a69da8e9273fbe27d53c7428292d713cdcc926533238ea0dd2","target_token_ids_sha256":"67f9de9099caf2a8542295769dccf90b9868362889179ae8905fbdfffbfa259d","token_span":{"cache_exposed_start":221,"cache_exposed_stop":221,"prefill_start":0,"prefill_stop":172,"scored_start":172,"scored_stop":221},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"df20597eac46fe01ffe3b097acab98b2815becb962637bf63848ef96325ccb51","anchor_positions":[16,34,52,70,88,105,123,141,159,177,194,212,230,248,266,284],"anchor_positions_sha256":"875344e57ef3cc2aeb5a93339807a07f348a76ff8e531ac8b017af57bdfad6d8","canonical_id":"686","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[16,34,52,69,87,105,122,140,158,175,193,211,228,246,264,282],"fisher_boundary_sha256":"a3d4c1cea37694ff9e48138187338acaf151bbb01faf6beacef99dbb88b7a3f4","horizon":1,"input_positions":[17,35,53,70,88,106,123,141,159,176,194,212,229,247,265,283],"input_token_ids_sha256":"e11b1d7a44256148ceeca485fc09b3a0bbcb82bb1fc8f781be0549ae79fe1f2a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[18,36,54,71,89,107,124,142,160,177,195,213,230,248,266,284],"target_token_ids_sha256":"89eebf9531df2b055714e83202f980d1b550ad965a5daf4c7bdb06671f9842cf"},"formatted_content_sha256":"7351330811ac6c42a440a1ec8bc6d19bdf453494f7fd6204e144c59693722149","generator_receipt_sha256":null,"identity_record_sha256":"66b94f7added65074e9aa08de692376e4e61c77477886d2f1f02bd7a5f752106","prompt_token_ids_sha256":"c9ee635d4f345c508119836f02286e8a883aaafc0a5504f5149f7ce65b3e8a58","ruler_category":null,"seed":null,"selection_rank":34,"selection_sha256":"15a952922cfb59e1c69a11112c4ca386841fc6ef24bc7d4b5506686777d97a8b","sequence_length":285,"sequence_token_ids_sha256":"fcc6b59259d07edcb03ac21e620699ab241c7ce75c923d03ee63d3146e5b591b","source_content_sha256":"84f9793e00ac40962b5d2488ed7ea603278d041e9ce4d1fdb4c80a3cb038287b","target_token_ids_sha256":"f0132a38a41b59f822f698b45717035a5147aeda26ea2ca039b6d45f96067ef7","token_span":{"cache_exposed_start":285,"cache_exposed_stop":285,"prefill_start":0,"prefill_stop":239,"scored_start":239,"scored_stop":285},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"969aa08fc8cf0fcef08b212a868ffd0ef6cba7360b8cdfd4c7e057bdeb42b613","anchor_positions":[23,47,71,96,120,144,168,193,217,241,265,290,314,338,362,387],"anchor_positions_sha256":"5de510290d52f91a9e5aaaf002df55793d821f7c4325536087b3fe975a00c5db","canonical_id":"722","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[23,47,71,95,119,143,167,192,216,240,264,288,312,336,360,385],"fisher_boundary_sha256":"5bcdece273089c73eeb060133d26006686f4ba55e2f679475c817201f0f06663","horizon":1,"input_positions":[24,48,72,96,120,144,168,193,217,241,265,289,313,337,361,386],"input_token_ids_sha256":"387e8436656479c468480e0d7ab6fb8a04966a25430d2cf7ba6b946e933a5988","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[25,49,73,97,121,145,169,194,218,242,266,290,314,338,362,387],"target_token_ids_sha256":"d4c1910bee8959bc2af95badfdaa811dfbfff9ce3708801ed5e9602140453d3f"},"formatted_content_sha256":"66886a05110a3b2882fef0d7f3d69fe9e4813e5a230a25f32ee61406ab9d5da2","generator_receipt_sha256":null,"identity_record_sha256":"dae3872dc358589c8a222ed2b8f364150c472c88b23243e89cbb751bec3bf394","prompt_token_ids_sha256":"e0dd20f7abb2fd8622ac76878d7d8dba366a9267d17eb23cde37996d51815bad","ruler_category":null,"seed":null,"selection_rank":35,"selection_sha256":"165983c8c5992c88338b5a83c3e955a3b379e2938838aa804f752b6af24364d5","sequence_length":388,"sequence_token_ids_sha256":"be18de19017b46d5b7be59183a8ecda4c60a214ae50f068dd8ce1b4ea6848323","source_content_sha256":"20253504a8b3a9c5fa26e7d29c98f783efb48d1960c35ff104acfdc24f41bd57","target_token_ids_sha256":"5e021e0c05f5c5bc55c907080f16d71f9b17ea980d1c4a1c35db2a4193541b91","token_span":{"cache_exposed_start":388,"cache_exposed_stop":388,"prefill_start":0,"prefill_stop":344,"scored_start":344,"scored_stop":388},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"da5073651bb69cf285a41628d42d9f0b3d2b37c1bf90aa7aab1b73c4d9096d21","anchor_positions":[6,13,20,27,34,41,48,55,62,69,76,83,90,97,104,112],"anchor_positions_sha256":"311e7702507c904e70997e0b842193b8c519ddd45b6861c3e52fdd12fcbe662f","canonical_id":"919","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,19,26,33,40,47,54,61,68,75,82,89,96,103,110],"fisher_boundary_sha256":"46adb9e95c41a48fc3a2299f313d710d273451aec915fcc44050066d7e62fbe7","horizon":1,"input_positions":[6,13,20,27,34,41,48,55,62,69,76,83,90,97,104,111],"input_token_ids_sha256":"27444c193cd674bda4df496bd61a9e9daaf7ab3670462bf76e183b0cb9f16771","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112],"target_token_ids_sha256":"0c19883626f27df1a14cd3ef14afef3bc348c0cd1bda790d1e08185bca73a595"},"formatted_content_sha256":"032c4c4ef8d61477f89f627c7da791c2a394cc3d48b63ae5e73ee2fc12217e59","generator_receipt_sha256":null,"identity_record_sha256":"3ff17e45b728b8142b6678e3a33eac324c6262961cd2c9f7c19d1310c440aeb7","prompt_token_ids_sha256":"23b1b41e6716b12d13650e7aa105d3a76602e537cb02e4accc5f592ffd9c138a","ruler_category":null,"seed":null,"selection_rank":36,"selection_sha256":"165a7fd7caa5808dac419a6b8728f561f43cf78710437e2638e5c6a82bb7617e","sequence_length":113,"sequence_token_ids_sha256":"4582364354dd4ea26e506fc7c1845a9a6a878d03218ac93a0725170aef66c313","source_content_sha256":"30238651c7d4eb4dd16bbcf4c54ef3c91884f1d06eeb02ff38b97cb5e03a6225","target_token_ids_sha256":"9f6a8727f6485bb7bce051a35976a9ccfdc83c3af9309cb39f3a93789a51f928","token_span":{"cache_exposed_start":113,"cache_exposed_stop":113,"prefill_start":0,"prefill_stop":86,"scored_start":86,"scored_stop":113},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0a798f70d770e513f2e911c70a33835f666289b6e23ff31551a773ef75b38aec","anchor_positions":[6,14,22,30,38,46,54,62,70,78,86,94,102,110,118,126],"anchor_positions_sha256":"5dd8de14849d09f9037c328d85f9f1208f24e656ecfb3354483ee5aa62c57bab","canonical_id":"636","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,30,38,45,53,61,69,77,84,92,100,108,116,124],"fisher_boundary_sha256":"5f2e24eb47dfbebce18f2c2f64fd45917172c9eab2bd9cec54bb4fd111eb2142","horizon":1,"input_positions":[7,15,23,31,39,46,54,62,70,78,85,93,101,109,117,125],"input_token_ids_sha256":"4dc0a187eaaa7554b02100bc49019061f57a4f625ac418d8d6b7708a4c85f659","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,32,40,47,55,63,71,79,86,94,102,110,118,126],"target_token_ids_sha256":"3904fbb873aaac279d16ea071f1eecd284ddfaf982e8b4b118e527a7c5cd1663"},"formatted_content_sha256":"38120716e8579745533e38a1b2e5c3ab60964ce9826f4692be5d38a23625c7d5","generator_receipt_sha256":null,"identity_record_sha256":"6b00dd2ed569a9e5852db2dae0aa92821b9dbc00778205f14f3b67a886091087","prompt_token_ids_sha256":"c53edd29ef5c93c57c169235bf2b32cb118e20df561bc94d9b39faebc2a587c0","ruler_category":null,"seed":null,"selection_rank":37,"selection_sha256":"198ca758f04e266276eba3d3b469b52d783bc89308214f93d72af2c6d4e11dc0","sequence_length":127,"sequence_token_ids_sha256":"60e0f64c1dcb89512f1401f29f0d940ea152343ebc4764e52809ea4c5588f847","source_content_sha256":"054f097d046f21661577c0ed913168b649add5d35032de9efef4938f8a1a24ee","target_token_ids_sha256":"d1bf3a7c3a1a467f3f8e1eac6c4887bdfbdc387af6b7c29fcb19d45df9277f3b","token_span":{"cache_exposed_start":127,"cache_exposed_stop":127,"prefill_start":0,"prefill_stop":94,"scored_start":94,"scored_stop":127},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"d11ddad979bbc50d6db14cf77f968c46cb786a0ce2dc77ba5f5b28d500fc3596","anchor_positions":[11,24,37,50,63,76,89,102,115,128,141,154,167,180,193,206],"anchor_positions_sha256":"c25b8f380f538bd903f3e5b66a99e3d50700aeecbba5fa891f2350659ebff705","canonical_id":"607","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,50,63,75,88,101,114,127,139,152,165,178,191,204],"fisher_boundary_sha256":"a15bb1e5b2d2a3301ccdb2e0cb8998f04674305617ecc9881b7d20c208dd21c1","horizon":1,"input_positions":[12,25,38,51,64,76,89,102,115,128,140,153,166,179,192,205],"input_token_ids_sha256":"7ffaff1442181b24e662e144cb14110cb590e37477796426bec7dca4fc4d39ae","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,52,65,77,90,103,116,129,141,154,167,180,193,206],"target_token_ids_sha256":"05b028901595cb91b630adba80dd4555b614c059c122db592c5aa980143feafb"},"formatted_content_sha256":"ef35e3704df515714d76b547483c483743548e7cc069f31e95b66dec72a510e5","generator_receipt_sha256":null,"identity_record_sha256":"80bb8f3d83b1925af377e081ac063f63989368883ef4e04dc49b8fd8f00c500b","prompt_token_ids_sha256":"5bd1a90f46714aa81aed8226efa90f47442e2892377804149115218b9fd3b0c7","ruler_category":null,"seed":null,"selection_rank":38,"selection_sha256":"19faecf7ffafd480a6fa766dc937ba0709e42a5d5263c93130eb1037763f8871","sequence_length":207,"sequence_token_ids_sha256":"020df0554b8bcb9a4b7f491a4513578dc9b60d47ffc0d439886b3997c36eb986","source_content_sha256":"c541b88b979a7ec60640a49c7a9db7db502b97f5fed4d1970da91470204697c0","target_token_ids_sha256":"28dd01dbcdd5f0d19a9bef9f9fbc6491786783fdb97ad1fb07f945a93ffc7cc9","token_span":{"cache_exposed_start":207,"cache_exposed_stop":207,"prefill_start":0,"prefill_stop":141,"scored_start":141,"scored_stop":207},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"46371c5d2e1a407eb39f03172e7b9a21c7b932f5d655758a3c0ce64ba8c669a2","anchor_positions":[23,48,73,98,123,148,173,198,223,248,273,298,323,348,373,398],"anchor_positions_sha256":"8d9eba8a281ab33b27dd14ed9d345bfd773fffcabf08f24d986bb5ecd2f5015a","canonical_id":"950","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[23,48,73,98,123,147,172,197,222,247,271,296,321,346,371,396],"fisher_boundary_sha256":"843c418f1af24b45838fcae84492105e4320ec3725733998cacab7b6853efde8","horizon":1,"input_positions":[24,49,74,99,124,148,173,198,223,248,272,297,322,347,372,397],"input_token_ids_sha256":"d9a45e7a4afdd75464b73c4c6185b6888c636708adbfe2ece93d9586f2e48950","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[25,50,75,100,125,149,174,199,224,249,273,298,323,348,373,398],"target_token_ids_sha256":"b25ab7ecce47ea372a3f59f87969ac12e02c7358ce537fa47921278d5bbace65"},"formatted_content_sha256":"99720600b7a4e17df2bcc2762c83cdf70cc53626e5ec631c9bb32d7398d93e45","generator_receipt_sha256":null,"identity_record_sha256":"6d22eef549de3a81fc0a232affc18a615e9f0fce2de79d390b05e11ebdbf763d","prompt_token_ids_sha256":"8801aebf9d0be65bf1d4cac5f7ba35ceab845d7b0e20d5f6bfe717cf70fcac04","ruler_category":null,"seed":null,"selection_rank":39,"selection_sha256":"1b30f64a04ff98347d4a23bd49f8964fc6d3976e5d73b89babb36d891b544775","sequence_length":399,"sequence_token_ids_sha256":"0c20245c48d7a840676ca88457d31004701b407e4992f56ed7e4aca800bd3cee","source_content_sha256":"67c27b9d1db42865751e90ac9eb8dce913fa954d522bc50f1a124ed869dcc3b6","target_token_ids_sha256":"80ad07987d1074046d11f85abd02309f81753af95f7b2510f15b49ce074774dc","token_span":{"cache_exposed_start":399,"cache_exposed_stop":399,"prefill_start":0,"prefill_stop":85,"scored_start":85,"scored_stop":399},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"88045b346b8da1ff4c8b2efa7461ce5138643eb8b76bb78bed0cb8b0de1efbf4","anchor_positions":[11,23,35,47,59,71,83,95,107,119,131,143,155,167,179,192],"anchor_positions_sha256":"97c036645b9b6a78e53e848879a7e3ee08a70ff938ed6e98aa7098b21ab4319c","canonical_id":"885","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[10,22,34,46,58,70,82,94,106,118,130,142,154,166,178,190],"fisher_boundary_sha256":"810182ed20f2eb26e385debb7eceaceabe95cdd144936b72eb2f3945a5610256","horizon":1,"input_positions":[11,23,35,47,59,71,83,95,107,119,131,143,155,167,179,191],"input_token_ids_sha256":"76887b11d917341d902fd087a797d1b8148960fe5a5c3bd03223991175a7d00d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[12,24,36,48,60,72,84,96,108,120,132,144,156,168,180,192],"target_token_ids_sha256":"6d849b678d0b30547bb1a54e0ea0ac4ce468ce61603c7c10ffdab4c4e48459ab"},"formatted_content_sha256":"a16af1d88bf778a540f042c4c4f117a8a55e110b844fac976861bf833b58928c","generator_receipt_sha256":null,"identity_record_sha256":"210e74e96d878fcdcc906fc9925e4f204d9363fe486a768730fb41f707b3047a","prompt_token_ids_sha256":"731bddd1572f8ac6e10c56476d8d41ce0f5b35ad3da79217ad289e53697fc68f","ruler_category":null,"seed":null,"selection_rank":40,"selection_sha256":"1bf4005ab722efdcf542cb1b134158d2a2ec80c4bb99a978bf37320204ea651d","sequence_length":193,"sequence_token_ids_sha256":"221d7396fd6ba0730b59b2d163b37447727eeada7ea48a5bc944748d4c03df26","source_content_sha256":"c09a41add9e81f949a02fef6dde8a12287741692b045d26b3459d9cbf51a329b","target_token_ids_sha256":"a28fddf10e4bd937c83e1787639bbae521e60b6a3d5d421086e7e8d04ced9afc","token_span":{"cache_exposed_start":193,"cache_exposed_stop":193,"prefill_start":0,"prefill_stop":82,"scored_start":82,"scored_stop":193},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"5631703787387bea51be9c3d8508c71583e012079113a9f10bbec4245f396cf1","anchor_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"anchor_positions_sha256":"debc2f860d51bf30c9ccfe7850d3428800e80a005f976ab2943a8642c33fb808","canonical_id":"646","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,30,38,46,54,62,70,78,86,94,102,110,118,126],"fisher_boundary_sha256":"ac073c216400b3bc77300dbc3f3bd0735b111cf717a637bf602d1117e5d70dd5","horizon":1,"input_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,127],"input_token_ids_sha256":"06dec0ee7d5561ebf4a6924837605f275d606eb9ead89de4bbe6b7ec86a0c69d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128],"target_token_ids_sha256":"819be8fbf775e438d651a64a9a89d31a86d639d85622f2e6a979e72bf5469879"},"formatted_content_sha256":"42d2696d713ac8654462eed80e51a0f4a279ae3bd0b2fac3a803e706e31df249","generator_receipt_sha256":null,"identity_record_sha256":"a62e891eaa4a9d3d4ea47d185ffa6c03a9b1d28dcc7dec3429aece7413ef0936","prompt_token_ids_sha256":"e5220dd3a6c5ee6f26bffcedb6d05bd93a890b9b7d74832a9669975115e37150","ruler_category":null,"seed":null,"selection_rank":41,"selection_sha256":"1de0cdbbd1b50e7684dc3deead2b93036bacf525241fea00a2c68aeb5d1e9cf0","sequence_length":129,"sequence_token_ids_sha256":"eba6b177f82a24193e70a824138922ac8eee52933ebb80cb81a777d6e0f85943","source_content_sha256":"14c86e5b0221ac25a5d693c8996c478c0b60503bd396981a5ba2629d0c6b6f99","target_token_ids_sha256":"6c75f6536a4d7c5a2942fd540f9268e2e1961c4c3dd4523408da058253f543b3","token_span":{"cache_exposed_start":129,"cache_exposed_stop":129,"prefill_start":0,"prefill_stop":88,"scored_start":88,"scored_stop":129},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7105aa2d8387cb83f89b844f1f0a3176a54c1be40233dec8811a37cb7bee09d4","anchor_positions":[5,11,17,23,29,35,41,47,53,59,65,71,77,83,89,96],"anchor_positions_sha256":"bbeb8538883fad44ee21dd0032311ae447587513a5cfd0b0e896721a67ec397d","canonical_id":"708","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[4,10,16,22,28,34,40,46,52,58,64,70,76,82,88,94],"fisher_boundary_sha256":"977350ea6b433e2f68043b28b121e4816ce8fb2f7358ecf47291b22c9adc1129","horizon":1,"input_positions":[5,11,17,23,29,35,41,47,53,59,65,71,77,83,89,95],"input_token_ids_sha256":"a2ea5635b427055cb1e2e99736b9dd1cada7580fac25e57e539b72697c2a8ea9","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96],"target_token_ids_sha256":"81ee8d37b3f38a8c027e847f395edfdc8428e67a0cfd5bb3a5d12bfdac304ab3"},"formatted_content_sha256":"c810e44140744e432b3b9ff12b48167b78e7a00f4f078b3d9b5a3f539a0438dd","generator_receipt_sha256":null,"identity_record_sha256":"ab8e48a41997cface254c352c24a699bef6c43ae1858c93baddd4ba95bbef8e1","prompt_token_ids_sha256":"5a9bc94048f86f543dd1e7c3e8040a400a75b1f99b6ef637101d4590969f22bd","ruler_category":null,"seed":null,"selection_rank":42,"selection_sha256":"1e4c5d9ae6b7070238249263248d1c8593c90a70aeeb70af89325a5b3be72629","sequence_length":97,"sequence_token_ids_sha256":"23e232e07e81e613d2b01d8540dfd4e20039938471bf3dbebd3ba8c57449e8b0","source_content_sha256":"a670341f8d1f903c87ab584beb9dd6355bf23c707ac087606d85d5e1094169ba","target_token_ids_sha256":"bc70ef552d90fd9d0e6a01a2dad09bad247647bd98b2af5d9afba83f63d97b8a","token_span":{"cache_exposed_start":97,"cache_exposed_stop":97,"prefill_start":0,"prefill_stop":79,"scored_start":79,"scored_stop":97},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"21fdfa6c62576c444b2ede715e8c0f394692bdae740be5657a09fb9ac5465380","anchor_positions":[7,15,23,31,39,48,56,64,72,80,89,97,105,113,121,130],"anchor_positions_sha256":"4b59ffbfdc6171d05db7e4ce13fd81c13fb1c2704ef54cbfaede26eafe2b1382","canonical_id":"840","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"fisher_boundary_sha256":"f1a95ea699018f29238a0e61c6067bc03c5622df1f9f38ee7310c244a4f3bc65","horizon":1,"input_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,129],"input_token_ids_sha256":"db9ca78375767d3dc1fc43cd6274a248a1200a4e579725f08ae34f2d1594030c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,33,41,49,57,65,73,81,89,97,105,113,121,130],"target_token_ids_sha256":"d4ee1e1651e25e76151be8dfefda3d59a9153f9a6418389ce485081fde471280"},"formatted_content_sha256":"9aced07cb336136d232b88c651523d949c3ed1b58608f3e91ab8eb1e0f92a7d3","generator_receipt_sha256":null,"identity_record_sha256":"baa3355a5d2917d2ff4b713073abc3c98d3168719b1d9a669dfc8d2745e62db8","prompt_token_ids_sha256":"9b97f8bc9144022005c6d732aa6daf7482cce1d0462c9f41e4e54f455d770a9b","ruler_category":null,"seed":null,"selection_rank":43,"selection_sha256":"1e5afc59d9964484ca6906bae912fa0a9c9871e246e9a5aa6f4e2d2cd74a61e4","sequence_length":131,"sequence_token_ids_sha256":"ac96beecd6d569ac72fd89a9d15d46adb2c7e2ff3ab53a38a47e2ebac4b5c19a","source_content_sha256":"144d4c36a2f15b6079c87286abc57c5c33d11947809eec3b55d2c56b0abdca35","target_token_ids_sha256":"60cc8ab098d798eaf3a7d1177d7c65092515a6756c2fb167019eb1588a7a5f57","token_span":{"cache_exposed_start":131,"cache_exposed_stop":131,"prefill_start":0,"prefill_stop":98,"scored_start":98,"scored_stop":131},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"92d974a85b650173fa2852040052d155e429f141353a391c32eb9c8b22bf9582","anchor_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,208],"anchor_positions_sha256":"e3c9aaa391385675ab3475417e405da92d531052d937e046ddc609185dc2c7d4","canonical_id":"793","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,50,63,76,89,102,115,128,141,154,167,180,193,206],"fisher_boundary_sha256":"bcee19793289e8c6540ea881a8f492bb16b99a21c7f08133aae2e4a62163e585","horizon":1,"input_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,207],"input_token_ids_sha256":"ea265790ccf2b7ad1bc5812d8c7ad1653088a1b938adf116bbe32e00c4b3161c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,52,65,78,91,104,117,130,143,156,169,182,195,208],"target_token_ids_sha256":"03166cef7fd0abd95fc728344274b68579c18cdff9751955cb55d37472750e8e"},"formatted_content_sha256":"5990b44a405865af219b0ebfe72ca5f4807c7fa9ea8d17a998d181a60e7e6e2a","generator_receipt_sha256":null,"identity_record_sha256":"7694eeb6f1cf845cfd47b3f8c037b4453af3e6f1af57cbc1b271567d78747feb","prompt_token_ids_sha256":"a215965eef2fc3b7d4240f33fb78e20e16353f44c552a86876020a731cfb23ca","ruler_category":null,"seed":null,"selection_rank":44,"selection_sha256":"1efdf31f8359ff37ddbd6972a5b1f70a89aad0090992f1439e95a68b17f5805e","sequence_length":209,"sequence_token_ids_sha256":"b0ada4ec9a33d0cbd5317f8965026c54cce944770d6db49dedf1045e3e120210","source_content_sha256":"f51880de3d4f3f8df181ab757d4737250300a2719eb925110ed1a7025baefea4","target_token_ids_sha256":"1bb894993d9298db11a3f618f8c67b31b484d0fa664e5d9a74950bae08de8292","token_span":{"cache_exposed_start":209,"cache_exposed_stop":209,"prefill_start":0,"prefill_stop":108,"scored_start":108,"scored_stop":209},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"467b274cf4fd1ebd3fc1f65e47b4ebcef7ec6daac053d9c806d526d02b1aa9d2","anchor_positions":[7,16,25,34,43,51,60,69,78,87,95,104,113,122,131,140],"anchor_positions_sha256":"adf29a4dd3126019eaa451ccdb6bbdf039b38d8dc8e8cbe224d799517db47f03","canonical_id":"860","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,25,33,42,51,59,68,77,85,94,103,111,120,129,138],"fisher_boundary_sha256":"4298d52c37b3378977ffe8fcdbbb8de8a2001e201f91fa007b5bc8caf344f153","horizon":1,"input_positions":[8,17,26,34,43,52,60,69,78,86,95,104,112,121,130,139],"input_token_ids_sha256":"ff88adf412ffcfaf2eb5e6577b0406f784344c338d07181aab928b4e45c4aedb","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,27,35,44,53,61,70,79,87,96,105,113,122,131,140],"target_token_ids_sha256":"cb42fbfadbf84d40f31a090f455333c32b8d82097fa30bbe81c411778c79d549"},"formatted_content_sha256":"9fa46af05168957f4bcef1fe5cc161d5fd1f8aec927a8f95b00c13ce334a2346","generator_receipt_sha256":null,"identity_record_sha256":"f03d6c0de51871dd6a4c550ca9bb84df67fdd8703da60597ef107f6b4b5998cc","prompt_token_ids_sha256":"a9df32740687672e9bcea5fc074876046b128fd5263c7a057d9f57e081343d25","ruler_category":null,"seed":null,"selection_rank":45,"selection_sha256":"1f2c607266f18f654c50ab67f529127bf81f03531b6d79c46ac97f5081f09a65","sequence_length":141,"sequence_token_ids_sha256":"13d911b52ef7a2c203854b8ab41f9083733e9bda1ca8ed67607907aacf4bb0f8","source_content_sha256":"6948e55f2c0c12a3d6c826e33977a3b358f503e8cfccd743ee937a4f53cbcd15","target_token_ids_sha256":"37c5ab0e675f04a104215395c9a2a8503e4871823234c8abbb133ac6babb0449","token_span":{"cache_exposed_start":141,"cache_exposed_stop":141,"prefill_start":0,"prefill_stop":95,"scored_start":95,"scored_stop":141},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"37d7574d8d2b723c82778a768a2a566b7f49e8ebb69db0fb5d1e6794a930a2a8","anchor_positions":[5,12,18,25,31,38,44,51,57,64,70,77,83,90,96,103],"anchor_positions_sha256":"307c6a1944f2d94c59465c6af96e251aaf0ea665f8f1a3e0e55e3eb0568d8551","canonical_id":"800","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,11,18,24,30,37,43,50,56,62,69,75,81,88,94,101],"fisher_boundary_sha256":"54b8850d26dd8a6a43bc711ad584ae0be7143cd933d1768dbf78ab9d2e4721bf","horizon":1,"input_positions":[6,12,19,25,31,38,44,51,57,63,70,76,82,89,95,102],"input_token_ids_sha256":"5182574bad58e65b680691603b61908c731b487efc2d522fac61ffb07e8601e2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,13,20,26,32,39,45,52,58,64,71,77,83,90,96,103],"target_token_ids_sha256":"8836df71d734324c70c31d4de1e0ff0cdde8edd0e57b420ca0ed65319cddd528"},"formatted_content_sha256":"b47d463d8300a4f8eb63bc1a7f94408c9b894a1853d6b4b63d05eed61f97e078","generator_receipt_sha256":null,"identity_record_sha256":"83d65e18df6abcd124b640d68f38f75296045dc30136a8095de98e9a56ff6059","prompt_token_ids_sha256":"f32d5f100044bbea939023be6610b696d665c2e49ac55c3b2478fd990e92c75d","ruler_category":null,"seed":null,"selection_rank":46,"selection_sha256":"1fda23ecb8320de65f93f8ffeca9bc66d1d6ba82fa80bd9e2d70cb080ea19be8","sequence_length":104,"sequence_token_ids_sha256":"2095ea60e58d7a748f80d99738c7213d620ebe1064bc23e828b8298509455223","source_content_sha256":"251053bc7b023ed45edfd2a618f866b0c06fd77cb43c5c6c4b6ad81c515873fc","target_token_ids_sha256":"8ddb65e178ceb2fa2d463b1981c1b2a0d308bd079411c8156ce1cba86f28173f","token_span":{"cache_exposed_start":104,"cache_exposed_stop":104,"prefill_start":0,"prefill_stop":83,"scored_start":83,"scored_stop":104},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e3228f24ef2326d02eac4d51adfc4dac5f98a046b3f1fa2c9d460bc590e7ad00","anchor_positions":[9,20,30,41,52,62,73,84,94,105,115,126,137,147,158,169],"anchor_positions_sha256":"1de8f9c2ad33c2ca63d619d4e737066294b86b501bd566dd6f84cd9fa731efd0","canonical_id":"867","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,30,41,51,62,72,83,93,104,114,125,135,146,156,167],"fisher_boundary_sha256":"6cdf085217e9a92a862d08a98b72af716319adcb02639b733b87d792f63b1261","horizon":1,"input_positions":[10,21,31,42,52,63,73,84,94,105,115,126,136,147,157,168],"input_token_ids_sha256":"b3f41b7d18076730b37f241371d12f38791b7734d302d899fcfb2ae98bfa7e20","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,32,43,53,64,74,85,95,106,116,127,137,148,158,169],"target_token_ids_sha256":"f68aa42c7401aa2aec59c231aba070689444ab604d38e31d8644873caf22a672"},"formatted_content_sha256":"c5d4a00fec65b64c0d75bc557b515c301fadade8597055f2ebb3d094051ad750","generator_receipt_sha256":null,"identity_record_sha256":"ac31836b26155996736a3cd0eebcf92e2d25df1fbba42f351c07926a34d557cc","prompt_token_ids_sha256":"3965eb8ca467ffc97d00622d372a5e88bf51d885d8f90daaad663ac6a3125362","ruler_category":null,"seed":null,"selection_rank":47,"selection_sha256":"207c6e9619ca7dfd647996b0ddc78892694e1df2e515a63deabfce4784b7300c","sequence_length":170,"sequence_token_ids_sha256":"a45517724ce7682620b190d1954ae73f1c66fd716f768ae8411b3ebd337b8523","source_content_sha256":"2da1fe1d0cc596300fd25d9a0f0e96da35c84b424c329057298f8e3540157f26","target_token_ids_sha256":"c7498f64ba251e766cd9dfe1464d55b2e2348ec90765cbe0ec26f908cbac2a9c","token_span":{"cache_exposed_start":170,"cache_exposed_stop":170,"prefill_start":0,"prefill_stop":114,"scored_start":114,"scored_stop":170},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2f3dfc43992ed341ed7978295644b024d0c191c9ec9c0ab7120ce608b8e69aa6","anchor_positions":[22,45,68,92,115,138,162,185,208,232,255,278,302,325,348,372],"anchor_positions_sha256":"3ce5659702a26cd1b59b1d16461526035d0580f1b9adb943021fa501179be80c","canonical_id":"622","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[22,45,68,91,114,138,161,184,207,230,254,277,300,323,346,370],"fisher_boundary_sha256":"3dd21721af5c559d0bf97f5a0d8335eac58669690242a9a65414cb07123ceab2","horizon":1,"input_positions":[23,46,69,92,115,139,162,185,208,231,255,278,301,324,347,371],"input_token_ids_sha256":"b7ce16f1b70f8a32ac421274ec67536496f84420741a5dccf20534b7d1c9f94d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[24,47,70,93,116,140,163,186,209,232,256,279,302,325,348,372],"target_token_ids_sha256":"d9f650368349431e7cbd102c1fa8783d9dc57c0d60cd0a5c4c784738af1f4c19"},"formatted_content_sha256":"216e5e12034d35e11932dcbdc1b82573653565c8944e50d962328af76cf74782","generator_receipt_sha256":null,"identity_record_sha256":"6ac868c67f1d02fb4ef0c5a4472eb0b28625461b9b81108f54d3aa3aed1b7d70","prompt_token_ids_sha256":"521456ee3e26ec55cb42c2589084173f76e5275561b51201d380d4eed502f265","ruler_category":null,"seed":null,"selection_rank":48,"selection_sha256":"210000c62df8127b95ae009f6518f2d6201b9506f6c8506835dca4eb8bbf293f","sequence_length":373,"sequence_token_ids_sha256":"1407f58699c66ad8f21f74fcf9606d047865d22dea2021a94e18700ba9dec3a8","source_content_sha256":"2000263b26b6240378d05d98725621f6c3e2bdd39265a3d356bf34ba4967d9ee","target_token_ids_sha256":"4676880dc99f1babf2689d76705c312671bc2ff81019432ec05b98c6156e2587","token_span":{"cache_exposed_start":373,"cache_exposed_stop":373,"prefill_start":0,"prefill_stop":189,"scored_start":189,"scored_stop":373},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"38589b6a6ebd3baba9bd23ed78bdceef6846e6529843ad4232525292bdec2cbe","anchor_positions":[6,14,21,29,36,44,51,59,66,74,81,89,96,104,111,119],"anchor_positions_sha256":"e83bd2e331c896fb6e9946121cbf13d77b4819161e7591880bc3055c1dbc069a","canonical_id":"677","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,13,21,28,35,43,50,58,65,72,80,87,94,102,109,117],"fisher_boundary_sha256":"683f76b65c4ebceff74d9ba1f3da44c4048119e0043947c4b3fe743eb29acf47","horizon":1,"input_positions":[7,14,22,29,36,44,51,59,66,73,81,88,95,103,110,118],"input_token_ids_sha256":"5605175529bcf8df342d6991176c6dcd594b6850a1f276f1e516f05daaf07e33","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,15,23,30,37,45,52,60,67,74,82,89,96,104,111,119],"target_token_ids_sha256":"25df50b9075a9d7a5a4d1e2bac00e89268616661a6d17f51d339ca2bb9a7ad99"},"formatted_content_sha256":"441853ff923cc424ab0aa7fec31c4aaa73bd11a24cff64d1daad0da0b63dda69","generator_receipt_sha256":null,"identity_record_sha256":"0d876619f10e2042398c541c7464cf9b347fb64e1ac8107841be3973e213cc36","prompt_token_ids_sha256":"0d2e23bc0f9b51a958475c21ec2c44735aa0f223221380fb0cbc53e877eea9e9","ruler_category":null,"seed":null,"selection_rank":49,"selection_sha256":"213beddb6f6d7179f5bc0d037d084c135da38be5e6cebc224e9ee64574990d1b","sequence_length":120,"sequence_token_ids_sha256":"ef85c6f458a4c28417a4ebf89c6986376b20129c419177fae9b89453d5ee6bad","source_content_sha256":"33110b4565d527f05bb143f7b956a5df85324d2da6b82595171ebb52b1c28695","target_token_ids_sha256":"cb45a27d57424658ef3029fa23e359436aec54c0636af1e913e0b7b0b1a63c83","token_span":{"cache_exposed_start":120,"cache_exposed_stop":120,"prefill_start":0,"prefill_stop":85,"scored_start":85,"scored_stop":120},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0af2d86cf1695b8e2f191abffee4365a9731009779912cf6c2993138f08a66fc","anchor_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,160],"anchor_positions_sha256":"cea919870992245499fb8d8a858d848e1e76f02a4e2230ada935442533c03198","canonical_id":"849","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,18,28,38,48,58,68,78,88,98,108,118,128,138,148,158],"fisher_boundary_sha256":"dacbce55717fb2ad9e2c1c565b4cdc586f652e71481489ba9f2fe0fbdb01d223","horizon":1,"input_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,159],"input_token_ids_sha256":"abbb2fdf32082b7edb9cffcb691959ba71fa426d6242d133388daf5b12207e54","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,160],"target_token_ids_sha256":"b6dd6b170c917647555dae8b2934c8757a2075a9fd62bdcc5c741d21a155b8c1"},"formatted_content_sha256":"b97e246c5fb6006e691f8dcc86a87679113f7a78d64aeac8c4bdcded6e3527a5","generator_receipt_sha256":null,"identity_record_sha256":"4e41b786817605fe72e6c9fdd1fee25d51c2700b517525bce5f2ed34d6095542","prompt_token_ids_sha256":"57f92b2bb27768c978d8e3c41a9efc9e0ebbc4d569c63d7d84dd70b17515d47c","ruler_category":null,"seed":null,"selection_rank":50,"selection_sha256":"21f0779cb4b05ae80d3df3e594cf70e52fde9b8adc7fd7457a46be4c7a8aaf45","sequence_length":161,"sequence_token_ids_sha256":"aec4c452b53bb10d8fe6bcd4d1f7770db728b6d1f1928e376b6757e469976806","source_content_sha256":"fedc2450a7af5f522a92d0e1052fa8033d9fb19a48f2d667a7f31093a26f405c","target_token_ids_sha256":"45ed26c261c710a2cc312afbf071387265818704baa2885344b040c5f1d5b70f","token_span":{"cache_exposed_start":161,"cache_exposed_stop":161,"prefill_start":0,"prefill_stop":75,"scored_start":75,"scored_stop":161},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"f8a20553c70e4c3efaf0458db6c7c1d06d0e2dfc9ab4f2783f010af84e82721d","anchor_positions":[7,16,25,33,42,51,59,68,77,85,94,103,111,120,129,138],"anchor_positions_sha256":"08b85b98703c7ea55928e16b78a87011e7254eb0d0516881c1cef449b5ad76ae","canonical_id":"756","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,24,33,41,50,58,67,76,84,93,101,110,118,127,136],"fisher_boundary_sha256":"859315451573e13275eaa90fef292f4abcb53e2d7979744ca1a559cd552531ab","horizon":1,"input_positions":[8,17,25,34,42,51,59,68,77,85,94,102,111,119,128,137],"input_token_ids_sha256":"c70dd0ee0947d887043e4ef090390d9b55dcb4940348ebb271a5299668f94c6a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,26,35,43,52,60,69,78,86,95,103,112,120,129,138],"target_token_ids_sha256":"844ca5f05e19572138e424833fe71cb51a25987fd79b470130b42fa975c1a39b"},"formatted_content_sha256":"79bb829fa41f1b56a259ec1d6f01ea3454814c377128ca4380d61741eac44e55","generator_receipt_sha256":null,"identity_record_sha256":"1eb2616b62b6f8e94a42917d3cba2e04e70983835585b47745bb90e8e2aba08c","prompt_token_ids_sha256":"f39e99e8861e0fba4e540dc710f8fcf78133eb7be50e166ae431c650dfe3c4c9","ruler_category":null,"seed":null,"selection_rank":51,"selection_sha256":"21f1cc6a98b0063838189643fcd01e4b5a84fbc7b4cc8effe18cc4d33580ac9c","sequence_length":139,"sequence_token_ids_sha256":"c8504b3fc7419f64b9e0f081c9f129180471f53a90800918c11449b486ba7987","source_content_sha256":"cb27d1dd0a93a0f5b25d8f2486a5017c8d78e16aaed3c07e71a04cc440d1aa5a","target_token_ids_sha256":"66b6c358b326ed60b26f60ece3aa3a72bae96813ca059d87e44cc7dfcdb72c3b","token_span":{"cache_exposed_start":139,"cache_exposed_stop":139,"prefill_start":0,"prefill_stop":91,"scored_start":91,"scored_stop":139},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e32814316554b0f79cc5669981964cddf032347b4da8365b25ddadb67a9a91c9","anchor_positions":[6,14,22,30,37,45,53,61,68,76,84,92,99,107,115,123],"anchor_positions_sha256":"4161f0e0da9e01e9fca23a34e70979683578e6b7ae825c0791088249cd49ed8e","canonical_id":"960","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,21,29,37,44,52,60,67,75,82,90,98,105,113,121],"fisher_boundary_sha256":"1e87180a8c7bf1f65c301b64b88ff3c34dd59beba4dc53810695b834c0c5e9de","horizon":1,"input_positions":[7,15,22,30,38,45,53,61,68,76,83,91,99,106,114,122],"input_token_ids_sha256":"3f2fcef30c7d427d14cbcac1db88f5e74dec170dbc068333bce564584133cb71","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,23,31,39,46,54,62,69,77,84,92,100,107,115,123],"target_token_ids_sha256":"cbc1bf9d8cb5df620060a192384df8638cb094155f8978e9bcebfa5f6b4e3276"},"formatted_content_sha256":"3e4da97d0bddb12409d97ddcd69beb695eb66abac525bc59990220b07aa13b42","generator_receipt_sha256":null,"identity_record_sha256":"f15b0404946c938be5d272c212d67ae55ec18c9f4abddd2bf888443f303e871c","prompt_token_ids_sha256":"d4bcbfe95a1ebf023872fe53289683ee41d253e100dab52d598c34e9239b284c","ruler_category":null,"seed":null,"selection_rank":52,"selection_sha256":"2296dea570e1139150ccd317f7f4064da5f91c662c46dc92bacd2cafbd8693bc","sequence_length":124,"sequence_token_ids_sha256":"9c259585007e9ef7b2dde2fe8a3ecce5850274464b4c662d95e215932a3c753e","source_content_sha256":"b9eab9a91cd92a96f9a16006fb04105309ec0c2a0fe6dcbb8b8d6fd5c5ad7e81","target_token_ids_sha256":"ff3e6cdf6ef65136f4d5fc6ee8370dd4b879f47033e6e5ea58283168cdb8eae9","token_span":{"cache_exposed_start":124,"cache_exposed_stop":124,"prefill_start":0,"prefill_stop":65,"scored_start":65,"scored_stop":124},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0dad65eaaf28f71b8151a7dd5bfd62b3f371591729ce9a325cd6fdb1ee5d99d2","anchor_positions":[6,14,22,30,37,45,53,61,68,76,84,92,99,107,115,123],"anchor_positions_sha256":"4161f0e0da9e01e9fca23a34e70979683578e6b7ae825c0791088249cd49ed8e","canonical_id":"741","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,21,29,37,44,52,60,67,75,82,90,98,105,113,121],"fisher_boundary_sha256":"b7544cc2c06d73eb766987d95229a15c4109c2c9f31ee7a8109e1411b1f01f8f","horizon":1,"input_positions":[7,15,22,30,38,45,53,61,68,76,83,91,99,106,114,122],"input_token_ids_sha256":"523dd1079553b7be63d71e5a3bbe2ebd41dd50b1105590b40352ba6d4ac94529","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,23,31,39,46,54,62,69,77,84,92,100,107,115,123],"target_token_ids_sha256":"2ee4f36ffbf325d3c0f22cbdeb5f755c29cd1766c4b85bc98367f2e4e3a615fd"},"formatted_content_sha256":"d9e493674ac0bc6520c04b25bf6f2a33ab4010024516b62c9111a1e85fd4b7ed","generator_receipt_sha256":null,"identity_record_sha256":"09fd441b743d4db344555f2991adfeebeff8cbe76ceaaaa25dd4bfc5c8a9c2cf","prompt_token_ids_sha256":"a05bc2e81b72ead55f4ab23081aa3740bd5e94b12c6910ff66b4d5cd8eadc104","ruler_category":null,"seed":null,"selection_rank":53,"selection_sha256":"23920e4407f636372457bfab6052016160b52662fc2489114a897651a77f02a1","sequence_length":124,"sequence_token_ids_sha256":"a1e0d450709d8ec3da71149a480fb07c80668b98425c657471e339ba4b919bc2","source_content_sha256":"1b782afb3ea77d326167c0cdc75cf3bafd8c54f38e6656373399c6745f9a6cda","target_token_ids_sha256":"97d18e68c69d67cc25a2770126b11ec9a4009b8103ec5fbc12cf9840d69f3f7f","token_span":{"cache_exposed_start":124,"cache_exposed_stop":124,"prefill_start":0,"prefill_stop":77,"scored_start":77,"scored_stop":124},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"5220a6e57a98f18ce0fc3020e6cb3e579946006f270473a46fff4ee28d307f6e","anchor_positions":[10,21,32,44,55,66,78,89,100,112,123,134,146,157,168,180],"anchor_positions_sha256":"ecfeff7f0aa8026e3a3e67c7fccf117f33e2cac59220b1af92163b27f13f2667","canonical_id":"763","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[10,21,32,43,54,66,77,88,99,110,122,133,144,155,166,178],"fisher_boundary_sha256":"10832530119cc9dc7e8f231ec358a5a567e0064379fc6c4d752e727059021017","horizon":1,"input_positions":[11,22,33,44,55,67,78,89,100,111,123,134,145,156,167,179],"input_token_ids_sha256":"d8e6511cf865ccef1847855dd93e81b5f9a377da5712f1e2b5e6b7a1d615ed7b","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[12,23,34,45,56,68,79,90,101,112,124,135,146,157,168,180],"target_token_ids_sha256":"98ea0caf56819312967e777bc6cb39044d229924079654c58f5313c625d125a9"},"formatted_content_sha256":"1b9193d68d5fbee2ffb750c200030e38b128c512a5dd53901cd0fbc6e9110bf6","generator_receipt_sha256":null,"identity_record_sha256":"a6f87c07fe76e3512de8f93c16a19c9ddf32b2cb8f73284e08fa27a983b4e54a","prompt_token_ids_sha256":"e163147f03192bd26bc2e593039155967245812898896a5b3ccb488a19a3c2d5","ruler_category":null,"seed":null,"selection_rank":54,"selection_sha256":"253e3c374826f96651ce10f0c8bf234e0019e15677351f0a5fa4ac234821073d","sequence_length":181,"sequence_token_ids_sha256":"b058735b952391450e5ec12b9e1a916c6996ae263f8a33dfff1ecab2519a6f62","source_content_sha256":"c7874b2b046cc08bdc1d3485818b5e8aeab82684fbbfe9b46528e3f93a8c34c0","target_token_ids_sha256":"6c2dfbee49b175c1cadd33dbec419143d5d228a7875f0d9be04cef929c3fa52f","token_span":{"cache_exposed_start":181,"cache_exposed_stop":181,"prefill_start":0,"prefill_stop":113,"scored_start":113,"scored_stop":181},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1f12a81f4bcfa4113d746de8d83b09b12d5bf22d5365a3e007cbd5533bc387bf","anchor_positions":[5,11,18,24,31,37,44,50,56,63,69,76,82,89,95,102],"anchor_positions_sha256":"09c822b94eb395a26f0742b1f48efa1c07ea626dba571c077685b9f6cf7f997a","canonical_id":"924","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,11,17,24,30,36,43,49,55,62,68,74,81,87,93,100],"fisher_boundary_sha256":"9076cfcc1469ae3fc7502f41192086d69ec6e14f6981282dccd0613cf5f2b721","horizon":1,"input_positions":[6,12,18,25,31,37,44,50,56,63,69,75,82,88,94,101],"input_token_ids_sha256":"9c80dd3459e67298ce46335c51e9430667ca9c812c445dd5bda165182570e1d0","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,13,19,26,32,38,45,51,57,64,70,76,83,89,95,102],"target_token_ids_sha256":"5e9a4a1aad513778895ed6ffc2980b76512073ab29d4f8af14ae4e26e16fedfe"},"formatted_content_sha256":"e2c6271b02f2f53ca02cc8d25e64ff5633054d7c8d080fc11d7ea4f2182c8230","generator_receipt_sha256":null,"identity_record_sha256":"3c4356ae7b50150676b6594053c05745a4c6c37d4741a1a66db1c3426dfd2121","prompt_token_ids_sha256":"ec37b31ed3931f32732446c5e1b53853e57d83b71f6a60ee0973fc8544644cd2","ruler_category":null,"seed":null,"selection_rank":55,"selection_sha256":"257fcddb413e4d8b091f6e8d2f933e3a3702db72c89eba0d10b57f6f9f5e5204","sequence_length":103,"sequence_token_ids_sha256":"cfe2397b7ef9e35d26711486c836646e4598be218d361850d477f09eb5e24830","source_content_sha256":"fd2c6ff7f8bea809d74c434b1e914cc7cad37c2c16e71760033867b551f8b17a","target_token_ids_sha256":"e984af6102dc89335e836cc40d95bac564e888d124d337cc3b4d2293c69a593c","token_span":{"cache_exposed_start":103,"cache_exposed_stop":103,"prefill_start":0,"prefill_stop":79,"scored_start":79,"scored_stop":103},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b4b039a0bf0a677663a27b0e6225ea5659bf81bf4816ccd4aa2be56fbfe16c56","anchor_positions":[21,44,67,90,113,135,158,181,204,227,249,272,295,318,341,364],"anchor_positions_sha256":"2f350e6327acbc5ad68acdc17f3114f18631a5418462d1efa07225d17dee32b6","canonical_id":"839","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[21,44,67,89,112,135,157,180,203,225,248,271,293,316,339,362],"fisher_boundary_sha256":"b3aaea1f73f541b61f04c11357ca1b117be9c6ed377f11f60a993f9550d29493","horizon":1,"input_positions":[22,45,68,90,113,136,158,181,204,226,249,272,294,317,340,363],"input_token_ids_sha256":"14db27ee204411b76eac1110cc2eae63a9f7b2b51b8dbe5b39e057e911aad683","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[23,46,69,91,114,137,159,182,205,227,250,273,295,318,341,364],"target_token_ids_sha256":"db3248528512ace9a5926f28510801745ad1c3f595f00a85a70220785378a26f"},"formatted_content_sha256":"1fa99f6cf27a6488f0fc65f628a1282d325113dc82a07170cace8f0ceb591a14","generator_receipt_sha256":null,"identity_record_sha256":"6b6998a44c9be26bdd92e8fd5c97777ab53fadd0893edc4f08c5b3238dd07613","prompt_token_ids_sha256":"4be0192a68dd883cc6c905e04481fe02c58edb320e51cffa3148acdb235c27c9","ruler_category":null,"seed":null,"selection_rank":56,"selection_sha256":"25acfdc7bd97d8b8918d952ccd13767a2fe3d9db392ff96de0914fe990c96767","sequence_length":365,"sequence_token_ids_sha256":"64f01ee3dd9e450ba6406f15a3eb8206442ffeb1f8bad35135cdc10036187849","source_content_sha256":"e16fff97f3fb1d8593c7f03f52f248825604aeb9e11bb511b613867c12fe0d69","target_token_ids_sha256":"beb8edb12ebba49a31efbb316488aaea2754c69c9a528a3fc8ce2e25cfd56f2b","token_span":{"cache_exposed_start":365,"cache_exposed_stop":365,"prefill_start":0,"prefill_stop":292,"scored_start":292,"scored_stop":365},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4e7d65195091102780a3b899d1d9c169f8999fd52b4fce6cd4807da888ce72a6","anchor_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,207],"anchor_positions_sha256":"db093e77a831efd90fa9c853cbfd07cb4a1ce073a3d26ebd584589b17a67dcac","canonical_id":"870","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,50,63,76,89,102,114,127,140,153,166,179,192,205],"fisher_boundary_sha256":"e55af1a7964dd59bb74a21c91ffa723d2be196dc03399d61f54ed5e27acc11f1","horizon":1,"input_positions":[12,25,38,51,64,77,90,103,115,128,141,154,167,180,193,206],"input_token_ids_sha256":"7e51406e9532ccd83f5838a79143de8b54bddcb3cd4af48ff57ce8fd2068004b","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,52,65,78,91,104,116,129,142,155,168,181,194,207],"target_token_ids_sha256":"8dfb99d49da149bda798f4f1bea7693afcc685391f66174bd7dbd7f9f212d23a"},"formatted_content_sha256":"0671c01c177d3d976dc26f74b1532f125da8dbbcba201bd969a8df89be404212","generator_receipt_sha256":null,"identity_record_sha256":"80065f067cbcd3b6b24a83d3b0920140345cdb44209861d9942cfc40e6285afd","prompt_token_ids_sha256":"11ac1cd8c2c5a0bfd9a43b00c6dbafc129c255c7fcab76b0fccec71b0a1ca47a","ruler_category":null,"seed":null,"selection_rank":57,"selection_sha256":"25d5e31e60deea1f490886549d37023191fddfed4a7488c2ecb258dc7b690a34","sequence_length":208,"sequence_token_ids_sha256":"b95e7638baa81487e766778b92d51418007619c24c88596cb9d255d17707ad82","source_content_sha256":"25b998a219b3e624a18b48a424e9201d591646f6ccffd2c34ccd994d27c33467","target_token_ids_sha256":"2c9edc0f2e71ceb46be5cf74a768dd85d0f0568d35da9346930b4437902992c2","token_span":{"cache_exposed_start":208,"cache_exposed_stop":208,"prefill_start":0,"prefill_stop":171,"scored_start":171,"scored_stop":208},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7179a32a80e84307ba53490fd3d26ee4976520fb95463fd4d99c5eeea5f95e3f","anchor_positions":[10,21,32,43,54,65,76,87,98,109,120,131,142,153,164,176],"anchor_positions_sha256":"1a7be41f5dd5dc6797d88e73a587747cdd29dd38e9373f3f584d8051e8e93686","canonical_id":"862","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,31,42,53,64,75,86,97,108,119,130,141,152,163,174],"fisher_boundary_sha256":"1eb3dcd5d0f40487914047d0829182878435e58383687684c8e2311b203aae37","horizon":1,"input_positions":[10,21,32,43,54,65,76,87,98,109,120,131,142,153,164,175],"input_token_ids_sha256":"ab632089e4e539c8bd8472cc5c658a6c7457d8f007cff89f0e00204509423230","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,33,44,55,66,77,88,99,110,121,132,143,154,165,176],"target_token_ids_sha256":"a1faf4cf5f2ef1412bad68d667096b1c2dd483ffe1a6e641ce38ec7132214cfd"},"formatted_content_sha256":"16b2614d3f18e7e4f72755cb6542e4c740bbd15964d402f5b8deeaaf2091d4c5","generator_receipt_sha256":null,"identity_record_sha256":"40dd54e4e34693f385579ecbdbbfe9d627292003fa2bb0e3fa8bb8b2106964dc","prompt_token_ids_sha256":"a20a042120ba3b2dda09c804a843b20532bf4809c88d3fbe22620ef7ac56bdcf","ruler_category":null,"seed":null,"selection_rank":58,"selection_sha256":"26d4c81b5d924024df658aeaf2f322dcb51e74e59299d6c5bbd15f755f66be5e","sequence_length":177,"sequence_token_ids_sha256":"f51964e8ff8c85d8f090c0e12f2c0d8d6234b49c9b6256141d05ab65cbec2f56","source_content_sha256":"4ca1375a378716ffef02c6f4407d8935736ee95bd694754667893b4d008fe701","target_token_ids_sha256":"643aa7765da4c82ea6a56654b78d8a8b5489c8fe86d2c44a8b4ac1960295ddbf","token_span":{"cache_exposed_start":177,"cache_exposed_stop":177,"prefill_start":0,"prefill_stop":130,"scored_start":130,"scored_stop":177},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"c008fb00c4b75ed5a7d4956b63ea532e7def48f60c4527826df4af3da44bf419","anchor_positions":[4,10,16,22,28,33,39,45,51,57,62,68,74,80,86,92],"anchor_positions_sha256":"adf3767c4c3aef64d448368c378ec2e8623df9971d13f03dfbf0a245e0d32c61","canonical_id":"877","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[4,10,16,21,27,33,38,44,50,55,61,67,72,78,84,90],"fisher_boundary_sha256":"2b4cbf4ae8461adf7ece8f2d75ed8ba8fa4760a94e8ccbc37beabe48a2e53822","horizon":1,"input_positions":[5,11,17,22,28,34,39,45,51,56,62,68,73,79,85,91],"input_token_ids_sha256":"11d9eb276cb6e38836f4280a877421b19798f0a03c8c6a955cf798cffbc5e066","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[6,12,18,23,29,35,40,46,52,57,63,69,74,80,86,92],"target_token_ids_sha256":"9a807734bc0b75cf522407e78a2432d110e1c4f600034c299f2c1f1025bf651b"},"formatted_content_sha256":"f0d53963f87750fbc186ec354a9b0e5f12a9937a4a1e0aa5b14a393ae9209437","generator_receipt_sha256":null,"identity_record_sha256":"b69efa65a89b1b762f27440c2fa6d016a863c179526e602947c476c8cce0c500","prompt_token_ids_sha256":"decd083790a01b0d5ff5f3cedced5d9fc60db4ac239215f87d18bd13829340ee","ruler_category":null,"seed":null,"selection_rank":59,"selection_sha256":"27e1898502ba84f86677c9a8c281f209e5e5ca3ef3fa0ac74ccb2e5a0f96925e","sequence_length":93,"sequence_token_ids_sha256":"0c186015d98c4b3b835615571d680037c4e9ff03bc0bf360d7edcc64ea7e02ce","source_content_sha256":"a4d868515f48168874e3345764d92c8f9bed1be2211fa137d653955ce91e9231","target_token_ids_sha256":"05325f40cfa6f59d7dae81f8cf700730aebfff74fb1d0bfbf6df3b22888a7ec9","token_span":{"cache_exposed_start":93,"cache_exposed_stop":93,"prefill_start":0,"prefill_stop":71,"scored_start":71,"scored_stop":93},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"69bc3d226f2cb8a6dd05d7b71e6b82bd59279baf47672ff9264b7f90657b2ed6","anchor_positions":[9,19,29,40,50,60,70,81,91,101,111,122,132,142,152,163],"anchor_positions_sha256":"868d4ebf721b69ebc32ed80fb23287a93509d3cda6e8df7dfacdd2b56b0be622","canonical_id":"672","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,39,49,59,69,80,90,100,110,120,130,140,150,161],"fisher_boundary_sha256":"824632fca0be6e4fc45bd8a157d6c3d7331848be3ae62303610c9ced5cdc4fe5","horizon":1,"input_positions":[10,20,30,40,50,60,70,81,91,101,111,121,131,141,151,162],"input_token_ids_sha256":"4a43f0478e855c786a9ba41f412abef099a0aa6d336add8b63253a02536aff3a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,41,51,61,71,82,92,102,112,122,132,142,152,163],"target_token_ids_sha256":"5a469a6d9ca0e01f116bfc1713d0adff5f5c3384fa311978e20183e1b4e4cf16"},"formatted_content_sha256":"bd36f5a6ad55b181f08602b3596e02127fc18650dff36d4e78601272f208d8d5","generator_receipt_sha256":null,"identity_record_sha256":"b5b377fb132254b8031c1002255dcb18bec16d00c60ca2ad998a8649aee007f6","prompt_token_ids_sha256":"6b00d6059701e0be4b83e9dac571ad728ef3c38c00c26f87d6b67a85ef08f81a","ruler_category":null,"seed":null,"selection_rank":60,"selection_sha256":"28dac04649a15d3243c93bc4206efd4f0acd9190fdbbdfa2ab5345af799d2bf1","sequence_length":164,"sequence_token_ids_sha256":"db9853f0252606fe408f3f653940737ffa69d88b087eaf4b282f42551da66ed9","source_content_sha256":"2e61b0813b9f940f7d87c5121e73ed7a0b8b2d507012929e153f7b120c8c94ea","target_token_ids_sha256":"4282f90a8268a9ed7bc92f471b79152055c095686127816dd9ad5e82abe8893d","token_span":{"cache_exposed_start":164,"cache_exposed_stop":164,"prefill_start":0,"prefill_stop":87,"scored_start":87,"scored_stop":164},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1de6107371ef2883413c366d370b6b037d94c76ec7232ca2b34bb6d3efeed728","anchor_positions":[13,27,41,55,69,83,97,112,126,140,154,168,182,196,210,225],"anchor_positions_sha256":"87cf1b468f68c1b9ae4a445f5ca6ddac019379b906bcb9569e18bc77f8dab19a","canonical_id":"894","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[13,27,41,55,69,83,97,111,125,139,153,167,181,195,209,223],"fisher_boundary_sha256":"50541ef032b9ebd546fb5146735c5dc646a875ccea81f95e3135cfb092f777e8","horizon":1,"input_positions":[14,28,42,56,70,84,98,112,126,140,154,168,182,196,210,224],"input_token_ids_sha256":"29b3d6ebd87eeec326946bda12535cdd931e5c75481c27025e5044b0cf8d5ae0","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[15,29,43,57,71,85,99,113,127,141,155,169,183,197,211,225],"target_token_ids_sha256":"468a710354f96e062d0f534f2df0e373bceff5193c12fd9ae67b05776f4cb12d"},"formatted_content_sha256":"85fe4f431f987f9bcccff54e9fff464b3411229d14fbb421bec0c42bf02159f6","generator_receipt_sha256":null,"identity_record_sha256":"78c4954da5b1f2cd2dea1777612b390c51dd7f053d98f84f8dd93e4c16182ef1","prompt_token_ids_sha256":"366bcf6fff0a082cca6c159ef4ee352857b3adb5dc747a33fbe9d5a187cf1a70","ruler_category":null,"seed":null,"selection_rank":61,"selection_sha256":"2a07715411e7dcd5d13fd6bdab1eedf96aa272627f4dcec5410c9273f87bda59","sequence_length":226,"sequence_token_ids_sha256":"060aa83ed0ef449b43aebe0dba81de96624ea81a91b9f2bbf5346ec7ebba5d2d","source_content_sha256":"25f8c228a65fe0501ff126cf8215ef8c40f73ba446adccaac8696c0592d9cb99","target_token_ids_sha256":"5d62950c056f097d29c50fc4fa74da37364692566b85b25d8801ce7fd0cfae6e","token_span":{"cache_exposed_start":226,"cache_exposed_stop":226,"prefill_start":0,"prefill_stop":198,"scored_start":198,"scored_stop":226},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e9fc3e673c65e510fa4e60bee32295e16ef953843d59dc679325a149cb63954e","anchor_positions":[13,28,43,58,72,87,102,117,131,146,161,176,190,205,220,235],"anchor_positions_sha256":"a5ffcefdcb22ed36cff0992ac04165c50a214c58858b64309c2292bae4ad2df4","canonical_id":"911","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[13,28,42,57,72,86,101,116,130,145,159,174,189,203,218,233],"fisher_boundary_sha256":"a343b73549c45e626b75cab9d67da7ec20d01d1df92c5ea43c27940cd5ac7b0d","horizon":1,"input_positions":[14,29,43,58,73,87,102,117,131,146,160,175,190,204,219,234],"input_token_ids_sha256":"fd590f614ed1c54400b44a8923440159f0d548ea06103f72a56c7794eef8d355","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[15,30,44,59,74,88,103,118,132,147,161,176,191,205,220,235],"target_token_ids_sha256":"84f73363046430aa4e0c6c4dcd11b1cf627b0582b1c34fc1e0b65e35a6e9f9db"},"formatted_content_sha256":"3af453e7ae9642b7b3a6fcb1a5eeaa953e64e0f8b2a19e654ef7c76e72f1ff54","generator_receipt_sha256":null,"identity_record_sha256":"f4ea17bad40bae1caf3eba8695e7ad325112bd00019e2511ff3614fb4f3a37f4","prompt_token_ids_sha256":"0b618b4457d4e22f6ca61e219aee86cfa88f0ae704ed487315c854f83ad3c11b","ruler_category":null,"seed":null,"selection_rank":62,"selection_sha256":"2a3d19972c5985a3c0f494d32fc1cf627ec69e3ceffcf0239862d2c14a69ee16","sequence_length":236,"sequence_token_ids_sha256":"4b476aa638606f6f73086d180b8d59e8f59f03a7f0df8b934e603b7a121b0987","source_content_sha256":"fdcefafb28d63d4587a84cb4bad47b369563ee74a482a1216341f7f6dba5f2d6","target_token_ids_sha256":"de9d6a1e7f34b8648d9cc024467db18a0100561d5d78da9dd043028f0c4b1058","token_span":{"cache_exposed_start":236,"cache_exposed_stop":236,"prefill_start":0,"prefill_stop":171,"scored_start":171,"scored_stop":236},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7b01d8ec76dac2b7f8b13a0dee07ad2225b142de86be4c08fe417f349fab06b2","anchor_positions":[11,23,35,47,59,72,84,96,108,120,133,145,157,169,181,194],"anchor_positions_sha256":"366062d114b4b8abb7d67fff75b399eddce3c91d9f37bfd07f29cdd16e658363","canonical_id":"907","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,23,35,47,59,71,83,95,107,119,131,143,155,167,179,192],"fisher_boundary_sha256":"3e6fc1c5625bb4f9b6651ced2a909d037dc5aba4278dcfc13b85144e9142af0f","horizon":1,"input_positions":[12,24,36,48,60,72,84,96,108,120,132,144,156,168,180,193],"input_token_ids_sha256":"ede325c0962debe0dc616d99ffbf44bf49b11833fab47e5f51463bdf2ffd6011","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,25,37,49,61,73,85,97,109,121,133,145,157,169,181,194],"target_token_ids_sha256":"1d7cf9fe79dc3a0f99a9c1c0dc95972f78b47a89b20d976ae513c41b67a2b4ea"},"formatted_content_sha256":"99bd7655cb5e40f7f33cb82dee142cbc78854fc3a5fe6777c50f8ee8f6340cbd","generator_receipt_sha256":null,"identity_record_sha256":"8d7818a7f7ed0b3732c51529861e8d1f37b442f565e0755ae9412f4e544cb0d8","prompt_token_ids_sha256":"1d66bbe818bc1c82dece92e787302ed8be7bcca6dd90590e6de294c4d367d8be","ruler_category":null,"seed":null,"selection_rank":63,"selection_sha256":"2a42596fa08d9ebf6f8f895ed62f34cfc00f343a1b8fb9eaba4fce50e57632ff","sequence_length":195,"sequence_token_ids_sha256":"0239420cf0bc52b4ff133acfa7d4ba2d445aa4efa7ca99f95e10b1e7fb7bff32","source_content_sha256":"523274b507fb693b2bb38a9148d6f328b15704929202f43a1eadb3f695c038b9","target_token_ids_sha256":"8225f2a151273aa3c714667358b8a7f304f775c7bed4fa3e4aececc369b6aff8","token_span":{"cache_exposed_start":195,"cache_exposed_stop":195,"prefill_start":0,"prefill_stop":139,"scored_start":139,"scored_stop":195},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"dca37f2fe51bc394708fd6c2564b0cbf119163fd665608a4d93cd495b3998ad1","anchor_positions":[7,16,24,33,41,50,58,67,76,84,93,101,110,118,127,136],"anchor_positions_sha256":"b394d1b64998869298fd2b292d4ab84e6faf0b5a40eeb811c668f2563bb4df64","canonical_id":"744","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,24,32,41,49,58,66,74,83,91,100,108,117,125,134],"fisher_boundary_sha256":"31ee43d6395a61f1005ea89235db2d9169957c4fa0ecd817d09d92731e4bfff7","horizon":1,"input_positions":[8,16,25,33,42,50,59,67,75,84,92,101,109,118,126,135],"input_token_ids_sha256":"305b211372837f28d3c09a9195ff9da8870eb18255d1a5ade3aa23447ae066d7","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,26,34,43,51,60,68,76,85,93,102,110,119,127,136],"target_token_ids_sha256":"55671dc7b5a880e5aa6b49754d8ad5916d76392b74ba47f6eb14a67a99a65dd0"},"formatted_content_sha256":"24bfd2c8acd42c24dd0175dade66586d4ef9c8fba6740ab43be8a40d2344a8f0","generator_receipt_sha256":null,"identity_record_sha256":"0d3378f11bdab707d5a5e866bf31ece77bae782c5e98264606c8669811279e26","prompt_token_ids_sha256":"45cbb79cde4efb50926321115cb572632eaa8fc91f1b03e56edb1ec881aac0d1","ruler_category":null,"seed":null,"selection_rank":64,"selection_sha256":"2b946010dfbfbf789ca965f025d6a4ef990e3f3685bc221716f664a8af147cdc","sequence_length":137,"sequence_token_ids_sha256":"27302f59061cb4d77641b855190e68d1082efe4fa19ef17f6d1886ea443d271c","source_content_sha256":"388472fd029ca253a1e8ebe0b058b306b19e8d0a99d80ac0ff0cc80f6ef6401e","target_token_ids_sha256":"f529366ede3a080401140af1ed24d8b5dc889784a050980a1baf21988730f4ef","token_span":{"cache_exposed_start":137,"cache_exposed_stop":137,"prefill_start":0,"prefill_stop":106,"scored_start":106,"scored_stop":137},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"72f1915bbcebb3cd41116e7d18a62d81645a57d6ab6aefc5dd4d833040f03e86","anchor_positions":[10,21,32,43,54,65,76,87,98,109,120,131,142,153,164,176],"anchor_positions_sha256":"1a7be41f5dd5dc6797d88e73a587747cdd29dd38e9373f3f584d8051e8e93686","canonical_id":"858","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,31,42,53,64,75,86,97,108,119,130,141,152,163,174],"fisher_boundary_sha256":"dce1d589b941aa5d007836b839a80510d6ed27d69a6bc4826bc4f9d1be58fa8a","horizon":1,"input_positions":[10,21,32,43,54,65,76,87,98,109,120,131,142,153,164,175],"input_token_ids_sha256":"1f8edd352b430cc0ec42d176428e4dc421cd81f094568d0cef527f3c289d718a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,33,44,55,66,77,88,99,110,121,132,143,154,165,176],"target_token_ids_sha256":"bd43a32b864185de3d60dd13a588ea5c44371aff58aa9439eee1749afedfd427"},"formatted_content_sha256":"ffcfea4052a70d3b74318b9aa27367efd191b6bb39153e56b0d8c6be270bf7dc","generator_receipt_sha256":null,"identity_record_sha256":"4303eb5887d9002f18afa44a53d86830e876c3605b807968cb2327574864c1ad","prompt_token_ids_sha256":"7e2722d66f52a4b3e9bc2b499fe7271693a49bfccaf075db172096a06590501c","ruler_category":null,"seed":null,"selection_rank":65,"selection_sha256":"2bdf2de5737b5e9e9c804506d9b637c0b3e4c429c713402914877db9e163698b","sequence_length":177,"sequence_token_ids_sha256":"1bd4f0e350599ac78e054ba1f3313444f17aca30b3ab8a7d1b97b0a4db36795d","source_content_sha256":"30117a9c1f5c8fa1a71aa5d47b10e7b6a6ed631272d09a212703db54dc4e2b48","target_token_ids_sha256":"8597d05e46c539c2c6176333f41f2d43081ba451957d9e6fb3fab21ad0354486","token_span":{"cache_exposed_start":177,"cache_exposed_stop":177,"prefill_start":0,"prefill_stop":161,"scored_start":161,"scored_stop":177},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"90e208ba484962acfe5a450f2238cad06fa1b1e78cb9b693ad7f8e119d289e81","anchor_positions":[9,20,31,42,53,64,75,86,97,108,119,130,141,152,163,174],"anchor_positions_sha256":"3a23db0bdf537a9cc8cf4af3c5b682f02edafbf8e11933d0867e7f5a84a5d5fc","canonical_id":"908","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,31,42,53,63,74,85,96,107,117,128,139,150,161,172],"fisher_boundary_sha256":"3ddff55453bdfe39ff17279167d0bf79d250df7e3566058fbc8400d75148e0cd","horizon":1,"input_positions":[10,21,32,43,54,64,75,86,97,108,118,129,140,151,162,173],"input_token_ids_sha256":"a57955a9e52e60a3866dc92a0785d9ec79382d7f1e9d9a6fbe931b41cbb23f05","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,33,44,55,65,76,87,98,109,119,130,141,152,163,174],"target_token_ids_sha256":"08a89b4fe30bb44a957a32e4dbff9cce9f766b22eb990ab1c15ed78cf1409870"},"formatted_content_sha256":"542026924d7c553982b9f71b589a6b03aa17c1a9071232b6b352a8d6de880b4f","generator_receipt_sha256":null,"identity_record_sha256":"9ab67c828f5ebad51f93eefcc25a6f64e7b2efa682ddcaa0b13cdfaae34bb357","prompt_token_ids_sha256":"92dc6abd8d2bcf9552cb9117b89f426545c93d4bf69ad859fe2e22691583d830","ruler_category":null,"seed":null,"selection_rank":66,"selection_sha256":"2bfb0c5069394a29410fe17d912939039348a3279be0932efb83b1a7c531f5cc","sequence_length":175,"sequence_token_ids_sha256":"470fc1e9439dda64ef76f847132fd61d9cc29b4d1eae89d0d27b45dcf9b147bc","source_content_sha256":"7f2ca7ee05543293bafe5fc5ea510834c2eadcb6c383bdf04c72ec8b68a255ba","target_token_ids_sha256":"fad64c8dc2d584216528c7db193f37eff8c35a166fa9f081fea4f50fc1596f9c","token_span":{"cache_exposed_start":175,"cache_exposed_stop":175,"prefill_start":0,"prefill_stop":143,"scored_start":143,"scored_stop":175},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"ae2a13b9227b88c92f44ed18f4a1e74a1af336a5e505f3993199f634831d84bf","anchor_positions":[9,20,30,41,51,62,72,83,93,104,114,125,135,146,156,167],"anchor_positions_sha256":"eaf3a0e814f88177633695c30e9a05b6b492aa64c90bf98da98cb9be3917c812","canonical_id":"695","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,30,40,50,61,71,82,92,102,113,123,133,144,154,165],"fisher_boundary_sha256":"00b58a22e1ec099f9f89a453c68d32a00b7de79e01353970212f4486aea7a91b","horizon":1,"input_positions":[10,20,31,41,51,62,72,83,93,103,114,124,134,145,155,166],"input_token_ids_sha256":"0d02b321a2c7a93e523c4102c0a515f78295358243305ecf63572a9b9251f793","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,32,42,52,63,73,84,94,104,115,125,135,146,156,167],"target_token_ids_sha256":"479412a3d9bc5c68516840b01dfe7229e1ffffaec904e9405309df32f66e5318"},"formatted_content_sha256":"2c108150869bff59ce632e2f1cf6126ae8d1e5449fb81fa352e6080ece8e18d8","generator_receipt_sha256":null,"identity_record_sha256":"b8d1fbbaf80e50801cbfefb607f87f2ee0be785618f9d8733e4ec1cec193070c","prompt_token_ids_sha256":"12d9523360c9b9bbbc77331eede017f4df2b2073b06793fb6caaa4264ed356eb","ruler_category":null,"seed":null,"selection_rank":67,"selection_sha256":"2c8457d859c30b49c20f86b5a6d57cb9d3f87babdd78e191666700bc6b19c2a9","sequence_length":168,"sequence_token_ids_sha256":"61a8f24e31170e6510a3fcb1c13e03df674e6054bc07e8c1b50b2869262279dc","source_content_sha256":"f26ba058dff18925efba9c51a7c584d713db98910e092b2b8efad4050d705a97","target_token_ids_sha256":"331d6876269e5c72808d45d0ff8043245793020b7f0c47f8dff86c3d5896440b","token_span":{"cache_exposed_start":168,"cache_exposed_stop":168,"prefill_start":0,"prefill_stop":124,"scored_start":124,"scored_stop":168},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2776f19ea8ad43a4ca532ba12548c47a68fb5d577db33f2001594d30daae4b47","anchor_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,127],"anchor_positions_sha256":"be2b9b89820981ce1f31651cf7733d2ed9a608d27c6984c26e1c299ac3d463c0","canonical_id":"674","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,30,38,46,54,62,69,77,85,93,101,109,117,125],"fisher_boundary_sha256":"9fb839c7596b94297977a40da9a8bdea3ac37720cc556e470eac01fad7d8fd26","horizon":1,"input_positions":[7,15,23,31,39,47,55,63,70,78,86,94,102,110,118,126],"input_token_ids_sha256":"4f531382690861c06fb87e927c47ddec1b08260da41fc61f8ade85e2dbfa2447","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,32,40,48,56,64,71,79,87,95,103,111,119,127],"target_token_ids_sha256":"58f5f25b01c19e81aae2e0327e1c394b668618a5926e174aeb8d4db536ad7b5a"},"formatted_content_sha256":"11e4332442cb1751b96af99218e5ed832e0741b4bd279519954cf8ff75df3000","generator_receipt_sha256":null,"identity_record_sha256":"91c60f95fb1de5699a93eac54f860ec4b636acdf00d4bf36d47cb7038568d0e6","prompt_token_ids_sha256":"fc30e67c09e80fb8040481e2b49df5d4de0abb2ca503e59d5f7fe81b9053ea14","ruler_category":null,"seed":null,"selection_rank":68,"selection_sha256":"2c910566f006288010eaa2a19e63636d5cdb71c4dbe532bbb2f800b8bc2d83e8","sequence_length":128,"sequence_token_ids_sha256":"c6637a5c180bd2cb7b6ae0114077150e40aa103770a4414d373677e6197d3eb0","source_content_sha256":"b085122555255bd8b48a5d7202b27dcfa83512407504a3a7d36c26ab0971701c","target_token_ids_sha256":"bf8896134a16b2defd1fd3ad3a1e041bed0c2522bb90fb8750e2d750970281f4","token_span":{"cache_exposed_start":128,"cache_exposed_stop":128,"prefill_start":0,"prefill_stop":92,"scored_start":92,"scored_stop":128},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"62ab5323d823671ba9386621641396cc44f5d6d879e1ef5cedfd73f5d42ea49d","anchor_positions":[5,12,19,26,33,40,47,54,60,67,74,81,88,95,102,109],"anchor_positions_sha256":"04b0f7fc4a0267db36244bf79b1775bc8efc674979a8a894100f0164758c7107","canonical_id":"937","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,19,26,32,39,46,53,59,66,73,80,86,93,100,107],"fisher_boundary_sha256":"d621358bf5b31a9a6e1bf8b7861671a5c3b7d8f4b833117f22528a32214d9059","horizon":1,"input_positions":[6,13,20,27,33,40,47,54,60,67,74,81,87,94,101,108],"input_token_ids_sha256":"6b99376f37916da3177e56d8dbe872f5b54a721dad5a56e3d0f7641537953001","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,21,28,34,41,48,55,61,68,75,82,88,95,102,109],"target_token_ids_sha256":"6caf9fd439e8d278a1020748e4978b74310b3608c9d071b94f12ce03d0b9be62"},"formatted_content_sha256":"3f1c5b9ff9268e2a55000da43f44ffd96c58159b9099ecb106bfc6d0e882d405","generator_receipt_sha256":null,"identity_record_sha256":"2aef9e476b0e90a8d4c54f89251fdd678232c30c0588a40c71c3ea135c3a140e","prompt_token_ids_sha256":"de6bf797fe9bca2c9736422f1027832ef62d4311eea81a92442bdf94a157a32d","ruler_category":null,"seed":null,"selection_rank":69,"selection_sha256":"2db747f786476cd53b2b76e66050c7c9524096bc2cc634dc8949c6700bbb91e9","sequence_length":110,"sequence_token_ids_sha256":"f66fae77a66de49a19604647a7a1dcef25341e73c3beeb2dfc54b05038100bfd","source_content_sha256":"9427a32773d2dda1a56f98cb0bdbee88ed5634a53a96ba4f2adf32764392e2d0","target_token_ids_sha256":"9d11355be764566fab4daa7e657723edeee80c1c0d05f8dcbfe888b93d368076","token_span":{"cache_exposed_start":110,"cache_exposed_stop":110,"prefill_start":0,"prefill_stop":73,"scored_start":73,"scored_stop":110},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"be38fbf54d72a7f68805fbe18ccbdf7981341fa3db7c90146ed5584b8a5632cd","anchor_positions":[25,52,79,106,132,159,186,213,239,266,293,320,346,373,400,427],"anchor_positions_sha256":"2c63cecba171fd759882d1a320b87c2dfaec15e4490a663f9e1099a229e8e934","canonical_id":"783","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[25,52,78,105,132,158,185,212,238,265,291,318,345,371,398,425],"fisher_boundary_sha256":"df1eb852686d1c5fa6f464d0608a8ec972c4dfdb569e90a9d61ac80e9dbbd261","horizon":1,"input_positions":[26,53,79,106,133,159,186,213,239,266,292,319,346,372,399,426],"input_token_ids_sha256":"b4cb4bc55e5f1df30d752a6409003f2a8cb96722509b10f768d46b67657faa6a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[27,54,80,107,134,160,187,214,240,267,293,320,347,373,400,427],"target_token_ids_sha256":"22fcb95368b58ec5a4c13f727526665b92519bc8752f8ab11b799ee2ec751029"},"formatted_content_sha256":"0d8b83c24ac54c8b3c1a41092dc210a7f174f870033838f69387ee0a959cb7be","generator_receipt_sha256":null,"identity_record_sha256":"fd3b5341382c42b888558f7ec9d99892ab4bc6c62b5fea552dfd262f705be6f8","prompt_token_ids_sha256":"3c17fc671c321556046900784e286f9b2efcfec63411dcd19bc16d80f81efb33","ruler_category":null,"seed":null,"selection_rank":70,"selection_sha256":"2deccaa412b35e79530ed8315fcf11416ca3ef9bf945badf8e69214d6c67f0e2","sequence_length":428,"sequence_token_ids_sha256":"923d6ae58844ab43d7c92cc96780db2f0a68359b436399f8f4b84e8bd73850df","source_content_sha256":"3bf5dd95dc2a714c277cb345634f0add55f38f0c23e30c02e2fac976cc5e3560","target_token_ids_sha256":"89d552cd5c74ddddddc518b6c4d8776915650ab6c296da75bf7d8dc8d5e8a3cc","token_span":{"cache_exposed_start":428,"cache_exposed_stop":428,"prefill_start":0,"prefill_stop":201,"scored_start":201,"scored_stop":428},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"d8cc6a91dfe0b8725032b9dbe17d00c2267d1f3903f0f46de298aef09a0eceb6","anchor_positions":[7,15,23,31,39,47,55,64,72,80,88,96,104,112,120,129],"anchor_positions_sha256":"c03e2e14b7ad2333a14fc199839382794d3c90c5842d21c3c2d26acb100d0b9a","canonical_id":"792","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,127],"fisher_boundary_sha256":"36cbcaa5f2a243260e4b704ad965c1eb8729226a615436df507566bab0a15d14","horizon":1,"input_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128],"input_token_ids_sha256":"43ea65b95cf29e7cb303d7b58a8466e49d86ff3b94f481b088aedd213565b50c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,33,41,49,57,65,73,81,89,97,105,113,121,129],"target_token_ids_sha256":"09ed1cfb8661ecaf52c064021fdaac9e5e45d85d244adc29abbadd32cbb23a1b"},"formatted_content_sha256":"84324342a9988ef788199714ffd6a7b7f6493516f982634aeb0fab6e891dca3e","generator_receipt_sha256":null,"identity_record_sha256":"9ea4728a95faf70fc41786042f177a0a108a52c817d7f798a50fa559e6bfd72f","prompt_token_ids_sha256":"e2b25c9110fe041500ee35bda9d717a5b8764c0af9684a331d986da4d503a12d","ruler_category":null,"seed":null,"selection_rank":71,"selection_sha256":"2e13ba7ae758055511fc93fd8b21f4c8279b266645abb65bad13f929bab6d4ec","sequence_length":130,"sequence_token_ids_sha256":"173382b09d1a8419bd02b4281c9d8ea5342d4cee26d9006629bdfe38b23fc3ea","source_content_sha256":"8b4ec72b56e7b421cfd5660acfeaf5323dcfcb622b00a3b4f8a55c2bb8a599ab","target_token_ids_sha256":"45b965687a80da78d6902f0642a6a41c15d5b6065995e352b178741376df5858","token_span":{"cache_exposed_start":130,"cache_exposed_stop":130,"prefill_start":0,"prefill_stop":117,"scored_start":117,"scored_stop":130},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b526b8b7bf92e5b467bf63c4c44c32a7d3baa99496c16d829dfc10c90ef8e179","anchor_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"anchor_positions_sha256":"debc2f860d51bf30c9ccfe7850d3428800e80a005f976ab2943a8642c33fb808","canonical_id":"848","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,30,38,46,54,62,70,78,86,94,102,110,118,126],"fisher_boundary_sha256":"eb5082f8394245be99a854bb9e95a0e8a24b830d5b4e12d9acf89075a71132d3","horizon":1,"input_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,127],"input_token_ids_sha256":"2488867fcffe8e28aca7f38c1051a25ba6e3576b5816a851d585a91e5d1fb9b2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128],"target_token_ids_sha256":"67aae898043f0810d7b6435c2dc11fdfee0f51cf372142272e7060f3d549bd8e"},"formatted_content_sha256":"030fb32fa54afa33ec26144f6963d266d6aa4b969906d8ea12cf708055b33d67","generator_receipt_sha256":null,"identity_record_sha256":"992345ddf332a9b3994c60bd506b547f20b7a90c821dab684435a5333bc84659","prompt_token_ids_sha256":"f4528ab0cef6a0aedb91c8fdfdd0fd4d1d19f26b5c42b52cbb5e9d55c87e496a","ruler_category":null,"seed":null,"selection_rank":72,"selection_sha256":"2e8927b105ba240d08737397b6e4ae7e731e45937512629b5d05c9e422358ce0","sequence_length":129,"sequence_token_ids_sha256":"bdb29154778fe78320ce09bc48fb722bce1a1cb08026d6843fa036de3c59c397","source_content_sha256":"2a1f6cc18c5e358f530bd92c65c306de3dce59f6ab5bf9e55ea9c27c9608156c","target_token_ids_sha256":"d4d73429c6a8b75c9e50462ee645fac5d41af8e10ac5bbeb75778385684a24e5","token_span":{"cache_exposed_start":129,"cache_exposed_stop":129,"prefill_start":0,"prefill_stop":96,"scored_start":96,"scored_stop":129},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"a8b715cff28765151d67536c865a8213c840d7a8b7b255ed28fa0f4f4d86e4c3","anchor_positions":[5,12,19,25,32,39,45,52,59,65,72,79,85,92,99,106],"anchor_positions_sha256":"1ec66db99ded67a6b74f7dd5259127f2ba61fd9a33265762c1ea04710558d7bc","canonical_id":"973","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,18,25,31,38,44,51,58,64,71,77,84,90,97,104],"fisher_boundary_sha256":"9a38f0cf45f0d18c982e6cd4e3041a5c05aaa808ae9c3cc11acd1e5bd81ba1ef","horizon":1,"input_positions":[6,13,19,26,32,39,45,52,59,65,72,78,85,91,98,105],"input_token_ids_sha256":"beda690cd5123bf4bea71d0d29b89a37efdf1c91f271681dad45a48428687cfd","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,20,27,33,40,46,53,60,66,73,79,86,92,99,106],"target_token_ids_sha256":"b2959f450541356aa81bb0e5965581933b267e9e8e2f7bff19f500f4677cd5d4"},"formatted_content_sha256":"03a49d9fc1d2e40de0d006e558a4cc366fe5fc106b78e9dcce28bf991852272b","generator_receipt_sha256":null,"identity_record_sha256":"92e9205a716392965076d0c07b2927b8ceb264553c8ddf5e51d643b2c4447675","prompt_token_ids_sha256":"a60f5de6590c6b61e5ed13b415d33c77fd42dcd611c101939112501a21966e85","ruler_category":null,"seed":null,"selection_rank":73,"selection_sha256":"2f2ec7824d30ee016341d9347bf3033c9e8baefcf7f210237aece22c50a0c793","sequence_length":107,"sequence_token_ids_sha256":"02a78c628534fda7b94bf5fce8dc4c42f06805ffe169c5e88e972974217f23cd","source_content_sha256":"535cf2dcfdda3927e31a863cc6592099702642031c1efc3a80742b6f3ea4493c","target_token_ids_sha256":"8ad5d65ce29243858983f2e234e934222f9ebfb842ba646ed2857ea2ba6d2d5a","token_span":{"cache_exposed_start":107,"cache_exposed_stop":107,"prefill_start":0,"prefill_stop":81,"scored_start":81,"scored_stop":107},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9356ecdbe5e79b88db404f8fcc1cc4066488ab817a3a8d2c38916554dca11b22","anchor_positions":[13,27,41,55,69,83,97,111,125,139,153,167,181,195,209,224],"anchor_positions_sha256":"1b59984280d92d0f5b008fda81e8c1ee99edd8adebbc3555a45f9e3b6ccc7b43","canonical_id":"648","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[12,26,40,54,68,82,96,110,124,138,152,166,180,194,208,222],"fisher_boundary_sha256":"6a4524107689341a41d25534167f49a57a6232f42670a7ccb3c3c53953ca52d7","horizon":1,"input_positions":[13,27,41,55,69,83,97,111,125,139,153,167,181,195,209,223],"input_token_ids_sha256":"5d718cfe7aa9550f8596ed8028ffa484e5dde6a58cf75d5e564d46b03ceb80e2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[14,28,42,56,70,84,98,112,126,140,154,168,182,196,210,224],"target_token_ids_sha256":"29ce1ee9257823ee60fc561f31cf4387cbb83ef9304ba18e1878faf829fa4c12"},"formatted_content_sha256":"03ce2215aec8d5a70b9b7d5d9457a3049c63a14d3fb41d268cb7d640203b379e","generator_receipt_sha256":null,"identity_record_sha256":"47caea8e58a24ad7e9189aecc0d5cd34719c3296041ecb933c3966a471cb646f","prompt_token_ids_sha256":"c856d4839a1ad8ebc26aefea0a9fb07e5436dd63c6a305c7b4112bee6193dde4","ruler_category":null,"seed":null,"selection_rank":74,"selection_sha256":"307a74054631fd4611a4637c7dcc3b5902cc9b2530ad55e1817a7b66e0244a87","sequence_length":225,"sequence_token_ids_sha256":"996568b0b6b68ffab6ac7da97e43dcd7e5fdaabcdae5248fcf638e8c03d9454e","source_content_sha256":"c078071a1981a4c9334368583a6c9b84c5bca8b86cdedb85a12e8be1b1203123","target_token_ids_sha256":"c8d5a3a6d636c7fc1ee25da058f4b6f9c67be6202458fb01fa42861b8939a7a5","token_span":{"cache_exposed_start":225,"cache_exposed_stop":225,"prefill_start":0,"prefill_stop":172,"scored_start":172,"scored_stop":225},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"806ebe5a2818da11223ca67ee7cf2265f233845126be6d099c9dcb15c55fa08c","anchor_positions":[9,19,30,40,50,61,71,82,92,102,113,123,133,144,154,165],"anchor_positions_sha256":"f50a8793906ae0ec9418f440c4efb82f7805f24bf465b189ea08ad0bcb356f8f","canonical_id":"772","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,40,50,60,70,81,91,101,111,122,132,142,152,163],"fisher_boundary_sha256":"1b7250139134ecd49bde104b1a6a2510822a0f9dfb9d2d4e9e5eecf763a8b9a8","horizon":1,"input_positions":[10,20,30,41,51,61,71,82,92,102,112,123,133,143,153,164],"input_token_ids_sha256":"3e9f5ed8311d8d669c372b59caccce107e54dd1559931f673bbb25676156cadf","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,42,52,62,72,83,93,103,113,124,134,144,154,165],"target_token_ids_sha256":"43edf81c690bf9f561ba94748611186c5eed3e9186c085b2808196492de0f6e4"},"formatted_content_sha256":"b065a8530085bc045a0533e1eeaf8aaf8188f595824a24047f8870dde69f04dc","generator_receipt_sha256":null,"identity_record_sha256":"9f12e75bacb719e1dd7725bb446586e29d53441710a6bd0bee305318116aa5ba","prompt_token_ids_sha256":"deba03b89b8f2bb57ae57611e69505484898b45ff63c512c33597cb6fcdbb1b9","ruler_category":null,"seed":null,"selection_rank":75,"selection_sha256":"3116bf2b1674d3472c0f8a996afd05f7fc073f3720ffdb297b5d89cbc3eccae5","sequence_length":166,"sequence_token_ids_sha256":"beeb3758c81c70dda2bba27ac2041d61adaa245c5d8d0edf701cdf0f91f45c4b","source_content_sha256":"1ae2ef8f6c027aabd4091686f02cddd085364dc8ca4f35da3877f375ddbecf79","target_token_ids_sha256":"920ea1a628b222d262ce367d32d278f0b295a5028fbca3295770bcff04e208d9","token_span":{"cache_exposed_start":166,"cache_exposed_stop":166,"prefill_start":0,"prefill_stop":117,"scored_start":117,"scored_stop":166},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9b187e5fe961b945d44953c99d7e41a4cbc730ca0aee47b0408c755b6bd9ad40","anchor_positions":[6,14,22,30,38,45,53,61,69,77,84,92,100,108,116,124],"anchor_positions_sha256":"fe7a10db15891cb54d23372b10af49697c0401f54d8f6b37c01417545e9ba930","canonical_id":"764","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,29,37,45,52,60,68,75,83,91,98,106,114,122],"fisher_boundary_sha256":"e6be121ec0065a1a1aaee9d9e86ba55d2577c56a8617ecadebe8c4e3e6fa2ebd","horizon":1,"input_positions":[7,15,23,30,38,46,53,61,69,76,84,92,99,107,115,123],"input_token_ids_sha256":"8b87308fede122de1697b74353b230a58de366e630c17539cb050893d296a569","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,31,39,47,54,62,70,77,85,93,100,108,116,124],"target_token_ids_sha256":"a3e5f3dfab994ff9d370511f35cb4ce1a6409b31076b0c2fbfc672290cd7dbf4"},"formatted_content_sha256":"dd3955ad349be1f3bbe53adc3231a0d62ae00e248214be3a7b5e980d326fc9fd","generator_receipt_sha256":null,"identity_record_sha256":"700a6d8a14194a1306ce6fb464aea0085a25f117d07fcf265169ed7ae1eb2406","prompt_token_ids_sha256":"66cf8821e075a67969485c8f8e2ae3f57b8384381fd08c0db59c080fc7ed3267","ruler_category":null,"seed":null,"selection_rank":76,"selection_sha256":"315964c2edf3979db8272d920dfef4d6429237f6cc495a6ee3cd3d5549a6b648","sequence_length":125,"sequence_token_ids_sha256":"516180a745f3962c6178902c00099642c3d74217bedb6c055bb189918a4cff6a","source_content_sha256":"690c9b90840fcf0cd5119cfd171717a21c533ed2145ba5803ba93e78d2229bc2","target_token_ids_sha256":"17e59e0812b0551e0a212266dfde313e3b8b138e97f059cabb2ac99726994b05","token_span":{"cache_exposed_start":125,"cache_exposed_stop":125,"prefill_start":0,"prefill_stop":75,"scored_start":75,"scored_stop":125},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e95a3332e3f7eb017e36ba83694e18fd59804b725bb678bfc239e300e0f3db3f","anchor_positions":[9,19,29,39,49,60,70,80,90,100,111,121,131,141,151,162],"anchor_positions_sha256":"937bf130ee4ebb63677095c86a2168cbc15816e3cb8813d07c1ce8340b5fe4d7","canonical_id":"710","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,160],"fisher_boundary_sha256":"6be547ce84d00ff5e76edeb90cf3d18f3f42ed8471ed55e075a934fcf78abcd8","horizon":1,"input_positions":[10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,161],"input_token_ids_sha256":"110af952e2d5d1fb75f8a65fa643ba6711e84effc65a18d37b8308133a6bfb65","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,41,51,61,71,81,91,101,111,121,131,141,151,162],"target_token_ids_sha256":"b28186d7106e163a34e3f412a706d4588d43b0357f9c57bee118d5b51160b7ee"},"formatted_content_sha256":"88240f422a8a69eea08b888c6b9b3c0bb24a6241a7a906d6c5bd4e4f75a9e655","generator_receipt_sha256":null,"identity_record_sha256":"1781d9f90e6dd08f1aba55a3d324d8fa14693f707ca4f071b3b392360bbe0408","prompt_token_ids_sha256":"be734d8f240da897ca816e1ebc35469363413920aebb0b334817102e135917d7","ruler_category":null,"seed":null,"selection_rank":77,"selection_sha256":"3214d5450dc1d7505961e5b95aa8e71c3b102461efcea184dc5d581da13805e4","sequence_length":163,"sequence_token_ids_sha256":"8b2dd644e6f00860a2723ce8ab4d5c0a9ab6aa316b27d063fc4a39356a29e075","source_content_sha256":"15fd5fe74d1087e4941f9d296c749df402bbce707b6f506e7dfb634abdac93fb","target_token_ids_sha256":"5b619991d998bbfbccb9502f7a36880ca09dab9061d34c4dd351784267f88e5c","token_span":{"cache_exposed_start":163,"cache_exposed_stop":163,"prefill_start":0,"prefill_stop":130,"scored_start":130,"scored_stop":163},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"8623504af00a9b25be7bc8e91298e9bf3d79e889943b43f81d79a76ff7b2b08e","anchor_positions":[20,41,62,83,104,125,146,168,189,210,231,252,273,294,315,337],"anchor_positions_sha256":"d86064c4e7cc49d03ee7921a0260f3727647f42da5038fce7e70f58100a9b429","canonical_id":"615","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[20,41,62,83,104,125,146,167,188,209,230,251,272,293,314,335],"fisher_boundary_sha256":"f7c95c163c354425c2add0a88c1c4185cf084e48e914cfdd216090db1b963bf8","horizon":1,"input_positions":[21,42,63,84,105,126,147,168,189,210,231,252,273,294,315,336],"input_token_ids_sha256":"9379deab7e7b61df7302e3f0d09b7e567d9ba26caa1886692eba4506c94310dd","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[22,43,64,85,106,127,148,169,190,211,232,253,274,295,316,337],"target_token_ids_sha256":"80d5c49f9ff9f2ebebef47d67d9b86438b015a070f8c6e2dc84883e9240b9ab1"},"formatted_content_sha256":"b9645bc5ec977dfe41735d4ce5d1f4a734b1f86ea78c4a5646565ebd8c15566f","generator_receipt_sha256":null,"identity_record_sha256":"9b128354be7b44adac95032db55fe4b66adf4ee89cf51ce82d87f3159b662f3a","prompt_token_ids_sha256":"51676de173a112846b34d4fba7305f57b81fb3f9d18e638cc2fa74f8e3a3e293","ruler_category":null,"seed":null,"selection_rank":78,"selection_sha256":"3285b6f67988631b43e377c409844a0c984cfd47e40b19a98584df2d8f61b362","sequence_length":338,"sequence_token_ids_sha256":"6bebf94cb834d1fcfe47dc541cf15d6fc87989ddf47fecdd7be2a90d0663d24c","source_content_sha256":"fdb94f9efd3d2f52a851821fe1708964cb54b4e59675eb6fbb5b37edf3910703","target_token_ids_sha256":"9714d922e9b518657f61cc75eda60ded6a3f1c50ebe5b59d1d6800481296fdd7","token_span":{"cache_exposed_start":338,"cache_exposed_stop":338,"prefill_start":0,"prefill_stop":310,"scored_start":310,"scored_stop":338},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"96cee3de971a921588b61de2e2cee579332da466f101701e569463c8c6a8fe9f","anchor_positions":[9,20,30,41,51,62,72,83,93,104,114,125,135,146,156,167],"anchor_positions_sha256":"eaf3a0e814f88177633695c30e9a05b6b492aa64c90bf98da98cb9be3917c812","canonical_id":"929","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,30,40,50,61,71,82,92,102,113,123,133,144,154,165],"fisher_boundary_sha256":"a4409da62e8624d4a36ab120ae46d80d5347e0949ddb59fa84433d595ab39c86","horizon":1,"input_positions":[10,20,31,41,51,62,72,83,93,103,114,124,134,145,155,166],"input_token_ids_sha256":"ef3502c0e2121d90870bb103716d7830c424bfec40f477dca54ebfeac256169a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,32,42,52,63,73,84,94,104,115,125,135,146,156,167],"target_token_ids_sha256":"0afc3393efde3a0d92114c3ef929fe4bb20b15a93254a54991f7854ac5d8fb1c"},"formatted_content_sha256":"8720c8227a224ac2e19aea8120b6280befcba03b75052032de9bf8e95ca77b04","generator_receipt_sha256":null,"identity_record_sha256":"1386ffcc0e108e711ebccb2df6fc089ab088b8bb5bf75534b414141f41a74a75","prompt_token_ids_sha256":"d3e5efc050ed2ac9f9ae235e854ba1d481168daa7af573882848337902db8cbf","ruler_category":null,"seed":null,"selection_rank":79,"selection_sha256":"3296d51e868118ed66c05092fa54df28b0ca574c48a0cba377e10c955ebf1545","sequence_length":168,"sequence_token_ids_sha256":"e74df24c78d5f48aa647690ed6d124d47ca43f1d31e9c157bbbd81fae882c552","source_content_sha256":"bc0b73708502c4d4ef970ea6606ef292432812c718cd0e89507be65055cb1c76","target_token_ids_sha256":"e0d8a0f88aa5d40291247005bc203563ae5853bbdc6122d0bc21fb681b59aab3","token_span":{"cache_exposed_start":168,"cache_exposed_stop":168,"prefill_start":0,"prefill_stop":145,"scored_start":145,"scored_stop":168},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9209ac953f696c6753bd12b7e3aee86341dec2daa6e7890b20b1f6ca9c923fa2","anchor_positions":[11,24,37,50,63,75,88,101,114,127,139,152,165,178,191,204],"anchor_positions_sha256":"48082390de232d2e911509462169a4b3ebe8a03329ea579a66fe38623c52b3bd","canonical_id":"872","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,49,62,75,87,100,113,125,138,151,163,176,189,202],"fisher_boundary_sha256":"099898e807c53f06b4fefe6112588ccaf699892b69781fb1af96a8f953dceff3","horizon":1,"input_positions":[12,25,38,50,63,76,88,101,114,126,139,152,164,177,190,203],"input_token_ids_sha256":"e90e1cdb40861c5ce546f0f793eca7fb22ff9a2d1fcd28ac90c1c3c41b2a9a8a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,51,64,77,89,102,115,127,140,153,165,178,191,204],"target_token_ids_sha256":"dfe340d203294bbebece21c27b39e76b677b6c6f99f69c75f1c05d25a052a6b0"},"formatted_content_sha256":"ae7cccb0f44aa7dd8ae1d59a7bb423feb46060df6bccc538c0d729941cf62f09","generator_receipt_sha256":null,"identity_record_sha256":"33a68c486b6fb35b93a963a3c1ab2bc5ff188b6005bdce71a974210943d77edf","prompt_token_ids_sha256":"c587ed25a2f2bfc478fbc770f969490260f64e578d5a442189b63d9a9743cf1f","ruler_category":null,"seed":null,"selection_rank":80,"selection_sha256":"32eea1bbd2a2735f50e8ed7192ad280642b4f864877e2e12c74229735f3eb364","sequence_length":205,"sequence_token_ids_sha256":"5cd1de46f6a6a54a60046c16ee57b387b99886733c7d130b2e28f1ae13d4ae9a","source_content_sha256":"7a15fb9218af2f77deb400a4399d8ac77545eab7979f345c0ea992388693aa5f","target_token_ids_sha256":"27908577789a6b8b4d0224aecb42ddeb2609c5ec146dca642424eb2a076e6e7f","token_span":{"cache_exposed_start":205,"cache_exposed_stop":205,"prefill_start":0,"prefill_stop":183,"scored_start":183,"scored_stop":205},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9e18bfe18f9c4da802617b4dee6331466e0e733317487a86f7aef33753525ee9","anchor_positions":[7,15,24,32,40,49,57,66,74,82,91,99,107,116,124,133],"anchor_positions_sha256":"6c1029515cd9fd840ee9baeeb3bbf16de22d914596dc66d6c0b00a017e82aef5","canonical_id":"804","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,32,40,48,56,65,73,81,89,98,106,114,122,131],"fisher_boundary_sha256":"0654213b540ddd6054c3a7a54b4ed68fe2ce52d54601d589e8dd8432c5e03c9b","horizon":1,"input_positions":[8,16,24,33,41,49,57,66,74,82,90,99,107,115,123,132],"input_token_ids_sha256":"e6dbcef9e02b4ece7b578fe3e362efe755dbd02afcae77d6d6f4edcd4de4f4cc","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,34,42,50,58,67,75,83,91,100,108,116,124,133],"target_token_ids_sha256":"0b20024476a22a5a23bff385afa2fb325aef7aecc21280ce17cd65007c86f1e0"},"formatted_content_sha256":"58c227c113ff5dfa4a446ad44a803eb54659e0fbefb3f5f80dbef45d0994bdfd","generator_receipt_sha256":null,"identity_record_sha256":"01f7ec235052c97847150a68e3de4b9c2d022df8fafdb46a04092b4a7ee18c27","prompt_token_ids_sha256":"927da8d69d8b88559b5811091f4fd329f0e47607f28c399059242149e3c7336a","ruler_category":null,"seed":null,"selection_rank":81,"selection_sha256":"3379395523a918b0a3ecf21510bcc8704655d52083f91c7d3bd323ce8ea3e272","sequence_length":134,"sequence_token_ids_sha256":"bc6201b3951727d59c9b6c88db4d0722a0b56896e2812e646426b637a86b085c","source_content_sha256":"0b30c40b1ec355476196d7e4b579e4c837c30947a206b5da87928bc338b51aac","target_token_ids_sha256":"ce0bec16e6ba57588fa2e620c280ea4fdf1f2da0b33b6d8c1bf55efd9f79a60e","token_span":{"cache_exposed_start":134,"cache_exposed_stop":134,"prefill_start":0,"prefill_stop":93,"scored_start":93,"scored_stop":134},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3078f2a2b5f48560ec6b5e149abdc2ec94224715ce8b8b558250fe31cd4dbf52","anchor_positions":[6,14,21,29,36,44,51,59,67,74,82,89,97,104,112,120],"anchor_positions_sha256":"7f8463e6889d5cbbd872abcc6f675ca04c3c0d8d69d266000933478b69489b8c","canonical_id":"713","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,13,21,28,36,43,51,58,65,73,80,88,95,103,110,118],"fisher_boundary_sha256":"12bd3ed950f8f6e2d1ea75538e2472a2bb7fcef26eeb014dce2f68e39c0ef080","horizon":1,"input_positions":[7,14,22,29,37,44,52,59,66,74,81,89,96,104,111,119],"input_token_ids_sha256":"8bf93fb6683667ca762c7e68944d126e5f2c2b6c44cd212a8000cdc09acb27e1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,15,23,30,38,45,53,60,67,75,82,90,97,105,112,120],"target_token_ids_sha256":"bb0137c2b21eefda36fe32ac45aa551c534cfa2ebce4a392aed849f5496ed9bc"},"formatted_content_sha256":"681d439ccf9ce8b1da99801aad0074b3790087d6efe044ae79d7e49571d76ac7","generator_receipt_sha256":null,"identity_record_sha256":"bf5c49f9a99a125e0728cdbdb1103ef16d8f41d27323feb6d66db4c28cbdf8e3","prompt_token_ids_sha256":"689f474ef4fa8ccced0bd23ab2314dc97972bacba54c2a54dc4d440098402b38","ruler_category":null,"seed":null,"selection_rank":82,"selection_sha256":"36a09fe9c145944020ea5abafade01ab7f4a1c538f55c32b2daf644698464fd3","sequence_length":121,"sequence_token_ids_sha256":"df03ea9ab5af6fe2ad270a5fbae4db0bf61e07b1919c60f124a8070bb517a052","source_content_sha256":"9197df3cdc3a76aced1eefeae0cf812e2f43b1c25afc3bd73c6e27fa82f1aeee","target_token_ids_sha256":"6b06cbab3e1a4e869733d23ddc800cba139bbf3020eda6b3ab38843880a60fa3","token_span":{"cache_exposed_start":121,"cache_exposed_stop":121,"prefill_start":0,"prefill_stop":90,"scored_start":90,"scored_stop":121},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"63b9e82e307a72c058464d1b1d8a7f3c58d7fb5152977d0a46c053ff527a82dc","anchor_positions":[23,47,72,96,120,145,169,194,218,242,267,291,315,340,364,389],"anchor_positions_sha256":"8e2b268c6192e66f463da8f9c055e0c9184253267e4f791d54da3db3eed553a6","canonical_id":"709","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[23,47,71,96,120,144,168,193,217,241,265,290,314,338,362,387],"fisher_boundary_sha256":"967b90e282da8354335e13a711b1db1fc3dc1cd2d5f3733153901609f0b7284b","horizon":1,"input_positions":[24,48,72,97,121,145,169,194,218,242,266,291,315,339,363,388],"input_token_ids_sha256":"14ec48de9ee49ea137494da470356e32fd2626ad45fbf1122fb508edd8791526","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[25,49,73,98,122,146,170,195,219,243,267,292,316,340,364,389],"target_token_ids_sha256":"eee798bbc46c0f86ea4c32017a079b8d80703d9f4c5b22ed2e2e6a9edc5c6096"},"formatted_content_sha256":"377241d6ba849220fead3711b06a3790ec65647bdb509adac60123fd052d34e6","generator_receipt_sha256":null,"identity_record_sha256":"1d437dc4e8346fe37a0eee9d9a423bd60824e5834a88120acefb46d1be5ecaf8","prompt_token_ids_sha256":"fd90d23dea7f7557e683b5a171cd89091ccc9e500b8c27afb3c9221f5d9788c4","ruler_category":null,"seed":null,"selection_rank":83,"selection_sha256":"372cfe33288b920e01911b67999a2534620fe6fd180024291e1b86f7f221909d","sequence_length":390,"sequence_token_ids_sha256":"c1e45e96c9091049c070bbc66c111935f57c6b5469213a4d3ca91d3d27085868","source_content_sha256":"1bd1bf4c9a1573325504ef4695c6a08c9641c60e85f7cf68942681af59f0c13d","target_token_ids_sha256":"2c233f19acde716535f8aed74a2b4c7a6ff031e2df4d090d3f77cfd880649eac","token_span":{"cache_exposed_start":390,"cache_exposed_stop":390,"prefill_start":0,"prefill_stop":309,"scored_start":309,"scored_stop":390},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"755396a4616427092f5857ae9f027899125cc2f2dc290df40ac96b2609861e3d","anchor_positions":[13,28,43,57,72,87,101,116,131,145,160,175,189,204,219,234],"anchor_positions_sha256":"019a1d4f16a491584b0b921a03e82e05ab8f21be64e2ef381494510440a2dbba","canonical_id":"854","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[13,28,42,57,71,86,100,115,130,144,159,173,188,202,217,232],"fisher_boundary_sha256":"3978f984679fc70aed874ca972bca2f58c59e79999bd8b5dbaaf51586a103e4f","horizon":1,"input_positions":[14,29,43,58,72,87,101,116,131,145,160,174,189,203,218,233],"input_token_ids_sha256":"f50bcffcfe9946342cfe243e55c6a21e7d36fb6cc50a570f25fa3180e679507d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[15,30,44,59,73,88,102,117,132,146,161,175,190,204,219,234],"target_token_ids_sha256":"d2eb23c855ed7281d5d4f214ea2ec6aa0309bff553e222cc09435b17b3a89dc7"},"formatted_content_sha256":"ad6817ab94e23314f897ae56c55d36b1bfcbf69a7b8d35fb818c81c03d39b3c0","generator_receipt_sha256":null,"identity_record_sha256":"341e83cf4392f6688e220933fa083c53bd3f177ce67aba89005be759ab1ef581","prompt_token_ids_sha256":"c469e78789638c0e407409fe9aebd074eba88c7033481e56ef2dec96fd271eaa","ruler_category":null,"seed":null,"selection_rank":84,"selection_sha256":"374c60a9450ed67d1bb35eb2406c9e00aaf02021508333ed279cecdacc65f059","sequence_length":235,"sequence_token_ids_sha256":"3434905939b518880cc623791311c79432ac4833d7f9739c8a2fd83b9590d85f","source_content_sha256":"2f5b2b2d153d57ad5deccfd519cfcd97185b0cedd8a8e32fbcc6bb3429ff8755","target_token_ids_sha256":"8109cfc76d8f3759529fc35b4e397ce62b4a87dd10005164d11c77adb9a1a048","token_span":{"cache_exposed_start":235,"cache_exposed_stop":235,"prefill_start":0,"prefill_stop":209,"scored_start":209,"scored_stop":235},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3125b8323a121faaca1072275076c923a67557e8121f55b5373de0a2d7742432","anchor_positions":[21,43,66,88,111,133,156,178,200,223,245,268,290,313,335,358],"anchor_positions_sha256":"8cbf29aa79890d46e276ccafa41f46a9c68b76575359288a1ba31b9a687111bc","canonical_id":"698","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[21,43,65,88,110,132,155,177,199,222,244,266,289,311,333,356],"fisher_boundary_sha256":"c83dfd1738f12889d5c8d8fa14927532b8398e8d2ad099c912c8a526d1c14c23","horizon":1,"input_positions":[22,44,66,89,111,133,156,178,200,223,245,267,290,312,334,357],"input_token_ids_sha256":"b4fa8aaa429dfdeb479beb6db13cf536dad62e221419e87d8cd4cc6e49562547","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[23,45,67,90,112,134,157,179,201,224,246,268,291,313,335,358],"target_token_ids_sha256":"0e14956d01089b21589756a95abe69218dcb1a764bacd1f5efde70f9667949f1"},"formatted_content_sha256":"829bb0068e2180b28903791519d5fd2ef9c4873981556f5c4fb97915943ceec4","generator_receipt_sha256":null,"identity_record_sha256":"92efe3c0141538bc1fe1ae06136691ba5bfdf72bae549e16e258bdad5bc9a4ca","prompt_token_ids_sha256":"e985782bb265579625ef2c2a88762e93185a635ff5c9104060000f4dffeff366","ruler_category":null,"seed":null,"selection_rank":85,"selection_sha256":"383631dc03a19eb73d617831ef3de08685e43b584ca707ed72bff063479a80f3","sequence_length":359,"sequence_token_ids_sha256":"4f8db022db57407107163fbccfb9fa14d20b6c9ebb4167358a92b76de52a8cf6","source_content_sha256":"040369c5072294811f7d2b172d9db244cffb410fb58a0ff0ecfd4ac76e9ecb95","target_token_ids_sha256":"df7cdff2b03251bf8483757a1fe829176f9debe6a93f970241f7d8a5e45d2923","token_span":{"cache_exposed_start":359,"cache_exposed_stop":359,"prefill_start":0,"prefill_stop":310,"scored_start":310,"scored_stop":359},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"27e366c7bd906286182fb5657b877a3d983450e230b7e29de32c33658d233999","anchor_positions":[7,16,25,34,43,52,61,70,79,88,97,106,115,124,133,142],"anchor_positions_sha256":"231f3d432eea6372c0ecf521691b115566b5045b38a4ebc776666c6712167443","canonical_id":"847","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,25,34,43,51,60,69,78,87,95,104,113,122,131,140],"fisher_boundary_sha256":"8329a9007c7d4b358ba485f2d16e86dbb70099e16057eba7d09ee5bccd1208a2","horizon":1,"input_positions":[8,17,26,35,44,52,61,70,79,88,96,105,114,123,132,141],"input_token_ids_sha256":"c5de0c4ff62e358587b0368de1f213abd9ae23772b8a469ddfe3d5703c9d1012","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,27,36,45,53,62,71,80,89,97,106,115,124,133,142],"target_token_ids_sha256":"98c5bf9c6ce5ebcc0886bb8c1a84fb66bffb85cf9331303d2fd8e801f62f2549"},"formatted_content_sha256":"072b0e2d3b510043865ddae77cb50d35becf17f9e8114b0d23ee65976876342f","generator_receipt_sha256":null,"identity_record_sha256":"45013aa8aca9961b02b0a4a7cb8344d4f74b6e33cd7e17bde68f4bcc9830f5e6","prompt_token_ids_sha256":"cbc2d13d695e49c699eabbb0c37174336cdd76b1c47e1f754455737973e861a6","ruler_category":null,"seed":null,"selection_rank":86,"selection_sha256":"3856a5bb20d2c0009500867054a1e894dca05be3fb5db8c3123f8dcc626b3326","sequence_length":143,"sequence_token_ids_sha256":"d3b45de44d96d01b589d50a86f386609c7543385f067532a7174592d3748d795","source_content_sha256":"06648ba18a8fa484b8f21f1e5f0581005f1c25c15cbc72488833b9a12475640e","target_token_ids_sha256":"0fec8926851dfdf2d26e890a92aff900619119eb83bb757d608676dac4ded5d9","token_span":{"cache_exposed_start":143,"cache_exposed_stop":143,"prefill_start":0,"prefill_stop":132,"scored_start":132,"scored_stop":143},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"a71bad1c8e08458e094f4e03eef72e9b22e929c4ebde165642691319a2f08b8b","anchor_positions":[11,23,36,48,60,73,85,98,110,122,135,147,159,172,184,197],"anchor_positions_sha256":"7e13e3a1913a1aa40d4138d69b9cd0e10357d1735e6198f7373eda076b3142df","canonical_id":"627","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,23,35,48,60,72,84,97,109,121,133,146,158,170,182,195],"fisher_boundary_sha256":"fa4d10a3491f2cbd7a778d5d9cca07773dcb82334929e2637a6782e847dbba69","horizon":1,"input_positions":[12,24,36,49,61,73,85,98,110,122,134,147,159,171,183,196],"input_token_ids_sha256":"44dc69fff0abd93231a72100f67a65e70f884191a50e851eec9626b5f6d3ecea","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,25,37,50,62,74,86,99,111,123,135,148,160,172,184,197],"target_token_ids_sha256":"737d45bbae07d3bc02b3df44539a526e40a1a65dfb9bbd68a8552b4e3b8a3396"},"formatted_content_sha256":"0a0b695997f5097537b328213722620e421fc13a6da1c75ddb049d61551d4ed8","generator_receipt_sha256":null,"identity_record_sha256":"a708abf4f23637f8abca094ec7198056051b374088f997be873b42a6e20807ab","prompt_token_ids_sha256":"f19ff189894482dc79997369f05a455e1a3ef3d68bf8187d519557d6d6c6ab5c","ruler_category":null,"seed":null,"selection_rank":87,"selection_sha256":"3878b1b7a110136224b32091f63168693baa7e86e3c51b5a5536e13dad73f9d5","sequence_length":198,"sequence_token_ids_sha256":"0f9fcb1d27f0fb0d37209deaa4fa835902344d189e823d011638a2011ede20bd","source_content_sha256":"12ac418a9b4a0d9ae374ce607cfe72460e1c30e426106ea273a46ac4ba95ccac","target_token_ids_sha256":"cf5bb6259eaf6bc72ef3d80bf3723b99a1c4a093c8a643a5b9a663a259d73de4","token_span":{"cache_exposed_start":198,"cache_exposed_stop":198,"prefill_start":0,"prefill_stop":111,"scored_start":111,"scored_stop":198},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9c95bca00064ff088717e40df85e1ee14f4f14ac42b57c865b2a27f9d95db37b","anchor_positions":[17,36,54,73,91,110,128,147,166,184,203,221,240,258,277,296],"anchor_positions_sha256":"92fd03c25ec622578164ea2f5bba6632c7a3ee726d6e1c9caa74a3fe54646d5d","canonical_id":"936","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,35,54,72,91,109,128,146,164,183,201,220,238,257,275,294],"fisher_boundary_sha256":"e2680422efa4a11ce200109baaec769ad06a6773088de15eee3df9953e7958f3","horizon":1,"input_positions":[18,36,55,73,92,110,129,147,165,184,202,221,239,258,276,295],"input_token_ids_sha256":"06925d4de4cf692d9d7ae2fb8defd1d78ee16de18017fea89fc095caf7b42551","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,37,56,74,93,111,130,148,166,185,203,222,240,259,277,296],"target_token_ids_sha256":"a529ab740b332e7c438cb708428ab5907c7a59afc2ac4ac51145f549dfdc5580"},"formatted_content_sha256":"afa72e90063431bd6756ace794a9050cc1610077201aea2fd9b4bda86c662ffb","generator_receipt_sha256":null,"identity_record_sha256":"92f30344725d406c50725f35a3f5b576e19bbd592073bffc0ff773be3cd5ef1c","prompt_token_ids_sha256":"6a725c770525d60d682f99ffbb259336c6f4946dc97f839cd4ca68ee7d0cf425","ruler_category":null,"seed":null,"selection_rank":88,"selection_sha256":"38fb5a0b645f175f62d3e012cf15aec73e7982d5713c59591a4c1d1a08561d18","sequence_length":297,"sequence_token_ids_sha256":"9f51cb1048ef1942bb0af6b3e3d4087006b31ee6c48ec3a0f524b1e08cc51f15","source_content_sha256":"1effae4b4c21d10e095a0a5f927b7cffd241b64be8df419dc05137e160edabc0","target_token_ids_sha256":"6a6840a3339ec51d008e341f8b430ed9e10a11c573b431942795dcfdb1e62c77","token_span":{"cache_exposed_start":297,"cache_exposed_stop":297,"prefill_start":0,"prefill_stop":255,"scored_start":255,"scored_stop":297},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3b13375846101ec060f2a2ecefd25b66c836c8f9c53225732689232378b5cc44","anchor_positions":[9,20,31,42,53,64,75,86,96,107,118,129,140,151,162,173],"anchor_positions_sha256":"5eb476494a95db7e5ff69d760daae6c1b59a438e7d071342cc2f8648e535c378","canonical_id":"827","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,31,42,52,63,74,85,95,106,117,128,138,149,160,171],"fisher_boundary_sha256":"d2950d5bb22979ced35eeb6c35b7e0f442e13245822cf644ec5b7300565507ee","horizon":1,"input_positions":[10,21,32,43,53,64,75,86,96,107,118,129,139,150,161,172],"input_token_ids_sha256":"995c75651110d6eeaf5f7790a1e5988aa866d3ec3f148b43040167cca7bd163f","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,33,44,54,65,76,87,97,108,119,130,140,151,162,173],"target_token_ids_sha256":"4d18d212a638d29f456dfa535bd673c510d66ec32e43594822d9bb9241ca20e6"},"formatted_content_sha256":"4790f2d60d6a241d89f28294cf7110652356521e1216cc3a9e2ce7e0b065bbe2","generator_receipt_sha256":null,"identity_record_sha256":"e93e8a672f73ac122851223c0114eb0974c76a960f1a880f4d4f64c9eb90f40c","prompt_token_ids_sha256":"9cc3fc3392444e349135b1065a5227e3391859f3554be674981d424cbdfa9fbe","ruler_category":null,"seed":null,"selection_rank":89,"selection_sha256":"39165caf08bb54ef81c9f6ea0a6ae1eac6a1c3f0b6f9748dcd2e2d953b5a44c6","sequence_length":174,"sequence_token_ids_sha256":"f5f5521cd3bd481ca316528bee26f7f9355182d73550ff14da89d91f6558600e","source_content_sha256":"c60d7f94f875619bd70591ab5f80b6e8fb8d336938e07fbeed1b32564c1a0b38","target_token_ids_sha256":"fac10609f7b6c41ca9b4bbbf7495988b94cc3b4df19e5dac66d84079bd3d4b1c","token_span":{"cache_exposed_start":174,"cache_exposed_stop":174,"prefill_start":0,"prefill_stop":148,"scored_start":148,"scored_stop":174},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"ad1b36a9a3ef4aa1d78cc9a2babd28dfdd79a75b4a7e69563454a05c10b6e2bf","anchor_positions":[7,16,24,33,42,50,59,68,76,85,93,102,111,119,128,137],"anchor_positions_sha256":"451653062c723e6fe2316470c66684b5f06f6d96d14e7b3bb2fb0c9e7d88845d","canonical_id":"930","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,24,33,41,50,58,67,75,84,92,101,109,118,126,135],"fisher_boundary_sha256":"a46d5960cd9fe3147cb09c59211821b555240915a05f4d76e9a91b70e253bf77","horizon":1,"input_positions":[8,17,25,34,42,51,59,68,76,85,93,102,110,119,127,136],"input_token_ids_sha256":"e59435b4be9b10629e72f1905eae42cfc9f3fd8b77cbff58f09164e96872cde6","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,26,35,43,52,60,69,77,86,94,103,111,120,128,137],"target_token_ids_sha256":"e841c04edad63a6a7f6c695115859daa7a4c3eeb9fe4f2f7094561aff448ee28"},"formatted_content_sha256":"dcb79d95145dcc945bd4f4ac20fe07c33528e7e8d8ef4cfea0312e5b02e817b5","generator_receipt_sha256":null,"identity_record_sha256":"5b49551d1068cd3633b13af41d2b4e4ab38c2383efee9d8c5370832e1a78e65e","prompt_token_ids_sha256":"2eeaa907d87be72e61b726a9ed4672463343033ccb6bf2376754766f298a566d","ruler_category":null,"seed":null,"selection_rank":90,"selection_sha256":"3934218a3cdeb9fe23fe0820bbaf2003e6426a07af797c8318d3df525a7b9d57","sequence_length":138,"sequence_token_ids_sha256":"5c11b18fe521da6db909428adfb48961064e310ff957a9d2a1ade8b72d391175","source_content_sha256":"435474a45f03a3373a0a7f5f78526f3621d01a4daf6907075965fc04440e323f","target_token_ids_sha256":"3fa7542edfba5ed95e02424207a7fc3d1bdc29ef1bb3fb055507740f8a0860d6","token_span":{"cache_exposed_start":138,"cache_exposed_stop":138,"prefill_start":0,"prefill_stop":90,"scored_start":90,"scored_stop":138},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"926ef081a870c670e181e6f21a6dbb6e67418bbe21df0f8841a9da25ad315d93","anchor_positions":[14,29,44,59,74,90,105,120,135,150,166,181,196,211,226,242],"anchor_positions_sha256":"3859b527bfcde30950aa1fd4919bae6af3d771f9dcd051365cd8ca16d468c677","canonical_id":"751","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[14,29,44,59,74,89,104,119,134,149,164,179,194,209,224,240],"fisher_boundary_sha256":"af0bf62935327f2467d479d54e944e5f444b6c444118efcf42d997ed4da7cc37","horizon":1,"input_positions":[15,30,45,60,75,90,105,120,135,150,165,180,195,210,225,241],"input_token_ids_sha256":"f52f3b35b365e905e8af0a0912e656ae9d1e3ad2906698dfa15eff817cdc47e1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[16,31,46,61,76,91,106,121,136,151,166,181,196,211,226,242],"target_token_ids_sha256":"c71c870c1fd6b3ebc3243a03cd21567b7db14cc45512773ebc666af0317504a3"},"formatted_content_sha256":"b9749edd133bc0cefb8007a7358f767d918467015bff6d845fa78b29bdd5b3c2","generator_receipt_sha256":null,"identity_record_sha256":"bb8bf4056e66c85336deaf5c6bf0614c6c6eea646b46a3dcc0b05e265ed2293b","prompt_token_ids_sha256":"4d217c659514cbec1da37d7ca1cf02b1de4d40d095c106f7fd89d4517b8e1800","ruler_category":null,"seed":null,"selection_rank":91,"selection_sha256":"3972ee7b79e05ce309d4c65459acc2d264f7821184dc4baf556f892f50507741","sequence_length":243,"sequence_token_ids_sha256":"22a137779b1349921b5784c3f3a7da9f3700715398d21e5c69536216ef7c5d82","source_content_sha256":"04de6155b488684214357c82595e2a1b0b4162588221bfc1f959c3ad02c64302","target_token_ids_sha256":"4a34e467583136f16ea6deeb64a30a0f625052cbead11e0c2460872fb5a60d9d","token_span":{"cache_exposed_start":243,"cache_exposed_stop":243,"prefill_start":0,"prefill_stop":129,"scored_start":129,"scored_stop":243},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"bc742a938d14512badf1f6f2dc626345160bae405ccb62cc78f8fcf2fbef198b","anchor_positions":[11,24,36,49,61,74,86,99,111,124,136,149,161,174,186,199],"anchor_positions_sha256":"cbd5b119373faf027bdb11d72340904f9e8568f13150612cfe8c0d28d26f0a32","canonical_id":"707","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,23,36,48,60,73,85,98,110,122,135,147,159,172,184,197],"fisher_boundary_sha256":"f539aaf9624257146284435f180cb1ef2b1c3d2448b93f936510956f316e60d0","horizon":1,"input_positions":[12,24,37,49,61,74,86,99,111,123,136,148,160,173,185,198],"input_token_ids_sha256":"53699344c41ad6728511aaab294416c9cb8ab020e2cce05973c6abf72fe3830a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,25,38,50,62,75,87,100,112,124,137,149,161,174,186,199],"target_token_ids_sha256":"c13ae7cca2c350903d6e54179df99c8f0ca7fdd40021679bdcd12e59ab7d3f00"},"formatted_content_sha256":"b7943712ec68b060c30f63f9c1e79415e5d98891bad66512dd84aae0ea33779a","generator_receipt_sha256":null,"identity_record_sha256":"7e8098446b3e22dff19a34b8e6cb18c072889a0914269f5e6c46eda3e278f916","prompt_token_ids_sha256":"aedaaa76edf70a4335d76e3753a2e74b6358828bd7fd27089a9349b82dbc4f04","ruler_category":null,"seed":null,"selection_rank":92,"selection_sha256":"39d4884c3293b83c7da73a4192c68ac6ed7dd2a2b8cb6535c8bc27dabefb2152","sequence_length":200,"sequence_token_ids_sha256":"f84c8958f4ae3235716092ff6d50ea9f9eeeed151677fd0291eefd3f4c046e46","source_content_sha256":"835bcc97198da2e9313e8115ac8c70845ef11e086ca2f36a6cbb1bc74afb15e6","target_token_ids_sha256":"63a1e5e3435f4582095fe5bf45dca9f77c0635e287377e887030050e9576dfbe","token_span":{"cache_exposed_start":200,"cache_exposed_stop":200,"prefill_start":0,"prefill_stop":82,"scored_start":82,"scored_stop":200},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b368a5e3bf6a4d6a9d65ed071a6ae293e6addf944c619d41b4db74fd479ea1c6","anchor_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,208],"anchor_positions_sha256":"e3c9aaa391385675ab3475417e405da92d531052d937e046ddc609185dc2c7d4","canonical_id":"946","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,50,63,76,89,102,115,128,141,154,167,180,193,206],"fisher_boundary_sha256":"6717c7a65af61cfd12e44550f289c401161af90d68d57dd17e3c519d22a1613c","horizon":1,"input_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,207],"input_token_ids_sha256":"35c588f0b227da0babb6432ce2d27c1f6f0ddf842b8da0c7b8c5def4f5567635","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,52,65,78,91,104,117,130,143,156,169,182,195,208],"target_token_ids_sha256":"3cc14e363f767b18bf8fb37c1c7a1681237d5f34c0d8244fad4f8ea124be7de9"},"formatted_content_sha256":"6672cb487be13a90610c98af3babace38c6413c484e2a543e785501685eea0d6","generator_receipt_sha256":null,"identity_record_sha256":"f4090de3fc3464141d40f4b023f18db5c2e540193c894d2b56d314001a2cc540","prompt_token_ids_sha256":"23c086ef1490fd86a6b3f23899489b782e5aef4d38689af88dbd19f7fb7b1d0a","ruler_category":null,"seed":null,"selection_rank":93,"selection_sha256":"3a31bfe17e95e3f782d9b40a9bad5670a4dbab98abaa87ce2414402b4d4c9d3e","sequence_length":209,"sequence_token_ids_sha256":"8002749920d9aa312887fb27e1c8d185977ea0ce5f145e2a0dc684e46b759834","source_content_sha256":"bd64b5ce22599f2fceb61fda2a89a9caed49e3e0b63694d888f83b548c800738","target_token_ids_sha256":"d1d65dd1bea87ea89e2cb46bc0c015ee3a4e8df03fe4a02a29bb30c5549b96d0","token_span":{"cache_exposed_start":209,"cache_exposed_stop":209,"prefill_start":0,"prefill_stop":178,"scored_start":178,"scored_stop":209},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"122dd2605483f8c364752af43aece851c5117635a2c390bbecc2d2c38ee203cc","anchor_positions":[12,26,40,53,67,81,94,108,122,135,149,163,176,190,204,218],"anchor_positions_sha256":"34e0f2438d49fed8c871a33d3818a40fc035b239ea8639ec66d64d1e59a9bb66","canonical_id":"853","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[12,26,39,53,66,80,93,107,121,134,148,161,175,188,202,216],"fisher_boundary_sha256":"41be834ada2a5acf9617b5b43955804d2e6a4ebe3234bb01e8e5dad38bb0a1a3","horizon":1,"input_positions":[13,27,40,54,67,81,94,108,122,135,149,162,176,189,203,217],"input_token_ids_sha256":"afd2edb984710f6fc20bd321b1036090c05a37d38b4b24a9acf50e7bd4160e2a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[14,28,41,55,68,82,95,109,123,136,150,163,177,190,204,218],"target_token_ids_sha256":"b5093288c3e5810d5c3d19010a1acdd8082bcba55f9489bdabb69e3196c53db6"},"formatted_content_sha256":"abf4e533a3c9388da04e8ab3ba22c031aaaeb56e90338a4793f35ac968218c95","generator_receipt_sha256":null,"identity_record_sha256":"e80d081ddd92bc2a2b3317fc55c504dc53c4973ed5fdd77bb04345fc653a731e","prompt_token_ids_sha256":"54d2e4ab6ff2869f39d935df89b9c87e44df50e0fe4ecf67a377211040ceb3dd","ruler_category":null,"seed":null,"selection_rank":94,"selection_sha256":"3a95db7ccef488fa57c922c5fa561a269f79fc7c743e4d7a0ba136629ed14154","sequence_length":219,"sequence_token_ids_sha256":"7c11f008ccc4d7cd53ee721463f7a84f45653c8215e85fce7deab133839b9b0e","source_content_sha256":"6e23a95d9b3ebcb264ca654a288e4f63ec19e20f70c8c2cde95ec7ec6f5b65bb","target_token_ids_sha256":"5da4da15ffc85692e7f49aaf3e35fa21d41d697277e633358e0285f47b164471","token_span":{"cache_exposed_start":219,"cache_exposed_stop":219,"prefill_start":0,"prefill_stop":83,"scored_start":83,"scored_stop":219},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4475867faff977bd5c0d96f97b251d767fa8320ea73f71a38a236d7f33bd9db2","anchor_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"anchor_positions_sha256":"debc2f860d51bf30c9ccfe7850d3428800e80a005f976ab2943a8642c33fb808","canonical_id":"824","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,30,38,46,54,62,70,78,86,94,102,110,118,126],"fisher_boundary_sha256":"652add26cddc5b2b11115550fc5858cab1d3a2bf00d5a3de3dd88ed6f9ee6844","horizon":1,"input_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,127],"input_token_ids_sha256":"f5e28e53573eb8607c244c66e29a9057150911f5b5348a120bb3a756c686e14e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128],"target_token_ids_sha256":"f2aa9adbe82c7f97119740cd60b33102619d0355f0db44358231d88ca624f304"},"formatted_content_sha256":"37af9d0afbb100c1787445b2ccb87b0a15ea15c7b2fac0560e9fc296bf6fe512","generator_receipt_sha256":null,"identity_record_sha256":"3933b2a829550c994be7ec3c13432e7fcc7eaa7429de8d86f09f2fdd0d54c382","prompt_token_ids_sha256":"e63b26e32e2d08ba0e1878bb31f438a8b4f0754fac3b288bf8ff4bc11ebc726b","ruler_category":null,"seed":null,"selection_rank":95,"selection_sha256":"3ae7adc084daf636812058b087ea0414390f1d765f67235a978166a970420f8e","sequence_length":129,"sequence_token_ids_sha256":"325eb23014b521a2ed7488ed8be2e2022f039e91aa7939745a5ea7decd8357c3","source_content_sha256":"f3f9375971e245fb7d4b9ecb4368b1469f96e8826d3d290349822de215c9524f","target_token_ids_sha256":"98ee353303b490751f3b0bf4a9db8ee9f144fd843e84aee5b61e6b8b237bdca6","token_span":{"cache_exposed_start":129,"cache_exposed_stop":129,"prefill_start":0,"prefill_stop":96,"scored_start":96,"scored_stop":129},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"95c07a4ae2c995c71644eb87f395b0201ab61998c92c88acd0deec2bdf52eb7f","anchor_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,208],"anchor_positions_sha256":"e3c9aaa391385675ab3475417e405da92d531052d937e046ddc609185dc2c7d4","canonical_id":"755","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[11,24,37,50,63,76,89,102,115,128,141,154,167,180,193,206],"fisher_boundary_sha256":"630fdfb50def6a0e9810e25416f083ca388d391614da06d1ea421eee032302e5","horizon":1,"input_positions":[12,25,38,51,64,77,90,103,116,129,142,155,168,181,194,207],"input_token_ids_sha256":"211b86c0afe55079edbfe3277616d686ed0a24152163e4ad422e5c2071214472","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[13,26,39,52,65,78,91,104,117,130,143,156,169,182,195,208],"target_token_ids_sha256":"279608d1d01474978b20236572dd00bca3894556f73197ecfd6a080fdab2cea0"},"formatted_content_sha256":"d7d00e78af67fa7b8d4f1123bc771c6c1838a42ca7940f09be045fa8e0258d3f","generator_receipt_sha256":null,"identity_record_sha256":"163bb04292002d23bbb74050efcec0fe6cb3a616a2ed635b0e83b30ad3599dbc","prompt_token_ids_sha256":"649e3b31340e844cf9aca294f09bbfb6ea6f59b8fa4339754a8040e2939a14ab","ruler_category":null,"seed":null,"selection_rank":96,"selection_sha256":"3b77b38c7af3c05a391dfb6e4427c1d6c8e1408b314a52ec893944d4dba7a59d","sequence_length":209,"sequence_token_ids_sha256":"5eadaf66f4c2a5b1419e30b0e9445bdcf5e76923985d3015fb3da1ae81a6112f","source_content_sha256":"f132344866b31da506e676ecd7f787c4f0b9c30ce84dbcf234d1bec372a4a3bf","target_token_ids_sha256":"b0d0429e68642809416cf5dc819cdacda9612ad5ec72ede704a6165361970f31","token_span":{"cache_exposed_start":209,"cache_exposed_stop":209,"prefill_start":0,"prefill_stop":107,"scored_start":107,"scored_stop":209},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e588a1ee6a1182055ad85a99122b37dca3e494260dd21a56a3847938dd6650d8","anchor_positions":[15,31,47,63,79,95,111,128,144,160,176,192,208,224,240,257],"anchor_positions_sha256":"58b00b7c1a9bcbca3b1a752019d06b8217b207235aef4a33f8862dbb76f94f4d","canonical_id":"705","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[15,31,47,63,79,95,111,127,143,159,175,191,207,223,239,255],"fisher_boundary_sha256":"7ddf7690ae9f4e218fef449a93f7970eefb48a98743e2cdab0de23b0a0dec118","horizon":1,"input_positions":[16,32,48,64,80,96,112,128,144,160,176,192,208,224,240,256],"input_token_ids_sha256":"fce5f6a7ef57e2e29ac18c5c07d311d066b45dcd8cc0d5feede90040805886a4","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[17,33,49,65,81,97,113,129,145,161,177,193,209,225,241,257],"target_token_ids_sha256":"7e886f55d5a1f767632ff6ce8330179e78261592f007a8d785bcacf263c6b5b2"},"formatted_content_sha256":"79b5d499f16fb4783be74bd200b7cdbb4738662932b42d91a8cb390646bd7282","generator_receipt_sha256":null,"identity_record_sha256":"fabe611b8a7c9e97b13d178a8020c782eeb92caeab58708b2acab9e36c7c9739","prompt_token_ids_sha256":"0906a739ae0536ce684360464813702b15ce64ce3043e3fad9895d8c7a910e81","ruler_category":null,"seed":null,"selection_rank":97,"selection_sha256":"3b7d0fef73308384ee4b6d98b48fefed969da32df7231a61067116bb7b9b77e5","sequence_length":258,"sequence_token_ids_sha256":"c89f8002d735b00177a7842bc2ce8332555e0cb94e7fc1f651c7d9306085ae48","source_content_sha256":"249e819abcbbc94c533d4966c357933213dbe7b58282534d182cfdd719ab0b02","target_token_ids_sha256":"f81a0d07e6c10d8a36f2da0ff7490f0c0aadc3fe1411b385e79f96372bff8f0b","token_span":{"cache_exposed_start":258,"cache_exposed_stop":258,"prefill_start":0,"prefill_stop":231,"scored_start":231,"scored_stop":258},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"21ba94b504ab5ded4ffa601325cf7df425e7f81789df808b86d2e08ba2117dcb","anchor_positions":[9,19,29,40,50,60,71,81,91,102,112,122,133,143,153,164],"anchor_positions_sha256":"e70afdac88bd21ff5ffd057462f0868a1aaa408a52296b58ecd4900f7f13759c","canonical_id":"614","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,29,39,49,60,70,80,90,100,111,121,131,141,151,162],"fisher_boundary_sha256":"b4910baa9b90efc3b7ffe399e1e19b050f46dd444b4bfa0faaadd835b9a3173c","horizon":1,"input_positions":[10,20,30,40,50,61,71,81,91,101,112,122,132,142,152,163],"input_token_ids_sha256":"09f0051a5092a284f02162790ff187253c2a8be10b80c2ff7b64476a0924f7b8","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,31,41,51,62,72,82,92,102,113,123,133,143,153,164],"target_token_ids_sha256":"9c3d0858143570f2b17fd9c211bbddf1beedb863bd39887f1982c64284451830"},"formatted_content_sha256":"b0e77c735851949333aa5a07a80654370a25c6f4c29b682d0a7296c28db21615","generator_receipt_sha256":null,"identity_record_sha256":"9e60edb916d1cba4b8692f32df62fc1c57894b2aa815769ff9a17da063de91f8","prompt_token_ids_sha256":"591d12041b2684415e1d05a0ea815d006597cc26105e19c6a0769dc0ea26a8bc","ruler_category":null,"seed":null,"selection_rank":98,"selection_sha256":"3c33175d44e2cc571159f6914ce6a7c57a1949ccdaa58585114ece582b46e4a7","sequence_length":165,"sequence_token_ids_sha256":"964232f668fe81da4ef5e6945e2164c11b771718b74fc029dbc71ae25a213e61","source_content_sha256":"10dc2964ee279825eee885f98c2883dcf718ce9958fca29aabfd2057996b5a19","target_token_ids_sha256":"fc3e58f7036836c2a7f90f009d9d954dbc547bc6a412334c087a495ec6166136","token_span":{"cache_exposed_start":165,"cache_exposed_stop":165,"prefill_start":0,"prefill_stop":140,"scored_start":140,"scored_stop":165},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"41e7124eadf2c8d65f12d91df35d39b81b0af62f8f0339d10d48d23001bed1f7","anchor_positions":[6,13,20,27,34,41,48,55,62,69,76,83,90,97,104,112],"anchor_positions_sha256":"311e7702507c904e70997e0b842193b8c519ddd45b6861c3e52fdd12fcbe662f","canonical_id":"835","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,19,26,33,40,47,54,61,68,75,82,89,96,103,110],"fisher_boundary_sha256":"ef27b3360779ff829870ac1205d40a423bbd1857f9d29564cf1a4bf273508b37","horizon":1,"input_positions":[6,13,20,27,34,41,48,55,62,69,76,83,90,97,104,111],"input_token_ids_sha256":"622f2d82715555a3a862da50633d47a53da0f6d05afb23f037425ec2460ebf97","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,21,28,35,42,49,56,63,70,77,84,91,98,105,112],"target_token_ids_sha256":"e4f44986dfe9f6afb783e79bc0788f586424b03e794cc20ab1129e7a75ca4b91"},"formatted_content_sha256":"16149587d9ee5b76bb9036394de7cc48051ba6deeced83ecf9552046bdf3d6fd","generator_receipt_sha256":null,"identity_record_sha256":"bcd976206ac1782d1fd75247a66068233d32764eef93e916f4303ac7216894e6","prompt_token_ids_sha256":"defa46cd3ab312e32247db6a9f87d5520f8ebbaa993a9af787a0d44ecc31a91b","ruler_category":null,"seed":null,"selection_rank":99,"selection_sha256":"3e545f645fe778609ea47f9d9d70db3a2d6fbf7c1b704f5c4b892a732ca4870c","sequence_length":113,"sequence_token_ids_sha256":"d87292b8689b41a713b9f23d42d1936617675fed9a389358046c37dee67270a9","source_content_sha256":"a2d12e2a125b7130ed036625cab03ca87319d5c2bc2f78d56e5b06e5d48967bb","target_token_ids_sha256":"d18a4e331e8bc3c665aeec8b4e8173eeaf216f691cec5deedcce9b89d1eabcf1","token_span":{"cache_exposed_start":113,"cache_exposed_stop":113,"prefill_start":0,"prefill_stop":85,"scored_start":85,"scored_stop":113},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4428369199e6df8c36558c477449442560d7dbefe1b1a34029eb0d3a540e8850","anchor_positions":[8,18,27,37,46,56,65,75,84,94,103,113,122,132,141,151],"anchor_positions_sha256":"99b598ab1b26514b0793ce6e9d9b8fd5fa21cfd91c15cf4ba944b02280eccc00","canonical_id":"785","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,27,36,45,55,64,74,83,92,102,111,120,130,139,149],"fisher_boundary_sha256":"224129c3bde0a7dc09aba43d89bfcf9df160c62a8d883b2421405de57da83148","horizon":1,"input_positions":[9,18,28,37,46,56,65,75,84,93,103,112,121,131,140,150],"input_token_ids_sha256":"d1cac9f469f540d35dd51eee4d84dc6489bdfbcb831d778ca3b910593caad5ba","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,29,38,47,57,66,76,85,94,104,113,122,132,141,151],"target_token_ids_sha256":"293d996a68091856a066009f8299d24def6c9ecd1505302fd1289596b4ce5929"},"formatted_content_sha256":"55cc81d66bd2973ef60996d3a13b658ddb8e10ba036a3a91808a8c4ae225b19c","generator_receipt_sha256":null,"identity_record_sha256":"70a4011b9bb314496b859d408f1205aaa618d08b8ca5504e4b0b7a8b3a794737","prompt_token_ids_sha256":"b5c378f6ed8ac42eb8be052f6d123a663ddfcd36c4bbcfc5decae012cf1c1926","ruler_category":null,"seed":null,"selection_rank":100,"selection_sha256":"3e7cbaec0d7be5d2de99697b23da0c61163c5bc4acecf283c4d9167e7b780eae","sequence_length":152,"sequence_token_ids_sha256":"ea3cdc2b1f803377a8cdf0588f013cebc2f019e5ad60fdf01294bf2d04d429ca","source_content_sha256":"06e8fb9b807385f5950f7335cb79a3b8105f621f1f5ac389155c9e737ff13d4a","target_token_ids_sha256":"a21eb422039b5c07e6a0e5f71858bbc8eaa8c315d06b39437977061a7cfdd253","token_span":{"cache_exposed_start":152,"cache_exposed_stop":152,"prefill_start":0,"prefill_stop":109,"scored_start":109,"scored_stop":152},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1cad8beb17ffee15f4de17ab4607544233d3683ae14b0cba6d063ccc9f11ff77","anchor_positions":[20,42,64,86,108,130,152,174,195,217,239,261,283,305,327,349],"anchor_positions_sha256":"3e58027edb04b4c749c71fe1c1753ad8300a599facd28b3d327dffaa160dd92e","canonical_id":"662","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[20,42,64,86,107,129,151,173,194,216,238,260,281,303,325,347],"fisher_boundary_sha256":"a261daa2db8ba7f05e7ff89074fdc6a0533b335ee67208179b8229ca3e70980e","horizon":1,"input_positions":[21,43,65,87,108,130,152,174,195,217,239,261,282,304,326,348],"input_token_ids_sha256":"d7e08d7e4f81bf68a6cf07dae075daaf7f79121f48d39ad9dba80fb057031e6d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[22,44,66,88,109,131,153,175,196,218,240,262,283,305,327,349],"target_token_ids_sha256":"8501644795ff2f929a4924fb2a3a1644ea894aa81320b17182c421175240b4b9"},"formatted_content_sha256":"91184e28eff6c3890010723ccfc61b5b4736a602d677efdffe59a220b3bac9b5","generator_receipt_sha256":null,"identity_record_sha256":"45b6269ce5d7d2ccefccb9d3aff10c4cc786d4482f036ebc1e0837d56e6c1923","prompt_token_ids_sha256":"ee07e3808c4e1142a8b3f904c8582078b899979e541c1ebe2297a3670112077c","ruler_category":null,"seed":null,"selection_rank":101,"selection_sha256":"41900ba952796bb2c024b96db8f08d2e63a757587e61fd96909dc934d50da156","sequence_length":350,"sequence_token_ids_sha256":"6890b69767a6c590160f23954c0338831fa0e3912f06dd6514c5b1552317c006","source_content_sha256":"3cf31926336b0e44d3630a63a486831d0b79632309d78e0b65f28e1c6f37353c","target_token_ids_sha256":"38e5b68f608a60d1fc61bff0194cd835196d4f8e8acb129c9e0744850d8641bc","token_span":{"cache_exposed_start":350,"cache_exposed_stop":350,"prefill_start":0,"prefill_stop":319,"scored_start":319,"scored_stop":350},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7d64f9e8b46b0ba25c21f37dee8e110df8862303d52cb3c2697353097a97db76","anchor_positions":[7,15,24,32,40,49,57,66,74,82,91,99,107,116,124,133],"anchor_positions_sha256":"6c1029515cd9fd840ee9baeeb3bbf16de22d914596dc66d6c0b00a017e82aef5","canonical_id":"732","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,32,40,48,56,65,73,81,89,98,106,114,122,131],"fisher_boundary_sha256":"2ea117a68a1f560fe1206940c2b26cdf6971aeeecd14dde580d013efe50b498e","horizon":1,"input_positions":[8,16,24,33,41,49,57,66,74,82,90,99,107,115,123,132],"input_token_ids_sha256":"46da8cb7bee3ade0de904115c405d0a4e2c1e215d9814066bfc5c50885b092d7","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,34,42,50,58,67,75,83,91,100,108,116,124,133],"target_token_ids_sha256":"3dd8aae9f6aff8d199075c33306d8bb5bd3b5f73666f2d236bd066a6d55fcbe4"},"formatted_content_sha256":"cf5f521e053dcaa96c55370f299514c9de27ccc98b8d846b0a51c4d2450341e3","generator_receipt_sha256":null,"identity_record_sha256":"1ea4e6a64df86bf2e3771dfc41f1716146cfac365a658d3864c030013a4cd84b","prompt_token_ids_sha256":"d705f66b234374474a57207de30809b5832cf7196bccb9dd73addbb879937264","ruler_category":null,"seed":null,"selection_rank":102,"selection_sha256":"41c7c18c717892ef0cbe99444d2f1d3d16142bd3128dc408df64987e27a8531f","sequence_length":134,"sequence_token_ids_sha256":"c98483cae5d714b2a8de61cd02257cc0ab084c30a0673971a4460e6d07f59d2c","source_content_sha256":"e5aeda9a803b146c76d557a6c8dc84720e938726b6f35982a5e61465a9a9d11c","target_token_ids_sha256":"15d06316804c670aad3848c3d01e3abb336656533c4feb76f93e90a0399aa760","token_span":{"cache_exposed_start":134,"cache_exposed_stop":134,"prefill_start":0,"prefill_stop":111,"scored_start":111,"scored_stop":134},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"a1bc1d7610d52bcb926390555c1cab77e3b88be528f3e256ea9e9a8106902d2a","anchor_positions":[7,16,24,33,41,50,58,67,75,84,92,101,109,118,126,135],"anchor_positions_sha256":"9058652c46869f6bf5cde3a730d23fba0f016461c119f6bbc8af3fb1c3031545","canonical_id":"679","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,24,32,40,49,57,66,74,82,91,99,107,116,124,133],"fisher_boundary_sha256":"810b5e7562357db03cc4e0193687f43bf7f4a08839cf3425a51f4e629ca7f5f6","horizon":1,"input_positions":[8,16,25,33,41,50,58,67,75,83,92,100,108,117,125,134],"input_token_ids_sha256":"03872be7640939a2770876b1cc144c05f59d3e06a6418658db2bceeecdbb5f11","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,26,34,42,51,59,68,76,84,93,101,109,118,126,135],"target_token_ids_sha256":"5c71155464943a602fc92f92c7507287819905cfa27f665ae40e126b898cdc40"},"formatted_content_sha256":"ffbf833b467a2b56b26bc50bcb320d893f41f60eed6341ca4c61813965020c7c","generator_receipt_sha256":null,"identity_record_sha256":"13c42ddbb25f335f5eaa4ef5df7c2cecd1ea19af1f8972ade37312c4fce78802","prompt_token_ids_sha256":"df73b549576d1784c30bb45b9c873ebb80267ac6c56afb4335eadca3c7c56849","ruler_category":null,"seed":null,"selection_rank":103,"selection_sha256":"41da3df61a258a1897eeac98083550d328825fdeec35836584dc0b7a76b060b4","sequence_length":136,"sequence_token_ids_sha256":"fb871500f94a5409b4acab60ab3e2aa6955924b9ceb87214232c4099d302dab0","source_content_sha256":"39571a4165c49d0610e00d19d3f5b5e1879232f2d0429100e2aaf915eaca9ab9","target_token_ids_sha256":"d6eaabcd23781ac4a910aadc33823fd08ab066d01331a124767f6e84f181ffba","token_span":{"cache_exposed_start":136,"cache_exposed_stop":136,"prefill_start":0,"prefill_stop":118,"scored_start":118,"scored_stop":136},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b8947897cdec3fc049ba3461c75adcd685b2264e6810778decb68c1db034f7b8","anchor_positions":[17,35,53,72,90,108,127,145,163,182,200,218,237,255,273,292],"anchor_positions_sha256":"c72256a78f816fa7bf8e540cc3aaf3fc1795369a3859efe48a9c4a8cce0d2489","canonical_id":"895","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,35,53,71,89,108,126,144,162,180,199,217,235,253,271,290],"fisher_boundary_sha256":"d12370caffbc875b01d6a59c8cf4c99d8fb4c93c519041b5500142fdac3a8340","horizon":1,"input_positions":[18,36,54,72,90,109,127,145,163,181,200,218,236,254,272,291],"input_token_ids_sha256":"a316e2b2888cea8097ab8420cf22051694e1d976c3b5b32f83f5207f4007d89f","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,37,55,73,91,110,128,146,164,182,201,219,237,255,273,292],"target_token_ids_sha256":"ea285c83c7e7736e7a436adedf9c66203844bbe2521ff7ad148dd51b419e98f5"},"formatted_content_sha256":"9b9360fb9e789a22496cf1d9c91e89cf7895942447ca7e16f15acbf2fe6515b8","generator_receipt_sha256":null,"identity_record_sha256":"080db8e8894bfd652b320dbf19c1d4e53c8b6ed2626bcb8f8d3cdf87268005f8","prompt_token_ids_sha256":"3ed029adce95edd3406c2359cabf621905d3ecaceb70e519fb1cc89a66899110","ruler_category":null,"seed":null,"selection_rank":104,"selection_sha256":"43113137520f4bdb68e26d41a096fa093a2f995386d5473618f0bbe4b2a0f73e","sequence_length":293,"sequence_token_ids_sha256":"24d56709934330dbe30c61a49434a07def76fdd2b31e47f84f71b050fc7aff39","source_content_sha256":"d1edefa8e37752a9da687c2a2e51086bdfe6401714de0bd593a892ecd4fe7bcb","target_token_ids_sha256":"e894e28b0e66238fd8988d8b0b978628f3354d87898164223cb7c7dc2ce5a596","token_span":{"cache_exposed_start":293,"cache_exposed_stop":293,"prefill_start":0,"prefill_stop":161,"scored_start":161,"scored_stop":293},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7d9d79586d3fd7a1376390bd1f085ee9ec35cb401fa471d99ddf45417339ce49","anchor_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,160],"anchor_positions_sha256":"cea919870992245499fb8d8a858d848e1e76f02a4e2230ada935442533c03198","canonical_id":"727","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,18,28,38,48,58,68,78,88,98,108,118,128,138,148,158],"fisher_boundary_sha256":"768624e8d0cb29118f002fce04ec89945eebe14e7587f370283fa04b2d90cf95","horizon":1,"input_positions":[9,19,29,39,49,59,69,79,89,99,109,119,129,139,149,159],"input_token_ids_sha256":"93e60eaabf27cffd7d1fee3301e42b3f62e183c9e40317db494ede3450b1ea7c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,20,30,40,50,60,70,80,90,100,110,120,130,140,150,160],"target_token_ids_sha256":"507cebbcf35bc57eb21aeff74f69ab933d22c31de894e77b573193730da7662a"},"formatted_content_sha256":"934eb88ee283eea8ac09e46d9333c08ae4e41a6910aa75e0638bc73403184864","generator_receipt_sha256":null,"identity_record_sha256":"aac6297b0c851ed51271119b880a93cb272776311e2b9e97f498c76785255100","prompt_token_ids_sha256":"ad50cbf08781e9a1429766f1a21e3ed4a76d9fc30e43b5b8d968dcda018da4d4","ruler_category":null,"seed":null,"selection_rank":105,"selection_sha256":"4338156536b7925e6044f19142d418110ae27757c47306d15815155b2d682917","sequence_length":161,"sequence_token_ids_sha256":"422567bd8e19fbdf69ab47274fe1bda49c8ac4eae311d7232ee13e0888b0fa23","source_content_sha256":"400a362fb6664f9084e588aa605b6e7ad1505849ab7c769574e0996829f6827f","target_token_ids_sha256":"c8e9d40bce2584a16cdb8ee232a9e1f86dd9c58c6fee7999125c37073c1f5bb5","token_span":{"cache_exposed_start":161,"cache_exposed_stop":161,"prefill_start":0,"prefill_stop":134,"scored_start":134,"scored_stop":161},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b50fd03bddce1923b315763ae4ef228d2b6c9cce2b80954554946c98c92a3760","anchor_positions":[8,17,27,36,45,55,64,74,83,92,102,111,120,130,139,149],"anchor_positions_sha256":"27ff593f69b04d82cc98f1ee839db50560b1d26eb9ecd645cc99a4e74dade2d0","canonical_id":"629","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,26,36,45,54,63,73,82,91,100,110,119,128,137,147],"fisher_boundary_sha256":"d8f311b4a33da5b28927ed11e2b7efbb2a1812456e5cd54c884287cfc1ce634d","horizon":1,"input_positions":[9,18,27,37,46,55,64,74,83,92,101,111,120,129,138,148],"input_token_ids_sha256":"f56a22b67052689f706d00c90e6fc5541583c78a589b0b3a26c6b93eda4162d8","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,28,38,47,56,65,75,84,93,102,112,121,130,139,149],"target_token_ids_sha256":"93c8a6c8bdc5c1ed56f7db15ae4424e87803630ba80e3845d421f97169235f34"},"formatted_content_sha256":"9c29f28fcb8fb63c6faa67311c4d67889b21525c18b625ca3fbd69bb6611342e","generator_receipt_sha256":null,"identity_record_sha256":"dd6a317d0ffc82ff3c3b71c470d6f7f4ee8c3276d55ea892d77526d03f80719a","prompt_token_ids_sha256":"ae80eeafd124153f30025382491abc9552e2fb6ac10aa0e25acea39bb6cab5a7","ruler_category":null,"seed":null,"selection_rank":106,"selection_sha256":"4357903439ea78adb2c1be88172806c1a5e5d57f1bfa26229be4d3d56bf15b5f","sequence_length":150,"sequence_token_ids_sha256":"58d577039503bfee517e7ec9f595d3378f0520d7b10780457232e42ac1f8b9e0","source_content_sha256":"d75386bde56ac72aa78e5b13019f0613da914d25303ec7ae96f618caf713bf8d","target_token_ids_sha256":"25dde1ab97315ca73c3e47f45297c03f015715e1dbf87dc40a90d1fff93faa1d","token_span":{"cache_exposed_start":150,"cache_exposed_stop":150,"prefill_start":0,"prefill_stop":109,"scored_start":109,"scored_stop":150},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"743e09d0b63e0d1d69cb1f4d92a19f1b612ce8d5ed318965c7cbc5ef10da4ad0","anchor_positions":[5,11,17,23,29,35,41,48,54,60,66,72,78,84,90,97],"anchor_positions_sha256":"3a7e2557ca5cb4f3e315ceb26b0546b57f9a7cdbff08065541d63492d4059513","canonical_id":"789","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,11,17,23,29,35,41,47,53,59,65,71,77,83,89,95],"fisher_boundary_sha256":"03a7963bc281465bc3ed237e9329c2d675fc5a2743cb29e2366d18c053e4354e","horizon":1,"input_positions":[6,12,18,24,30,36,42,48,54,60,66,72,78,84,90,96],"input_token_ids_sha256":"51ddc5b200c5449da711c0cb31fd958274770a19f7df1e0a919bfde5c62fa46c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,13,19,25,31,37,43,49,55,61,67,73,79,85,91,97],"target_token_ids_sha256":"5b9e09d236094c93cd27152a8362d6fc0907405ed836f4c806eb6a49ce662637"},"formatted_content_sha256":"95cda84841f0ed6748d2aa3eab9b3f2a1d602bef0648c2354cb235fbc49eca99","generator_receipt_sha256":null,"identity_record_sha256":"1b904dd9e776f5697d87583519b28c5869436344070e3c267f5d2f9cbe24f679","prompt_token_ids_sha256":"59d140fe1bf3dedbe286b918f769e76d9062fc0a8677694f8a29a4264c3dcfc6","ruler_category":null,"seed":null,"selection_rank":107,"selection_sha256":"435d524443c963ac0697492db5d755ed96c21825c6c12686e14ad1216a9d98d3","sequence_length":98,"sequence_token_ids_sha256":"f843f9ee0ba37389203b844c5d69910ebcaaa1266a694030f5cd56fa427be6b8","source_content_sha256":"528cefc12ccc28d572fc35698c3e55d5fa17974dcd83ab45fe269d5d96166d59","target_token_ids_sha256":"e052b469e73f16555fd8aae9d7e12029ab50d45bb5400880bce25ee85f7c0582","token_span":{"cache_exposed_start":98,"cache_exposed_stop":98,"prefill_start":0,"prefill_stop":75,"scored_start":75,"scored_stop":98},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"56264414b48afbfd02d06a09b83695f1be6a109c5e5917043688018a40e3a3c2","anchor_positions":[23,47,71,95,119,143,167,191,215,239,263,287,311,335,359,384],"anchor_positions_sha256":"5ecef0009b1f2b5ca1c05ae59fc5945152ea0233cf7a7da1d75b9641a0da44d4","canonical_id":"819","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[22,46,70,94,118,142,166,190,214,238,262,286,310,334,358,382],"fisher_boundary_sha256":"83c8e0577de0e56d18d8697c6d54b87b15a0a615623c44993bd8b9c2163d30c8","horizon":1,"input_positions":[23,47,71,95,119,143,167,191,215,239,263,287,311,335,359,383],"input_token_ids_sha256":"e56c492befed4cd096e4eb5258b978187c18cba26036c7ea511ea7b3828aea8a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[24,48,72,96,120,144,168,192,216,240,264,288,312,336,360,384],"target_token_ids_sha256":"b5f4bffe29cf172dbb04ddcf7fd6b1f217b698f7016f31ff8f6e48c727e2c9be"},"formatted_content_sha256":"616a3535fe1ec791a293ac6da5c856a7712282d6db633cd638270d4013cec624","generator_receipt_sha256":null,"identity_record_sha256":"c33ba3115c44c968c4ff9d9c33e133ea4f975549b4e7717c6581b9f976c187f0","prompt_token_ids_sha256":"de127a7cae4376da7d66c0f3f1eae83db093b0eee64a2a21dde2ac28c80f0322","ruler_category":null,"seed":null,"selection_rank":108,"selection_sha256":"439c6f2d57f5500b6cc21860ee22a1ebce8b83eeeb2bb034371733828ffa361e","sequence_length":385,"sequence_token_ids_sha256":"5ff88ef079fd63302a6d16c9b8d2bb597b0886ed6975c9ee0d7005fd42a6852e","source_content_sha256":"1b57006191af74fc5cabf13ded0936d54e33d0eb2d92e12170ffbdf83e11e029","target_token_ids_sha256":"f0b593d02c8c7048d24a55e2360b1e42ecae542816dcf5a6d3a7a5f0d3baa103","token_span":{"cache_exposed_start":385,"cache_exposed_stop":385,"prefill_start":0,"prefill_stop":268,"scored_start":268,"scored_stop":385},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"f83dda5f76b6c77a2f8f34d32d59899a5438aa93e284ca72500f57061c684a29","anchor_positions":[8,18,27,37,46,56,65,75,84,94,103,113,122,132,141,151],"anchor_positions_sha256":"99b598ab1b26514b0793ce6e9d9b8fd5fa21cfd91c15cf4ba944b02280eccc00","canonical_id":"886","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,27,36,45,55,64,74,83,92,102,111,120,130,139,149],"fisher_boundary_sha256":"18cbe59ca514ee6ee4de7a9ff59d5f2173303011464fcc35cb15114c03969d2a","horizon":1,"input_positions":[9,18,28,37,46,56,65,75,84,93,103,112,121,131,140,150],"input_token_ids_sha256":"7e8df804a72a77f8e56f283f6f3d3e0ccf7bdd663763c7ba3cf49b6cf2c8c479","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,29,38,47,57,66,76,85,94,104,113,122,132,141,151],"target_token_ids_sha256":"fa410d4aeb53ce61e53c7bf6ae6bf9008a5152a79caa4db0b21713c71de06b6d"},"formatted_content_sha256":"1be2ddc47aaf0ef4c6ce6979f16a7daed6383578cbb0e12a913e8c76157fe911","generator_receipt_sha256":null,"identity_record_sha256":"16353e62969e446cb1fd7fc9ca7a99ffad2f73e31b73381c626a08c7293cb32b","prompt_token_ids_sha256":"fdf8cba22d99356b2668bf60f4e3c5f33e9981ae71ede27ef250e9945224d770","ruler_category":null,"seed":null,"selection_rank":109,"selection_sha256":"43d5d553cb2e28075980d0bd0920a10d225c7497df856fe96c4367fbe7e6bace","sequence_length":152,"sequence_token_ids_sha256":"8853c262834e9d1226bdfde776d4ef0e7a0e7c8778a464180efae64bfb4f3c20","source_content_sha256":"434c4f97415e9ae869612f384ad8eb1d4459b91e71891d13598de2c0ecf58ce9","target_token_ids_sha256":"c73711379fdd03655ee8104d211db2039b2bd1b1d8757202f91122e2e8800f82","token_span":{"cache_exposed_start":152,"cache_exposed_stop":152,"prefill_start":0,"prefill_stop":120,"scored_start":120,"scored_stop":152},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9bfe15074048c1b3cc4e9a64f9f82a7762f5e3fc7b6286e9505bebdfc8ce78e6","anchor_positions":[17,36,54,73,91,110,128,147,165,184,202,221,239,258,276,295],"anchor_positions_sha256":"54ac14bb1cffe66a9bd2a063df3998236174c0775660ded8a46c6b4f3b34b4be","canonical_id":"669","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,35,54,72,90,109,127,146,164,182,201,219,237,256,274,293],"fisher_boundary_sha256":"e64ce615c2670462c794e8f67c34ff9ba26d820a1d46601098cb0321f7249f65","horizon":1,"input_positions":[18,36,55,73,91,110,128,147,165,183,202,220,238,257,275,294],"input_token_ids_sha256":"2147411fed9b01bc77a54da5a575818dd51a16507fe185b59c616201c9dbe92e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,37,56,74,92,111,129,148,166,184,203,221,239,258,276,295],"target_token_ids_sha256":"d9cc42057cf2f44e2193c2fff4ec48c3d54db14f1369ce5096749eb0a37be2eb"},"formatted_content_sha256":"fe004ffe4d515aa06edc2d7a6ff8f2900da04c2c06fe055101a34ba872ba040d","generator_receipt_sha256":null,"identity_record_sha256":"5a4fdfb2bb3bdb18316e26b3524adf61c217b8661a9502ef4888ec80cf44a517","prompt_token_ids_sha256":"545545ebb1988d32125cc7c0b6360272908d9ddf46dbea3b96b9923e8f61a5da","ruler_category":null,"seed":null,"selection_rank":110,"selection_sha256":"454527a603157418a471cc4d85900b7bc64b30edc7df17206aecd43f4b706a81","sequence_length":296,"sequence_token_ids_sha256":"0c56d0026a937592d71b7773e5acad5e8774badceb5806f30884aac130918445","source_content_sha256":"76e713c3e769794fd21cc2b169d3fd25155269e05a87202fa2ac7130dd7d9534","target_token_ids_sha256":"e5afdd086f0ae8795afe5c271dd9f9a4fae2f74fda8cd0e04e3f697b2d39bb25","token_span":{"cache_exposed_start":296,"cache_exposed_stop":296,"prefill_start":0,"prefill_stop":113,"scored_start":113,"scored_stop":296},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"bba06f38d7ef46d2274fcc3c62d6b99545aea08fc9a970bb2cf4112497f86d55","anchor_positions":[18,37,56,75,94,113,132,151,170,189,208,227,246,265,284,304],"anchor_positions_sha256":"94f33a91bb15b5b174db14bd5a18a220943ce79ec8eba0ee2655b965e84d64e0","canonical_id":"846","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[17,36,55,74,93,112,131,150,169,188,207,226,245,264,283,302],"fisher_boundary_sha256":"e56eaa1d53ef9cac5e4ea05eac5fae5d2ff32456ec4fd16aa432bb0ba682df16","horizon":1,"input_positions":[18,37,56,75,94,113,132,151,170,189,208,227,246,265,284,303],"input_token_ids_sha256":"b57a987371cd39d03b724b08c3b6837260a5e8ad281a18e85c9901110811e452","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[19,38,57,76,95,114,133,152,171,190,209,228,247,266,285,304],"target_token_ids_sha256":"adb018d6d0b2d7716ff0afe479b6ee4cb1e352197e41003e0e3962d666cf8a0f"},"formatted_content_sha256":"d783edb3ae5e40760a243586ed62d48e4d1fab758d36793b9e509e3e34c74a3d","generator_receipt_sha256":null,"identity_record_sha256":"3c715e7b82c94eee9feb5fcc68ac303cd191e6ec2068a3d300d3ca8e4dd6e5de","prompt_token_ids_sha256":"604c21c3a8e0575c89ca1b7911771a2a7ef7b4a915f2b165be5a3a2c56b206e1","ruler_category":null,"seed":null,"selection_rank":111,"selection_sha256":"45dda3a343ef8f05e6b253db4cedda79669dd8cedb0368cd7ba98b9b569952cd","sequence_length":305,"sequence_token_ids_sha256":"b0ea9f94ab26fc47af030e71000bddd166cb8e1001febd6121f29acbdf792437","source_content_sha256":"e03f7274bd5dfa833b5d69f144a32a2866a4d5e2df73779e455ab0a797e8b9f6","target_token_ids_sha256":"7c55e00d0effcd9ceef682e102dd49697117fe7a162f11e2bcd35a18523b96dc","token_span":{"cache_exposed_start":305,"cache_exposed_stop":305,"prefill_start":0,"prefill_stop":182,"scored_start":182,"scored_stop":305},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2f002330a14ccd49bbdf1186b94d7f4a660f3c79ca0935f6733429506fbac4dc","anchor_positions":[9,20,31,42,53,63,74,85,96,107,117,128,139,150,161,172],"anchor_positions_sha256":"9bb32a96de9c1958e4d75c68046a769ab85020ffa742f7286cba9f5cfe66172b","canonical_id":"809","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,20,31,41,52,63,73,84,95,105,116,127,137,148,159,170],"fisher_boundary_sha256":"b791fc560913737b45746c61831af949c9b84432325347422a7a866d99f55180","horizon":1,"input_positions":[10,21,32,42,53,64,74,85,96,106,117,128,138,149,160,171],"input_token_ids_sha256":"35bdf9fe7875294a945c0b7fa771287b6e892aade75b41750c2e0144945a8d4f","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,22,33,43,54,65,75,86,97,107,118,129,139,150,161,172],"target_token_ids_sha256":"901d22acbaeea26debc37b388abaf43ac50248b0422016d07fd3a292798c4cd0"},"formatted_content_sha256":"7d79ec67027ef058aa16aa49df49541f7821cc1e06a8283c93d05218bee5c30d","generator_receipt_sha256":null,"identity_record_sha256":"00931944a29fecfc4c8c6b60832e5269dd3d431f1fc0fec8f21cfc3d5b2e56c5","prompt_token_ids_sha256":"d2c4a745f10a0a0263c72f290f8ef232ab4aafe62f10358bad7a317ba0990e25","ruler_category":null,"seed":null,"selection_rank":112,"selection_sha256":"4639d4a26db93822b4b2e18576a8169bd874398e85f3a592904ffc3f972fbb17","sequence_length":173,"sequence_token_ids_sha256":"97308fe18d6a1e87ca3681748a23734c46402fa3f1992ea007b186d80d049540","source_content_sha256":"197f56fd23a55154213be6170da63f959f105eb46b7cd9cb707820e74a06529e","target_token_ids_sha256":"c3f58b0c22170cfdfc5245c921af88598385a2623da14949ba7503f68723c427","token_span":{"cache_exposed_start":173,"cache_exposed_stop":173,"prefill_start":0,"prefill_stop":128,"scored_start":128,"scored_stop":173},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"696c3eff19f8cd3ba60be744325a381bcade3a5bde2058ce2b7057bd86a602e7","anchor_positions":[8,17,26,35,44,54,63,72,81,90,100,109,118,127,136,146],"anchor_positions_sha256":"95e4d2457eadd7ff2ebe49af36c11afbb6034a7c8d08013efa9de5325f17bb5d","canonical_id":"797","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[8,17,26,35,44,53,62,71,80,89,98,107,116,125,134,144],"fisher_boundary_sha256":"0f290f46b842b2d9555c6342ee65ad579f7f6195e2840a98edaf138b7f77e879","horizon":1,"input_positions":[9,18,27,36,45,54,63,72,81,90,99,108,117,126,135,145],"input_token_ids_sha256":"45d165859afaa71c112d2d4db0282de95319684d4d160e6d34be3255f19e378c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[10,19,28,37,46,55,64,73,82,91,100,109,118,127,136,146],"target_token_ids_sha256":"dd11e1c91e7cf97f459b2696ad0d206a7e54b2e224552b8590c9c6431e8584a1"},"formatted_content_sha256":"8e313e746cb70b0f39a63b254855866da5978faa757f114d540a3e7ae9d63cbf","generator_receipt_sha256":null,"identity_record_sha256":"6952ce6716bd8931f4ca2c58ccaf72f466062aa0785261289d59a5104678a7b5","prompt_token_ids_sha256":"04b1d2f628e08d770e64feea8b70cbb7a64b347ed50b692622c6c3e0c7c8029d","ruler_category":null,"seed":null,"selection_rank":113,"selection_sha256":"46503a4924e57013ffda3af84d33a2db536b465afca8ee866ff2a2e87961e472","sequence_length":147,"sequence_token_ids_sha256":"dbf6194f0a08074850f461642c989526f8fe9049d05306dfd6a2da9a9987f8f8","source_content_sha256":"aa80156f16bf70bd62cebda84762b9a1a8ae437dc8871c0acb7d61e9d83b1b8a","target_token_ids_sha256":"d91b2a60c8ed8407d0c17107efc9a4cc4e719d8a094196cb04d089c6d8aea011","token_span":{"cache_exposed_start":147,"cache_exposed_stop":147,"prefill_start":0,"prefill_stop":91,"scored_start":91,"scored_stop":147},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"b907107ca4056a13d77b90b11cffa6104f002742e48e4d73d84701f35cda3ce3","anchor_positions":[7,16,25,34,43,52,61,70,78,87,96,105,114,123,132,141],"anchor_positions_sha256":"4a3ecd9aedde633817cc01e0837692eec5dd57faddeca3f7bcf5e18333ad7c64","canonical_id":"803","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,25,34,42,51,60,69,77,86,95,104,112,121,130,139],"fisher_boundary_sha256":"8c5dd3de4221407fb31879a8831c58de078b7bd96edff29efabeb0d4f941ca5d","horizon":1,"input_positions":[8,17,26,35,43,52,61,70,78,87,96,105,113,122,131,140],"input_token_ids_sha256":"612f42435d82b79fa57bcb8dc07f12f964553d764d942f85ac5c989daa0ac60d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,27,36,44,53,62,71,79,88,97,106,114,123,132,141],"target_token_ids_sha256":"20ecbcd0e6f4d0f38e6e34055d4306d4556676db803bddcbf4efb50dde35e518"},"formatted_content_sha256":"b80b90161540d8a9b1c0ba6896342db1e035ccefbe96663db9df9b74d82a3eae","generator_receipt_sha256":null,"identity_record_sha256":"cdbc48a4424d2515ce6141290d69310f055340010d03518c808f7666de9eecb6","prompt_token_ids_sha256":"9a94491891f327307e6c2d541dae50c594d32f7e7113f10a4ad421cd16f7efee","ruler_category":null,"seed":null,"selection_rank":114,"selection_sha256":"47a81fc27a92568642839ad9294c190dcd0b21f98ad4eca4472567609f81bac4","sequence_length":142,"sequence_token_ids_sha256":"4e6d4b6ce7a5a64ad6b5d03e8b6c0786f2a1206a63cd36a35f88e41282093685","source_content_sha256":"fcc558736fc46d07648e59309cc19de0322f28060b2265985cd5cb46f9ccac97","target_token_ids_sha256":"16fcd9ca012cf831e086d70b1c2f66e51e595f470fb2610a600125f50807abae","token_span":{"cache_exposed_start":142,"cache_exposed_stop":142,"prefill_start":0,"prefill_stop":82,"scored_start":82,"scored_stop":142},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"61f0fff09b788c1dde584ff85418eee082c77947d695e4df4e831eb33cf048eb","anchor_positions":[10,22,33,45,57,68,80,92,103,115,126,138,150,161,173,185],"anchor_positions_sha256":"fde5a3b64a487819cb5be9d572133781e4744390c11836e2f115ba2608fe7216","canonical_id":"644","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[10,22,33,45,56,68,79,91,102,114,125,137,148,160,171,183],"fisher_boundary_sha256":"3c448eaba6e976f4d8540a8018f1b6ee02f1a078e5ab97372fe15de20e37c79d","horizon":1,"input_positions":[11,23,34,46,57,69,80,92,103,115,126,138,149,161,172,184],"input_token_ids_sha256":"61a5b967b5a2460665ba8ac148a2ed8fda9b1a0e95c8d66a509db1c89e384bb5","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[12,24,35,47,58,70,81,93,104,116,127,139,150,162,173,185],"target_token_ids_sha256":"e6285297df83884b53ce8b429f39bfbc76734d91c897b1ef6b45e02c90bdc53e"},"formatted_content_sha256":"cc789f8b54535a69935595ebbc2d15ace8da8a517f3e3a766f4c9801ab975e73","generator_receipt_sha256":null,"identity_record_sha256":"add76763c425e50ca33682da8956ff29cc4547518dd84c9fea5b339de6f6125b","prompt_token_ids_sha256":"a672313ea736e1353ebd2a843db4d5ae378ddbec92e54cb81d21376a13087e88","ruler_category":null,"seed":null,"selection_rank":115,"selection_sha256":"49614c4b89fd4e064236884bd54d2b6744d30da34cae36de98a3f8d6dc005295","sequence_length":186,"sequence_token_ids_sha256":"2a629a0ceb8ed135d2d3814402c87b6b4d4925c16ec65dcf756469c7f70d28dc","source_content_sha256":"6ca1519c2cb7b633366e44afc37a261e40427c751412998cceb863343020cf4e","target_token_ids_sha256":"147bf03e5bc17953126d30a0640ae17e4b57cbaf4d65322290bee14c6f8c12a8","token_span":{"cache_exposed_start":186,"cache_exposed_stop":186,"prefill_start":0,"prefill_stop":160,"scored_start":160,"scored_stop":186},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0a09087a0815d40c6da7249de80b96629effaed44659b49eb51e112de7acf916","anchor_positions":[5,12,19,25,32,39,45,52,59,65,72,79,85,92,99,106],"anchor_positions_sha256":"1ec66db99ded67a6b74f7dd5259127f2ba61fd9a33265762c1ea04710558d7bc","canonical_id":"647","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[5,12,18,25,31,38,44,51,58,64,71,77,84,90,97,104],"fisher_boundary_sha256":"bbab335f18e2c0ccb810bd85c071826d79d9b490e590cef0c6d681341c0cc08e","horizon":1,"input_positions":[6,13,19,26,32,39,45,52,59,65,72,78,85,91,98,105],"input_token_ids_sha256":"46b16340d489ab9d5e99f73c42d4a7005f7436243eef5fc34fa92767754e2622","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[7,14,20,27,33,40,46,53,60,66,73,79,86,92,99,106],"target_token_ids_sha256":"761af83bc90049bcb9676e87032fb087e505ce54ccdace5bd3157cfe531b0e6f"},"formatted_content_sha256":"892bdd77965161d3f33ecae4371aaab245852c1a733575ea82a03f4c5162b6e9","generator_receipt_sha256":null,"identity_record_sha256":"096bf170f6323441d41cfdcd82e991e5330ce43bbca166fdaff8e2867cb3901f","prompt_token_ids_sha256":"9d1498f0e195703f3d0a9bab6ab2b95760b77762a386814e263445e614eebe8e","ruler_category":null,"seed":null,"selection_rank":116,"selection_sha256":"49bf4e9209616a951134062596b8e1762dbe5ae3d6768f1c37fc3ca1a831c8d8","sequence_length":107,"sequence_token_ids_sha256":"e9307774f416105649348d7b962aed8818afdd9670718378a8011aab53794a9f","source_content_sha256":"a4c7ff187da9b88bed24a5ce4e33b1d245e061cc2f7fb731157aeb3dfaf250ab","target_token_ids_sha256":"d3f2b417944e705153e271af9eeac8e255eb09fdbc9a8241cedc49106c05cb62","token_span":{"cache_exposed_start":107,"cache_exposed_stop":107,"prefill_start":0,"prefill_stop":82,"scored_start":82,"scored_stop":107},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4294762e8478186f4347187eec4203bd212ed4716abafee16d6c831b211a92d0","anchor_positions":[6,14,22,30,38,45,53,61,69,77,84,92,100,108,116,124],"anchor_positions_sha256":"fe7a10db15891cb54d23372b10af49697c0401f54d8f6b37c01417545e9ba930","canonical_id":"605","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,14,22,29,37,45,52,60,68,75,83,91,98,106,114,122],"fisher_boundary_sha256":"c098a5d59dd920acf5e6ecb372e0ff5df7d0b3ee1501bb011f699481cf66e206","horizon":1,"input_positions":[7,15,23,30,38,46,53,61,69,76,84,92,99,107,115,123],"input_token_ids_sha256":"faf2e5d622e9951baec003f38847af5d1319f255169b8c041a433d5b46952e3f","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,16,24,31,39,47,54,62,70,77,85,93,100,108,116,124],"target_token_ids_sha256":"26f6b36890856124d1f7f7a676b24ea8aeea3382d92cd1489d141c22982a041b"},"formatted_content_sha256":"80aa72fd69bd10f4c2e5403802378e6fb00d15a1e298252ded65c6ce760ab095","generator_receipt_sha256":null,"identity_record_sha256":"ee34d2c3691827aa084c190eddddebe9d1345888c7f48e25c87d60d765f6dfcc","prompt_token_ids_sha256":"9fd32241d4f2b36ee9f01d68e556fe7d2b96791f996bcb4bddb46b56225c1199","ruler_category":null,"seed":null,"selection_rank":117,"selection_sha256":"4c310c24967e920116a6713c985dc574fc27f51aa0e5f96102923b5848a2fea8","sequence_length":125,"sequence_token_ids_sha256":"d5c71f5ac8cca279a5091bc56ba019e4c7f2a97540c957843ce3658f1edad1ac","source_content_sha256":"dcf527a4f6986a9167a459248d6567bd87701d5fa34856a31f9be5ee02c91168","target_token_ids_sha256":"741ab7638f051c1d782ffdca7cafb68a673af7ec2773fdff58abe66e46d7eef1","token_span":{"cache_exposed_start":125,"cache_exposed_stop":125,"prefill_start":0,"prefill_stop":68,"scored_start":68,"scored_stop":125},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"05c8e3cbead3af43c5c1e6e8d87467ab8c03ec4648a4298153b82d936448e5ad","anchor_positions":[19,39,60,80,101,121,142,162,182,203,223,244,264,285,305,326],"anchor_positions_sha256":"f3e5839dc1785a2c00f87ad2ea0e816ae94270b3238251e62817f25d09a21cbc","canonical_id":"630","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[19,39,59,80,100,120,141,161,181,202,222,242,263,283,303,324],"fisher_boundary_sha256":"de76b71c7d9a5873e8f18d38d9bfe274b70bf5ccb7ad95749ffff39ac8269bce","horizon":1,"input_positions":[20,40,60,81,101,121,142,162,182,203,223,243,264,284,304,325],"input_token_ids_sha256":"b71f4ccff2a296d03b9b8fada60f1b7538853521101cb758ed05e1b6e41628d2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[21,41,61,82,102,122,143,163,183,204,224,244,265,285,305,326],"target_token_ids_sha256":"939dd654b199e687770731bddc84e26956b5c40203089d198687d9c12778ed94"},"formatted_content_sha256":"b39458254cec5c52b4b1ffb7f1dc476b0e99bab62c4d66d0998f147e9cd767cd","generator_receipt_sha256":null,"identity_record_sha256":"d20e005d97d9c08185e9a25884618ab3451741d9e00a6f7cf4842daa416fb1a1","prompt_token_ids_sha256":"92effc02a358ea378746b418ac301dfe5bacb980d7a8a9bfee9363f21a656396","ruler_category":null,"seed":null,"selection_rank":118,"selection_sha256":"4d4a3d035c7868292b96361ee3387aa0653c72bf82ff2ffbcdeee1c544f9b310","sequence_length":327,"sequence_token_ids_sha256":"11215c622904358795cd7771cb56cd3e68586e31dea00e30330fdece99fcd790","source_content_sha256":"d026e046adcf26a5522065e389d1e74a8d7ef963fd53e4f8db8952c3c3058226","target_token_ids_sha256":"59f027a42d76a9550f2a9d95174c87b9fd402df01c12dd6433c83e8bbb1834ec","token_span":{"cache_exposed_start":327,"cache_exposed_stop":327,"prefill_start":0,"prefill_stop":236,"scored_start":236,"scored_stop":327},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"865d6b59e048496b3bf7c4b287421ac71174d11943f8a146bf5f215f6e662bb5","anchor_positions":[7,16,24,33,41,50,58,67,76,84,93,101,110,118,127,136],"anchor_positions_sha256":"b394d1b64998869298fd2b292d4ab84e6faf0b5a40eeb811c668f2563bb4df64","canonical_id":"868","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,24,32,41,49,58,66,74,83,91,100,108,117,125,134],"fisher_boundary_sha256":"7f4687722337e0596bbc45f63cbf6b70e8988c03af12154439f27a9db973cb09","horizon":1,"input_positions":[8,16,25,33,42,50,59,67,75,84,92,101,109,118,126,135],"input_token_ids_sha256":"8d01b29f328a257d664952e75b1923e83d2777cbcd4b7dd6cee1855ba4bd0e23","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,26,34,43,51,60,68,76,85,93,102,110,119,127,136],"target_token_ids_sha256":"8b8ebd92cf550b5df42ff3360fbfe39d8efc120a7e745a701b2476dc01b5b305"},"formatted_content_sha256":"44d9a5afa4c9790a32cb254c2d7e1c8151d16636b7c10318c889176cec4bab1e","generator_receipt_sha256":null,"identity_record_sha256":"fa45bce52100c237b5694c601b0c6d6d61bafa11dc08e55e2816e2dc5931e145","prompt_token_ids_sha256":"b9f642149302b4aad7f0cd78c10cba633079e023b978bce75611464d15bbca52","ruler_category":null,"seed":null,"selection_rank":119,"selection_sha256":"4d7d9814d186699e85046419caf35e8bacfed3ab6650cbd36ef4705e2fe2367b","sequence_length":137,"sequence_token_ids_sha256":"b282f3d8b38678f37b26da12ae374fa98bd55719947e98f5670fa8a8530270c8","source_content_sha256":"eae1ffd3380c73bbc35bb8929b99e7549668c0ee1f54da998081ad923ebc2b98","target_token_ids_sha256":"92e8679c291f99696b3ac568510fbe994ddeef46b7ee37dd2252d2f7a9c4bf84","token_span":{"cache_exposed_start":137,"cache_exposed_stop":137,"prefill_start":0,"prefill_stop":78,"scored_start":78,"scored_stop":137},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3053356b37e0ec018499d9df09100d5d459ba083855ec3be6383459c76336c2f","anchor_positions":[18,37,57,76,95,115,134,154,173,192,212,231,250,270,289,309],"anchor_positions_sha256":"1d1fbb7874404570c3a9cce52a9bba801a787fd59ed00a041f9aca9922c13383","canonical_id":"902","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[18,37,56,76,95,114,133,153,172,191,210,230,249,268,287,307],"fisher_boundary_sha256":"b03addf9ce28714390d0c8aa72e4cf5bfc4d998e1e89125d619cc5de86bb2d27","horizon":1,"input_positions":[19,38,57,77,96,115,134,154,173,192,211,231,250,269,288,308],"input_token_ids_sha256":"a9d774c7e1c1f9d59bd7b757d0d21d9849e8eec8052afe6025037cde871559f7","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[20,39,58,78,97,116,135,155,174,193,212,232,251,270,289,309],"target_token_ids_sha256":"52cd56fb6fa5b1573d45b6aa37a594fb645c879bc5efa69d6edc441f4e348a6c"},"formatted_content_sha256":"5c1d9ec33b3a94931255d1220cce10d4ee097868c37884a8e19ae9f9a448966a","generator_receipt_sha256":null,"identity_record_sha256":"11c597e2cf580e07f77afc5041e62139cfccff5f5ab2c548565673fb0b500832","prompt_token_ids_sha256":"bf15bcbdbc30699f9cd0081bb00efafed21ea1abf4e5381da4e70c8a8ea2907e","ruler_category":null,"seed":null,"selection_rank":120,"selection_sha256":"4daa86751a582244f7d590b6ba730fc4e01e1d3eb676941a3552f1d490faba8f","sequence_length":310,"sequence_token_ids_sha256":"376f09bf804c7e1d57cc7e70ae46ecf6050c3c815f6a7cee4a95bd11199c01d1","source_content_sha256":"57ceedd084b1dac9a04f6d5be3b4a7e63c4474855a5cbb2577214912ffc07f8e","target_token_ids_sha256":"49502604cb2f2efcb1c5cee4993331ea6bb17ff438c88e61241bb8472ffbd75a","token_span":{"cache_exposed_start":310,"cache_exposed_stop":310,"prefill_start":0,"prefill_stop":278,"scored_start":278,"scored_stop":310},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"afd54f87610f85707a02f20f8b511f975ef944c51b61fa08f447b37ed40c6a78","anchor_positions":[6,13,20,28,35,42,49,57,64,71,78,86,93,100,107,115],"anchor_positions_sha256":"5af6cea1df262184ac4897d6d91d25605392e5fb08e41b5742c64172e9670d87","canonical_id":"955","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,13,20,27,34,41,48,56,63,70,77,84,91,98,105,113],"fisher_boundary_sha256":"1c62dce0ea57f80fb1a56a3ea511f2ca9f646f6413e8db3d99f5d99de6521811","horizon":1,"input_positions":[7,14,21,28,35,42,49,57,64,71,78,85,92,99,106,114],"input_token_ids_sha256":"ef01c4ac7ec6c6b186fdd4eb31a7f21ee8d3e4dc566355d91c713471887db336","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,15,22,29,36,43,50,58,65,72,79,86,93,100,107,115],"target_token_ids_sha256":"785091482f15a4ef0949a945686a1778f4de4cf07b8104861aa31a9db7b18f4a"},"formatted_content_sha256":"ef1384fcf9c059d64323dd5fa9880b686c5b7ea953ce70d245534bb1c6ed7137","generator_receipt_sha256":null,"identity_record_sha256":"f520632d9fe13e3b9ffeebdc9ea540d828dee798f54926584e9872d0fe7b87da","prompt_token_ids_sha256":"010438e2e3ae3ee5626141c874feb0f74e50fca32fd27c7d8e49a07e790ec560","ruler_category":null,"seed":null,"selection_rank":121,"selection_sha256":"4dd5c5d56c9d92dfa81aec10aa5beae38c2061e96e799b813b238234a0ce8e4a","sequence_length":116,"sequence_token_ids_sha256":"fcc6077ed67145358daf5bcf3eb1ece0e289f68f139666cb89192eec34ca4386","source_content_sha256":"8408dff25eaf845d972ce9ae510b9d76eaa3e7a6bcfb8b8c7a628d70d2dd6cf6","target_token_ids_sha256":"76411fdcb68c0c051819b5bc2c0fed5b4048dcb9000413fb60e5de16d537d568","token_span":{"cache_exposed_start":116,"cache_exposed_stop":116,"prefill_start":0,"prefill_stop":72,"scored_start":72,"scored_stop":116},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"6bfb1e634cbb65a8f9c6d30dcee64be9d19b2bf1774c4fa850c4b7b5ce5abb9f","anchor_positions":[6,13,21,28,35,43,50,58,65,72,80,87,94,102,109,117],"anchor_positions_sha256":"f40be3d1cf7064d7e86e3a810371d8f2d368583e1dcff4660d5d02fc4a37f43e","canonical_id":"640","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[6,13,20,28,35,42,49,57,64,71,78,86,93,100,107,115],"fisher_boundary_sha256":"261088f8bd3fe25be23146357491bf2699628f4ce493685b028cc1ddbac01eb3","horizon":1,"input_positions":[7,14,21,29,36,43,50,58,65,72,79,87,94,101,108,116],"input_token_ids_sha256":"db9064f4846367e8570c2c84fe797b835d18960e9211681a5e0daac0654b2347","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[8,15,22,30,37,44,51,59,66,73,80,88,95,102,109,117],"target_token_ids_sha256":"0a9ab2151f96a33137ea1ad001a0d76f499d390891517e6c83bd2ac4188c1207"},"formatted_content_sha256":"06826303346d9d86d61fc5abbffb92ebac62b2e090a9f70b82927eb026441a29","generator_receipt_sha256":null,"identity_record_sha256":"7b2ff5b497e0a82c3b0d260a880af1a82e4a4c08c8185dd3ce186597015d3a20","prompt_token_ids_sha256":"05a2a260c33e6e500d66729f3e56aa8f9ccd92ecf9825c6449d5f8fe340e67a0","ruler_category":null,"seed":null,"selection_rank":122,"selection_sha256":"4ddb9210eb7ae0fdab8792259d081bdba6eae451141c47a811a0245cec7ee62d","sequence_length":118,"sequence_token_ids_sha256":"20509fd474053e85eba21ed705bd626c6c6f337445f4daf2177bf8b506211954","source_content_sha256":"6438ac844fd844aa7880ed46075764fa684488592b7d562f4ae65218f8b35da2","target_token_ids_sha256":"fd3de534e5c7f8e743fbfaeaa6eb5d1a4c3d2acfdfd4a83cb4e9f6b0a9383173","token_span":{"cache_exposed_start":118,"cache_exposed_stop":118,"prefill_start":0,"prefill_stop":85,"scored_start":85,"scored_stop":118},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"f5ff8bcd127367dd786bcc2b0c253bbddf5577a9692a04a945dfd7611e658813","anchor_positions":[7,15,23,31,39,48,56,64,72,80,89,97,105,113,121,130],"anchor_positions_sha256":"4b59ffbfdc6171d05db7e4ce13fd81c13fb1c2704ef54cbfaede26eafe2b1382","canonical_id":"602","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,15,23,31,39,47,55,63,71,79,87,95,103,111,119,128],"fisher_boundary_sha256":"4366416cc3edb9f1e2ee5e0a55178de659ab8d05d59a8cbbce27f688cce7b2e6","horizon":1,"input_positions":[8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,129],"input_token_ids_sha256":"2db46a104b4295959d6a16c851488604e471fef800ab5243426a1d23cce719c5","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,17,25,33,41,49,57,65,73,81,89,97,105,113,121,130],"target_token_ids_sha256":"2b0416f82f51105c2f972ed61b19f300631a81a49326605f7f5bf45fc3d5957e"},"formatted_content_sha256":"48559307e617b830c4e7a2ef8b20df2f6a53a7ddd15f766c7415c7b8597ff2f9","generator_receipt_sha256":null,"identity_record_sha256":"baa5a63154617acc29a3b5192d3459d2edae4c0959c00d24e2dadeeb3837d757","prompt_token_ids_sha256":"0bb478e0d8162f79fa8ab3da8baed5b87413927171e2f2c8071c49ab30da5521","ruler_category":null,"seed":null,"selection_rank":123,"selection_sha256":"4e02d4ed1f0c6d170eb518cc4020d82ff4ab1bd18d4949c5913e319fa7cc9e83","sequence_length":131,"sequence_token_ids_sha256":"e7b30f61231d01e87b3afa4dee3faf2008c8e20ffd1b3a478e7a94932567aac3","source_content_sha256":"59b47992e5e199dff2a5b0af7ea0a02713ea803a5dcc6fc3db6ecda2c4879288","target_token_ids_sha256":"78bd9c18ec9dfb40ae35c333bd6dd45162c4cc291b415b84d5ae2a87dd1ea334","token_span":{"cache_exposed_start":131,"cache_exposed_stop":131,"prefill_start":0,"prefill_stop":86,"scored_start":86,"scored_stop":131},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"bf2d404007f3e1af973b816cfc665fdfc8257f337deb35322ac0077058cea4d9","anchor_positions":[12,26,40,53,67,81,94,108,122,135,149,163,176,190,204,218],"anchor_positions_sha256":"34e0f2438d49fed8c871a33d3818a40fc035b239ea8639ec66d64d1e59a9bb66","canonical_id":"712","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[12,26,39,53,66,80,93,107,121,134,148,161,175,188,202,216],"fisher_boundary_sha256":"957de9da750e31aa4980cb34b1fb160bb11d67e08faf67af18d3cd3e5289f959","horizon":1,"input_positions":[13,27,40,54,67,81,94,108,122,135,149,162,176,189,203,217],"input_token_ids_sha256":"f56a9ec29643c9eba2eb44791cb09e84d9f2b231c902427ad5756e74ec534316","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[14,28,41,55,68,82,95,109,123,136,150,163,177,190,204,218],"target_token_ids_sha256":"c0a62a7a97676edbafdec6adb2cfe3c67719d5ccdcfcc3a0ddba0c6331624ede"},"formatted_content_sha256":"b47cde36a934283ca86c0366ca05754f6267c0fa319aa8c9ab18bc5ff0e5d9b5","generator_receipt_sha256":null,"identity_record_sha256":"4cc5c35280b65e6893e928c4b6a5b2caa568e303f0840d0df3438ecc6b258676","prompt_token_ids_sha256":"4ef7f3aa586f42e443494b0dc4987d24f11b6cc50acb5df627b135819de32092","ruler_category":null,"seed":null,"selection_rank":124,"selection_sha256":"4e132dddf48254f73a1575b1a311842bc8036e6c5310eeac68d27fcfa9705827","sequence_length":219,"sequence_token_ids_sha256":"45c3e814d2308b690e19384d22ca303456c689d9292a9292a9974a96d2c27dfb","source_content_sha256":"5355ae6f3f743535c93a53ce5c0f05092f5c9e89585eeb219a5a424a29b6a20e","target_token_ids_sha256":"8c478cb16206b569ca9079b27faaad0f0bf99e2182d4dd8ff62cd8328afac5ca","token_span":{"cache_exposed_start":219,"cache_exposed_stop":219,"prefill_start":0,"prefill_stop":183,"scored_start":183,"scored_stop":219},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"36c3b31211a995bad7f56d142b515472a78c8c0bab6bb376f23f53c90f2b7139","anchor_positions":[8,17,26,35,44,53,62,71,80,89,98,107,116,125,134,143],"anchor_positions_sha256":"486e7f47d72e2b3d4a862156a2b41e9dbb05a70d78349fdcac0b01e09667750a","canonical_id":"606","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[7,16,25,34,43,52,61,70,78,87,96,105,114,123,132,141],"fisher_boundary_sha256":"bcfe34f783ad5a75f3e21e034bca0957d966c976fd05656202a26e596ce4d242","horizon":1,"input_positions":[8,17,26,35,44,53,62,71,79,88,97,106,115,124,133,142],"input_token_ids_sha256":"2b725b30a4b9b4a7e69480982037a0a07c57e454ad8e62b46c0f0d34a3b1774a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[9,18,27,36,45,54,63,72,80,89,98,107,116,125,134,143],"target_token_ids_sha256":"872de4a20f2b57da5f4caff92476bd417ca5b3e9d701ef7d101a9fd65f42abed"},"formatted_content_sha256":"c6fed4dd2cb64cca6e833bf23797bb7c1d862a9c875ccc65acb9b0897618cc79","generator_receipt_sha256":null,"identity_record_sha256":"a9e7062d3f002e0a39343fd4f410be18e7409cd9643a09c6dedbf4bf79332f30","prompt_token_ids_sha256":"67adef290bf71e36a6af78dc46a182ea33221b75714d4b89977dfbfb25d08ad7","ruler_category":null,"seed":null,"selection_rank":125,"selection_sha256":"4e211b7dc81626479060883502f3e73d8b09c6e4da35910a1305a8e23d6e76a7","sequence_length":144,"sequence_token_ids_sha256":"e35fd3a90e9e66af3e142095f74dbbe6c47e3a63d75cede17749f5f6d49e6dd0","source_content_sha256":"3a119437b4f93db7da36fbe2d787607dde294a4a838f8b29d327bc68a0eb0478","target_token_ids_sha256":"764d32b0ec71bb9f1d078ccc8e4a39c3ce7dd359fa503899ebfcef2b1653ed9f","token_span":{"cache_exposed_start":144,"cache_exposed_stop":144,"prefill_start":0,"prefill_stop":117,"scored_start":117,"scored_stop":144},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"7d1f918958bc691d31836ac50e195b86b748a5336a8a243a1b427eb6b289303d","anchor_positions":[9,20,30,41,51,62,72,83,94,104,115,125,136,146,157,168],"anchor_positions_sha256":"a040c10bf4081d1ff5f0840fe94899e3d5f4cc8b3513ac78d8caa23e146870a7","canonical_id":"656","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[9,19,30,40,51,61,72,82,92,103,113,124,134,145,155,166],"fisher_boundary_sha256":"c9142db1a8573b215599b5dc489eb1c6d88615d3cbf8ad1c104742ef7bfc5041","horizon":1,"input_positions":[10,20,31,41,52,62,73,83,93,104,114,125,135,146,156,167],"input_token_ids_sha256":"c32bee20d7de9784fc14e7f94e2b1e165c75b8c09339f3ecc44be1cbad01f8e1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[11,21,32,42,53,63,74,84,94,105,115,126,136,147,157,168],"target_token_ids_sha256":"abafaed5c5aabca6d515ea1a705691d431f604b08089f1f0b25c6df8b45c8305"},"formatted_content_sha256":"374f6e67c0234e75d236ea8f2069dcdf8a631ce916aaed870d7810e3347cde44","generator_receipt_sha256":null,"identity_record_sha256":"770edcf1d6f1f10e0e3b4c54ae7a7e8a532a07b1a3d17003f426dc095bed8679","prompt_token_ids_sha256":"b45a43c9c890b88c42ea8fe0a8f0a5d71c55ec6f01a772f30a952fad7f8431f5","ruler_category":null,"seed":null,"selection_rank":126,"selection_sha256":"4e7a2318f515292e8fc2e6b77f63c13218916891ac0b3f71650ab59f87fc1a5c","sequence_length":169,"sequence_token_ids_sha256":"c61b8bdf63e3f2d95107fee666460e20eb1e4a5d5420d41d523360fc8916c990","source_content_sha256":"7164e42f57848f0ec61d9fcc2a78bec0f09e6a54082574820bd69a7607d84721","target_token_ids_sha256":"52ef9360d90d7b7b754a31958af231bec9dfcb171dd416721cfb316918352b68","token_span":{"cache_exposed_start":169,"cache_exposed_stop":169,"prefill_start":0,"prefill_stop":118,"scored_start":118,"scored_stop":169},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"9613ae532d1ac6180936e714bd20e0522fe6966715f1926f3a3b8b7af31b80be","anchor_positions":[11,23,35,47,59,71,83,95,107,119,131,143,155,167,179,192],"anchor_positions_sha256":"97c036645b9b6a78e53e848879a7e3ee08a70ff938ed6e98aa7098b21ab4319c","canonical_id":"671","config":"full","configured_length":null,"family":"mbpp","fisher_boundary":{"boundary_positions":[10,22,34,46,58,70,82,94,106,118,130,142,154,166,178,190],"fisher_boundary_sha256":"730eb3d8f69aea1c1666e7307d73b83daf05f241bec613d1628b92a50ec3bcaf","horizon":1,"input_positions":[11,23,35,47,59,71,83,95,107,119,131,143,155,167,179,191],"input_token_ids_sha256":"1100089821e0f1740660a914b8b15a1db0534566177c6e09bcf34f670905693c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[12,24,36,48,60,72,84,96,108,120,132,144,156,168,180,192],"target_token_ids_sha256":"8d49d123f2607a365571c71836e7acda4676f3df261e0a4254afd127a53299ec"},"formatted_content_sha256":"22bd9c0d68e48a76c31dfe254036ce6008c9abd1f71c4c290e76fb4135d09ad0","generator_receipt_sha256":null,"identity_record_sha256":"4cb68a3b9440b00f53a5e655ff7024c46d2ecdc277ae03326b75f41268b173cd","prompt_token_ids_sha256":"63bd2370a08ace97ff440307492d1bee8479e8e14cd40ba9b40b1487a9ad53fa","ruler_category":null,"seed":null,"selection_rank":127,"selection_sha256":"4f4ba031f801028fbf78f774438c1e6ef3c15a58f65dccc230632c90a7fe6b5b","sequence_length":193,"sequence_token_ids_sha256":"d653c8bb051679b00cd66c5370bff1e217b990629e9ca260192942d1541dc892","source_content_sha256":"85295da947527bd6a8cb51dbe0f81aed64a26a245e12c289dab42daa0830ecdf","target_token_ids_sha256":"5233a54b205f9eef3872026a1b044403750ba40d368571e658bd36b909a1a85d","token_span":{"cache_exposed_start":193,"cache_exposed_stop":193,"prefill_start":0,"prefill_stop":89,"scored_start":89,"scored_stop":193},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"797956c3bb60ce0f54962ed315c69368b4d9168e36abfd675d14ccda36e8b2fc","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/36808","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"7e362e252ec4500305135b0f2ef134e2fe129c787fb0a31326c35439cc9ff4fc","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"d0e5311a6d6bde4f201973a4428fa08ba4fa97dc6cf2580fce9e2a4604922291","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"5b2da07174c560f02d4940468e335ad6777df3f471b2be9a3550cae36b1fd62d"},"formatted_content_sha256":"f6e3c61d5723ac6698db28d6198db5b270f51e83e2c8800f5ab2cd181a993a21","generator_receipt_sha256":null,"identity_record_sha256":"e7274663db91b5411d8af35ca6979c6b6038b747658e034486c7161db217a12d","prompt_token_ids_sha256":"5233822ab87654bb7050a9eff9f77a58c8ca4a74c176420316866641587b5ac1","ruler_category":null,"seed":null,"selection_rank":0,"selection_sha256":"0000d64fc1c2791eedccba1b0a014ae30d36a69ee95be69a7d2dc5541d9ff914","sequence_length":2304,"sequence_token_ids_sha256":"5233822ab87654bb7050a9eff9f77a58c8ca4a74c176420316866641587b5ac1","source_content_sha256":"ec7b13f2bb2766533058782bbf50f269e8c0743722581ba193e8dcf47008ada9","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"5ca98bf769dab4ebd4692e9fbb560efcc0f0f474c5a49e0732d605a2683da881","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/32024","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"2b3d247e247806b4dc881d8599ac43f4212a4abfdf27c4a4984cc7b6e286bcde","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"231119f3847a33472d36eeacd144d75fbf1c16d0971ecebd977dec5f84685ed7","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"bcf8ceb1b07052b45eaec59a1762d47a3fd14770e2014af6c7462da3ae912a8b"},"formatted_content_sha256":"056228535dc5d8d08c833c37eb05eb57a524d567c7137e2d8314c181aeca5137","generator_receipt_sha256":null,"identity_record_sha256":"48c93886d09d16bbd4b1c300f10b64a7a5908445969fd866133d7dbd65c13133","prompt_token_ids_sha256":"20d38634a5307135f41953611e0e9d2a8e22a29a1f912d78af0414bcca3e0dc2","ruler_category":null,"seed":null,"selection_rank":1,"selection_sha256":"0002ed283941f3da5106f697d697735d5064de6f22aa84e512fd5fb4792c49a8","sequence_length":2304,"sequence_token_ids_sha256":"20d38634a5307135f41953611e0e9d2a8e22a29a1f912d78af0414bcca3e0dc2","source_content_sha256":"eb06d835db68d2ccee63052eac9d025e120ca9331c84186892dbfca0b8c7b5e9","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"870cf6deead4c720c1131fda048c46f654bb50af3e403058e9370e06d21978fa","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/20279","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"9a2ec9cafe47a0a911539d7cf75996655cafccb2071b17063c4d74a9a215b6bf","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"b6c9a50b23af8aa52a00688ce9f3e2f1b86afb30dba305513f107d56499b7109","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"484ad2459b1ac7e91b8a406d092192990e1eb1f2ef4295cbc9ae9b0d8db832fe"},"formatted_content_sha256":"ad22fff4026303176b09131a60d9a8a6d1a6385ccaa59768e5f6a72ff8117f2b","generator_receipt_sha256":null,"identity_record_sha256":"01c99745bb7d41ee57ea15450fc853f25c79f09d2f8bc063c0bdd3bb66dcc3da","prompt_token_ids_sha256":"aa29883de5d02657b8eddaf1e1caece900a2d6c56fcf63a868f35f285132dee9","ruler_category":null,"seed":null,"selection_rank":2,"selection_sha256":"000d6a7d93865e2955a912d7fcafdf0e79cc24fdca9b059114ada982813dedf6","sequence_length":2304,"sequence_token_ids_sha256":"aa29883de5d02657b8eddaf1e1caece900a2d6c56fcf63a868f35f285132dee9","source_content_sha256":"b899a1b6c5b3ba61252a07cb53ea7d0f15f0c63315df7b872a9c7cacc58017fd","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"c3a77259d6b11a251eaddefe696ec8873dac79b38e498e4c19008e424264366d","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/35786","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"049cec2caddadd6c5a22d5eebe0f37af9a7d660e1f8a0134c85404ef60e34076","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"34af22d248242c1ad9044c3b1cc5988e257371a5867a4bc4f686e53596cf32a7","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"e400ca0c9462a4d8e94a16cb560f6a54f948fead3ee81377ae34f4b435d27bf6"},"formatted_content_sha256":"7e0c728729a3f6e258bf44c314097e48e5bd7a3d2ed6a9f7c33b0dc333de0830","generator_receipt_sha256":null,"identity_record_sha256":"182e8520c6e45d8da1ae7d54afd83292ddcbb68a76cccbeb96d22ffdde1b9464","prompt_token_ids_sha256":"a074a6e0e6624af64b673167b490ce4de6ff1fc5bc6a5a0429a482706d5047aa","ruler_category":null,"seed":null,"selection_rank":3,"selection_sha256":"000f8dbba0aa87b15dcf3bb9e900e6a1c95b96affc23284c0aad2871def7d41d","sequence_length":2304,"sequence_token_ids_sha256":"a074a6e0e6624af64b673167b490ce4de6ff1fc5bc6a5a0429a482706d5047aa","source_content_sha256":"d99d106567585d3e0dbdbd545794876258507c655b4b92af1dd705f6e4c984ce","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"6e96a1becbdeb74006e814bfe6a5fb240a5de9f6134d609b13117dbe7211fb78","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/29413","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"f68c33e180124100446c92def52974f274452469befeff1421363223dfd67323","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"9c4a80f3b25c53840a44de74f52c06907a16b90bcc7a8c077dc4eea1fccba752","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"85a930c5f1ed93345b9259991c2d9c7a90faace00b3ca7248e61b28fbbf27372"},"formatted_content_sha256":"aa37e0852c2a9d938e44d1f6050bbe8db2ca60ee6d89c2e40ed5ddc5f9c36a77","generator_receipt_sha256":null,"identity_record_sha256":"c8f11e6e9703b0eb65e3a6f5f79135730de277373c9e3a94e68eab94f5054d7b","prompt_token_ids_sha256":"5f2026b82faac8988768fc7330d60a392741c3a5f7b364d0b87bf8d596e74bdb","ruler_category":null,"seed":null,"selection_rank":4,"selection_sha256":"001145473f2b3b765745639cd14dac4b421dffa50f63df360f9059d362dc9b5a","sequence_length":2304,"sequence_token_ids_sha256":"5f2026b82faac8988768fc7330d60a392741c3a5f7b364d0b87bf8d596e74bdb","source_content_sha256":"12b0ce7832facf193492ad1f568ff0a0b25bc5e52381a9690df089bc8a9b1db8","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0244917a01282995b95e9ac9304e280f4253bfbab1626b6c1c57bf69a558b519","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/24903","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"d50289a8f2b5eec9977321d118785e94090fabd5a2f4938b5e6f1c63045b237e","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"ee472d2e8db03638388106e721863599f643316a3422a5245d0628a1edb69010","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"4b9781c37044a00962c0018332d1f3f966c9e3b9ff49f267b124cbe927314707"},"formatted_content_sha256":"32ac15efb7073af0ae11d3032d1da75babff6e7bd6406334879047edbd6e821a","generator_receipt_sha256":null,"identity_record_sha256":"b26e48a4638561a3f7f57dc2f6915accfeabce22cb26f87ddd65919ddf7a653f","prompt_token_ids_sha256":"7b786394510a818db1c521f6d71eb3b4052359b0d28d0a57d166057d80326905","ruler_category":null,"seed":null,"selection_rank":5,"selection_sha256":"001280ff6af9eba470ab76ff48bc2983cffe1995efb051d68cffd40cbaac4d3e","sequence_length":2304,"sequence_token_ids_sha256":"7b786394510a818db1c521f6d71eb3b4052359b0d28d0a57d166057d80326905","source_content_sha256":"14074cbd779ec8e06e0e5e91c6c6e6c02ebe2d185bb5310ab0dfdf64230019d7","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"67b1ede620a9bdb86c619e3124eef21332be86df3795cb4169fdf93c1bb4c46c","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/10609","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"51f6c6102f8fa772b2f8e1665ab91ad76a5f4f11205009bb3a8af2e9c9d4c06e","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"3d06c4debf0e8c528b85d65d243bbe561bcd13afdcb83d09b2d604fed1853965","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"9bdcf5cea67426ee4ade841fed2e6833ccbede63b7b5497449fa2279f5e54182"},"formatted_content_sha256":"8639254a29be3154bbc110c673738678316be09fe6f456878dac249bdeb1f4e0","generator_receipt_sha256":null,"identity_record_sha256":"a21cb3bd2e34a22b91569c2942d19d8bafaed2f11827ee9c58af3cbe366d479a","prompt_token_ids_sha256":"0697f0eea8df7d85bbeb6a688ab15d4294212694298ccd864b429ddac3ed7a4e","ruler_category":null,"seed":null,"selection_rank":6,"selection_sha256":"00131239a626272348186654735088a9b4c813c3bd83827262f7891e6056d43f","sequence_length":2304,"sequence_token_ids_sha256":"0697f0eea8df7d85bbeb6a688ab15d4294212694298ccd864b429ddac3ed7a4e","source_content_sha256":"49ff1f749aab76c1df7c47e6c79fdee9dd3fd191d073320f0ab1f5f361059fc3","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0d6376f0009c55e4f4efbf0d650cfe974856c049c8b14c303c991dd3f6c05c19","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/39447","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"7914f39477c00d3c1516c2e43c441114314ef252fe82526ad293b346ab359284","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"7d6865c318074116ceae5d7735c27fed5066ae326d49cbc3280f9f0289cfb7cc","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"ec95c049c76add2ddc5fa9d5301e2bddb64906cc8e42e8438a5c2aeeffb46499"},"formatted_content_sha256":"033b6ca5d9572a314560961826373bc34657c08759a1919296e81a4cb84c6471","generator_receipt_sha256":null,"identity_record_sha256":"f59cda6a20ec53af40493bf6be8ad0f898023c32531b81fe0ff0fb3b13340ea9","prompt_token_ids_sha256":"81890a7b8319487c5f4c312ca211202afd2d039e46e4931b65e09c6e446fa634","ruler_category":null,"seed":null,"selection_rank":7,"selection_sha256":"00166693ce38f4c5185c40c1258a1a602a290506112c351fa067daa039c2d960","sequence_length":2304,"sequence_token_ids_sha256":"81890a7b8319487c5f4c312ca211202afd2d039e46e4931b65e09c6e446fa634","source_content_sha256":"2d19f7657219986df6eb9bcf188544ab7660f95cb5521668082b48da38ac52bf","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"6f5f79cc3c1180f2cf00501aeafbc8d0f1dd6e6e7ccc69a8648c032e3469df45","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/33622","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"9b4c1f8e6fd129beb4bc474e6514815a95af657b4e2bc5fe7a7f32362dfb40bb","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"dcd27f6003c74d9c8a85711934161ee0f2636d2f0d27c4137e156f0d5064649e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"536e8bc4cb2331a406028d335233022e273cb3c563ac84e3db83762692164b14"},"formatted_content_sha256":"de944eeb38c1a6f2dfea1cb1657adb820f3d66ebe3aa361c72a39823aad3fa73","generator_receipt_sha256":null,"identity_record_sha256":"344ad881a15a994684e1ad39c738fc64b472325c7c46c63c837f255455d9f5e0","prompt_token_ids_sha256":"93be74d13c5df9550e8923cb9af9086891766e59060bc9997c00a1c14458302a","ruler_category":null,"seed":null,"selection_rank":8,"selection_sha256":"00191187cdc6cab1097f558bd88f30baeb7d1f08e9eaffd47828d8a1345ed53b","sequence_length":2304,"sequence_token_ids_sha256":"93be74d13c5df9550e8923cb9af9086891766e59060bc9997c00a1c14458302a","source_content_sha256":"dcf794cb44d57515b528594025b879dcacdc8e06e892b1a0e62d26319aa8d8d7","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"f602f879f02588bab48170a79cd2e407d35c19e5a4e05337a2615b62c1d0c64f","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/28065","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"3ab0b82eda5a33b4272654c9de3e352bda591b27e5499e28d3642e5a138ec7f8","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"f29080ea5125a83d114344a4897dcb5e6cf6135a3d28a1b55374cdb2836dd1db","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"34dd2e61b1ba68a0bd87b1f9e8650c6869d66f35aa98c549feab516d17c3afd8"},"formatted_content_sha256":"f500fe5cd15c60256f64965b23d5784b6badce6cc0da7c4792ac3017524b173a","generator_receipt_sha256":null,"identity_record_sha256":"73be35853dc6427eeab216a511cbdeba7dbfbab63feb81bd2642b1d99bbe5f6e","prompt_token_ids_sha256":"1c701cd7ff6691ab6d74de0874349dc37eac3d7dd6450cdb43b876a5aa7ea76b","ruler_category":null,"seed":null,"selection_rank":9,"selection_sha256":"001ac4d7e72067b598bc36c7582913c54df342db338498dd73a106eec9c8ffb6","sequence_length":2304,"sequence_token_ids_sha256":"1c701cd7ff6691ab6d74de0874349dc37eac3d7dd6450cdb43b876a5aa7ea76b","source_content_sha256":"0beeb0285771e4bc2cee42c31f1b35d1570cda724294b426b193cdac54b83393","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0d2186f26771b127e5a2350edc81554ef21f6eb9da3e69d77f7377a307b8a2ec","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/1253","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"4db1f2865c58e3a6faa8055be8d46b6c3f57be277f07fb5e39670a879cc599df","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"e20502727d9945c19d9f78c314ab8a1be19622e5a4316dd86e9823225d38135d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"be880e0cd3b16a5b9d737b8b2b7cd71353c9aad69f7dd469e024c5803d69f411"},"formatted_content_sha256":"a6e5b442be8b8fd1b45487216940754b214f379ae9a2021d30f82598049ca5f2","generator_receipt_sha256":null,"identity_record_sha256":"02492d4c2ba2a4f9435fd461ed91ea404250adb616636b5c5f6930fc4c350595","prompt_token_ids_sha256":"c55c7896671653ced092d5fc45ea5527e9457780b57375667e77dd586f161da4","ruler_category":null,"seed":null,"selection_rank":10,"selection_sha256":"00223684c9261647b075a573c94999d1d273100f31524ec62c5e6c73b4787787","sequence_length":2304,"sequence_token_ids_sha256":"c55c7896671653ced092d5fc45ea5527e9457780b57375667e77dd586f161da4","source_content_sha256":"6d1b9548166f4f9d82806a3af5ceb3ef3a1f644c2c946deb8aa86ec78d2c2fe5","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"2608f932273743963113f1c10bf0e1c74703b9252284d4ab16394eea6ee00a1e","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/17774","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"ea92971f2ed12679772d304c1af6ce6e5d20a0a07f9c6e29ab3b8b95eb0c9c7a","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"de2670faad91b518e2bdc4f93edd92e3cf3b6d9e12fe24803a9dbb2d6ee7d8c3","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"d1ca5e91572227a6a1cdbeac2c3e0023a33d11172238246640c98d613aeb61a8"},"formatted_content_sha256":"2a4bbc348f9010bdfc1eec2d9ac77a487ad6f51468cfdedc35c10316e95564da","generator_receipt_sha256":null,"identity_record_sha256":"aba94b34901cf76209e4f0517738794062a4e8978f75d8d8e46bd007b32f2d61","prompt_token_ids_sha256":"25656de4135fa58df4d27bf5e1bb9122210cd21754c03b8b5f83f9412a6e2f5e","ruler_category":null,"seed":null,"selection_rank":11,"selection_sha256":"002494cd7b130c2e8101df6704ee2628630640a6ea6d8c21e9a0419c810d8f25","sequence_length":2304,"sequence_token_ids_sha256":"25656de4135fa58df4d27bf5e1bb9122210cd21754c03b8b5f83f9412a6e2f5e","source_content_sha256":"c71f6b5999e91ecf1857fb9dfb3f2573827e4ae4c5c3c4af2b08e3ba6a349e07","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"112223ea4e8c7a17eec0768b0bef76c663105f97785440668c062c5080b3859e","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/26008","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"2eb3fc663ac2fcca42c750d5cba63ad3d0d118921b38a48382c1cac4734faa8b","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"ecda06dc336cf17705f48dcde65a60f0b7ff432af083c07918dd0f1dbf09d79c","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"e7eb1385302da9031a1227b768aaabecafafdb5614b3a0e2873e45ccfa3f7325"},"formatted_content_sha256":"fb608858dedf19c2f0a8e14d8710aa1d1c393be5a72ac8554731f0af33714904","generator_receipt_sha256":null,"identity_record_sha256":"af695a09abd9495d4e87989cbe7b352aebdfea88272863b3cade1adbd9d9c1a1","prompt_token_ids_sha256":"8eac346c569d9163ec8a9bde2355446690ca3c72a9a9f5c9bc00b745b6648f63","ruler_category":null,"seed":null,"selection_rank":12,"selection_sha256":"0027603c9d271c580c43673b3431be3adf061650ddff27e01b7869ea18673d9b","sequence_length":2304,"sequence_token_ids_sha256":"8eac346c569d9163ec8a9bde2355446690ca3c72a9a9f5c9bc00b745b6648f63","source_content_sha256":"469e817fab0a0502150988f96b4b16d5c4de03fb7f7bd185e1cff0715aee4b26","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"c447c8e41acc4393a51daf3ef28ed95971628266812cc78c05b887e341a38f02","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/38160","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"849f6474e4eb97b56439c66b9aaf2ac1fca275e49c29e33c48b2de0e806d1193","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"d11cb3a40f35a5034111c7bbbf1a8ff895d15396fea10ddb0cdbd97e8a47a5bb","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"ae53490b8b58ad6da63a300819db55d4a14e7b1bf25c07bbcd54b06469cc8b62"},"formatted_content_sha256":"b87156fd6d95b79ffe144b1b7448682cc84e8c2d06709db21214d82439b58315","generator_receipt_sha256":null,"identity_record_sha256":"f629ef55f22acb269a1825e56238e5770bf1bffc327d1d71d03370911d984632","prompt_token_ids_sha256":"55fb1408407a280bc2c65e78161f5cefb0ceaa4ceb04073817e0973d6a982b53","ruler_category":null,"seed":null,"selection_rank":13,"selection_sha256":"002a54c1b2c80d534795a77a68dfa9f3411107c3896ba33d31a868339d813293","sequence_length":2304,"sequence_token_ids_sha256":"55fb1408407a280bc2c65e78161f5cefb0ceaa4ceb04073817e0973d6a982b53","source_content_sha256":"f59885bfc6d88898f97c7bda7625006b3be0651da5929a1e272907e85a9772e3","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"14e443d1ba9ca076a4a9d065c90f0b3ccb2c24defaced1591fe7cc77f71e7527","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/2773","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"5a0b47a49833f2e6114f9b3e2d475b69a889ed7945db09507580c9536ef76448","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"f0dbd8116db99645f9271fcd040a856c59b111c6243748f64ddc1422928c64ff","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"78a78a32fdab2f986b289b32d0a4612fb33ea2014dccbbce3a60d65c16e765ac"},"formatted_content_sha256":"68c11a24ba674cdac7966cbaf0f71a9d2fc30a62a0d90a649a6554907ef00125","generator_receipt_sha256":null,"identity_record_sha256":"ddbf6728a3fc2c07defaa4546c34ea401f4fd05c829819301903562cd04a9519","prompt_token_ids_sha256":"8b935a7676a8137367066b0b1af7a39f635e0e3592fe3c1982a28bac24eb96cc","ruler_category":null,"seed":null,"selection_rank":14,"selection_sha256":"002d0ca3ff81dfc79dea1f54c4f28547f367149283e13dc83a14c114322353bd","sequence_length":2304,"sequence_token_ids_sha256":"8b935a7676a8137367066b0b1af7a39f635e0e3592fe3c1982a28bac24eb96cc","source_content_sha256":"be1775efe2cd9be13c65cbd65200b0cd27ae7d29b52ff493b27ed85e4b549c61","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"1fb610bd6210663f10a294d4b45df223b7ee4e3b6a8c0de390f3e50612ea1219","anchor_positions":[143,287,431,575,719,863,1007,1151,1295,1439,1583,1727,1871,2015,2159,2303],"anchor_positions_sha256":"3342b86eef5820a4c514553962a44f68effb99e6ea26db09286665d5c7b389a1","canonical_id":"http://www.gutenberg.org/ebooks/1219","config":"default","configured_length":null,"family":"pg19","fisher_boundary":{"boundary_positions":[142,286,430,574,718,862,1006,1150,1293,1437,1581,1725,1869,2013,2157,2301],"fisher_boundary_sha256":"92e822869a8cdca6f8c09e9495406404c8a345c0a99d06dc98a948b7bcd010d6","horizon":1,"input_positions":[143,287,431,575,719,863,1007,1151,1294,1438,1582,1726,1870,2014,2158,2302],"input_token_ids_sha256":"7ea3f80cfbc818f9210b87b0153e90af5720869a1fe3f26b0d5f723bfa9eead2","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[144,288,432,576,720,864,1008,1152,1295,1439,1583,1727,1871,2015,2159,2303],"target_token_ids_sha256":"c1725ca6a384a182c8f164ec35f4faded719127f4cef2baec3d812545eff3e6f"},"formatted_content_sha256":"cd30711364e668a3275113e8a3f92ecee4b268935f5a095cdb0f0c1f11205c1c","generator_receipt_sha256":null,"identity_record_sha256":"2d205b87fc59eabeed0bda29df3e0789eef230026c46ad11796d41b866d18b2e","prompt_token_ids_sha256":"65e443512311bce75a506b694195aa105b7831541ecb92c1768fe5a0ce285ef2","ruler_category":null,"seed":null,"selection_rank":15,"selection_sha256":"0032c753604107e647b4ccd7ef877562bc0ca9c0e232a26a761388befd002fea","sequence_length":2304,"sequence_token_ids_sha256":"65e443512311bce75a506b694195aa105b7831541ecb92c1768fe5a0ce285ef2","source_content_sha256":"6cb2a106560ad166c025fb7a83708e519f95bca68cecb9a7bff61e261a0b8384","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2304,"cache_exposed_stop":2304,"prefill_start":0,"prefill_stop":2304,"scored_start":2304,"scored_stop":2304},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"52b78dddc7c06fdc5e5ab3937802742ef3b3414f9743427d2bd71994c5cb1671","anchor_positions":[113,227,341,456,570,684,798,913,1027,1141,1255,1370,1484,1598,1712,1827],"anchor_positions_sha256":"9289ece2ebec2ce8ec20865a50beecbecea8ea6a3dc4b5c5c8ed9f6ad0d784ac","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multikey_2:length=2048:seed=12340:sample=0","config":"niah_multikey_2","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[113,227,341,455,569,683,797,912,1026,1140,1254,1368,1482,1596,1710,1825],"fisher_boundary_sha256":"a9016c04cfbd31eb6f72106eb0693c1545bbaa0451e228e6e447a7814d921eb7","horizon":1,"input_positions":[114,228,342,456,570,684,798,913,1027,1141,1255,1369,1483,1597,1711,1826],"input_token_ids_sha256":"dfd87b8cdcdc45c3aed78576c14b5dd53c5f3a8de7a4a69358857943c1f8ba1a","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[115,229,343,457,571,685,799,914,1028,1142,1256,1370,1484,1598,1712,1827],"target_token_ids_sha256":"4ed30d7e42a7866174c94f532bdcc6947a15ad58245c18f3aba71a4612f05990"},"formatted_content_sha256":"d1e07461677a5fe92f27926505552cc8a343cd60a5b5be86c2fa00090fae27f0","generator_receipt_sha256":"801e44bdb609773d985e8fd74f1d853f1afb30cf0a66d2cee0f7013ae8749c9c","identity_record_sha256":"a1cb4798036f5d2c63f1da08e09141e23d4fdec224e854eeb2d770ac8ba11721","prompt_token_ids_sha256":"793e38d71d7e91af366df94fb359ffb203e20cafb77f7afbb412db3ac5cfc5d2","ruler_category":"retrieval","seed":12340,"selection_rank":0,"selection_sha256":"26538bdeb4a99f8904324b26add0826a13744b51ec94fd9a004c72a2ba68c287","sequence_length":1828,"sequence_token_ids_sha256":"793e38d71d7e91af366df94fb359ffb203e20cafb77f7afbb412db3ac5cfc5d2","source_content_sha256":"801e44bdb609773d985e8fd74f1d853f1afb30cf0a66d2cee0f7013ae8749c9c","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1828,"cache_exposed_stop":1828,"prefill_start":0,"prefill_stop":1828,"scored_start":1828,"scored_stop":1828},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"75123434e18fce7bfe0e4b995fd12769a05a08d35fc315c68b8438e286e23f77","anchor_positions":[246,494,742,990,1237,1485,1733,1981,2228,2476,2724,2972,3219,3467,3715,3963],"anchor_positions_sha256":"dff504ff5227ef8cb24123e368d13dfb8cef71f15a3e715472122831820e3e93","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multivalue:length=4096:seed=12340:sample=0","config":"niah_multivalue","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[246,494,741,989,1237,1484,1732,1980,2227,2475,2722,2970,3218,3465,3713,3961],"fisher_boundary_sha256":"b36b4cfcb1258559a6eeafb235f1efb9a3bc5d2d3ec31389b39290f7e7209d82","horizon":1,"input_positions":[247,495,742,990,1238,1485,1733,1981,2228,2476,2723,2971,3219,3466,3714,3962],"input_token_ids_sha256":"dd9ed3a5d8b1bac29e9c251660e29d7a61b1d0cbb9846d333768dd85128cd708","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[248,496,743,991,1239,1486,1734,1982,2229,2477,2724,2972,3220,3467,3715,3963],"target_token_ids_sha256":"80f466fa56b4c9b1c24b8b51bb8cacb08c7290a7b0f1fd9ceeb119534231b14d"},"formatted_content_sha256":"f6bd8b2f8c5eb25a8ea8e08f2f4fb8e581a8d2080ff40194bce0a86f5deccc0e","generator_receipt_sha256":"fc75f4bdae8929325589536065fb7c8f5bb1fd737001da616d339365aa94920a","identity_record_sha256":"8091f9201ab1213aa14972869be942841763c3897bee5b30b77fb3d34877dd86","prompt_token_ids_sha256":"d498355b8380354e768abc57a79afe8cc73c5849b98f01081fd945329db5e77a","ruler_category":"retrieval","seed":12340,"selection_rank":1,"selection_sha256":"34e79c93add91edaaf2f631ffbc9e4c5350f61dc96769cf971f03ae113ccd1b5","sequence_length":3964,"sequence_token_ids_sha256":"d498355b8380354e768abc57a79afe8cc73c5849b98f01081fd945329db5e77a","source_content_sha256":"fc75f4bdae8929325589536065fb7c8f5bb1fd737001da616d339365aa94920a","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":3964,"cache_exposed_stop":3964,"prefill_start":0,"prefill_stop":3964,"scored_start":3964,"scored_stop":3964},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"532ac060a0cb86e868a3d0d5f0e7217475df142f3480655a5f9cc923addbf97e","anchor_positions":[124,250,376,501,627,753,878,1004,1130,1255,1381,1507,1632,1758,1884,2010],"anchor_positions_sha256":"baf07dbe6e81bfe17433570a4cb75b5ed267ca19947bc93119fe6c078f612806","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=2048:seed=12339:sample=0","config":"vt","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[124,250,375,501,626,752,877,1003,1129,1254,1380,1505,1631,1756,1882,2008],"fisher_boundary_sha256":"9a1bd7b90a51362eb876693f97e6c20ec0fa6bff1077ed5acd1237a823804b3a","horizon":1,"input_positions":[125,251,376,502,627,753,878,1004,1130,1255,1381,1506,1632,1757,1883,2009],"input_token_ids_sha256":"d43a3eb571f5bfc8ea7dc04191b8336f22f066f154918abe65e057791d13a43e","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[126,252,377,503,628,754,879,1005,1131,1256,1382,1507,1633,1758,1884,2010],"target_token_ids_sha256":"453114002e40922d3386b14ea8e4915c4aa872297dae554dfbf91fe2f3b3e9ec"},"formatted_content_sha256":"221e10f0973c46ef1f20a688046e4bffae24656998c1059a831d7b20726fede0","generator_receipt_sha256":"b9a9b27500644e9aa2532304d577b85ab988c4359dd5edf7630fc8a8bee4b60e","identity_record_sha256":"3c519c0d4df294b9bb4d57dfaf44f3fe4934ae8e697dc924d17dbf39644026e8","prompt_token_ids_sha256":"cd95e7b2f95f4b705b1b43dc6b201b8307b182fe379566cb2e900ef21733b21a","ruler_category":"multi_hop_tracing","seed":12339,"selection_rank":2,"selection_sha256":"48145a1ddcbd5ae2492d753c9ffdf69047346dcba3d4c88e131c036be6968d9e","sequence_length":2011,"sequence_token_ids_sha256":"cd95e7b2f95f4b705b1b43dc6b201b8307b182fe379566cb2e900ef21733b21a","source_content_sha256":"b9a9b27500644e9aa2532304d577b85ab988c4359dd5edf7630fc8a8bee4b60e","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2011,"cache_exposed_stop":2011,"prefill_start":0,"prefill_stop":2011,"scored_start":2011,"scored_stop":2011},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"deb02a9e929ebd4d409deb018ab87063b47401d9993356ae1689a17e44d9121f","anchor_positions":[248,497,746,995,1244,1494,1743,1992,2241,2490,2740,2989,3238,3487,3736,3986],"anchor_positions_sha256":"a5ad1bc8d8afa780351af0b99deecbf3b31c05390a133ecb9438c32584b13b51","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:fwe:length=4096:seed=12339:sample=0","config":"fwe","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[248,497,746,995,1244,1493,1742,1991,2240,2489,2738,2987,3236,3485,3734,3984],"fisher_boundary_sha256":"3ce3ab1d00aff5ae9a8f93284d14aaa6f1744a79acc88d0772877941983042b6","horizon":1,"input_positions":[249,498,747,996,1245,1494,1743,1992,2241,2490,2739,2988,3237,3486,3735,3985],"input_token_ids_sha256":"87a331f7d1b5d5467dcce6012238bb065bb86881942f6c071315f3911a04e910","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[250,499,748,997,1246,1495,1744,1993,2242,2491,2740,2989,3238,3487,3736,3986],"target_token_ids_sha256":"0c3be397f087a6670d953bee273680288fc932be2e83d55d3086f5f6090398bb"},"formatted_content_sha256":"0fead2f4a48fe6c038641241dc6133e35cf0290088d4b06bf388bbf505180d9d","generator_receipt_sha256":"7fe1d0a912ee13f2dc1aae6839a7f8e87a623d5df30a3408ac5c6059f88aed13","identity_record_sha256":"bef04af160238768f27800f450d76a76013d80a3c96282a8a684900ba5807d4d","prompt_token_ids_sha256":"1610d294ca00f9745a960e213dd10784d8a523eca88bdaded2c7bde663d09a1c","ruler_category":"aggregation","seed":12339,"selection_rank":3,"selection_sha256":"51065012a292ff5d1577ac8988306f3ebc15a9c955ac9c104f135c18f7ed7cc9","sequence_length":3987,"sequence_token_ids_sha256":"1610d294ca00f9745a960e213dd10784d8a523eca88bdaded2c7bde663d09a1c","source_content_sha256":"7fe1d0a912ee13f2dc1aae6839a7f8e87a623d5df30a3408ac5c6059f88aed13","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":3987,"cache_exposed_stop":3987,"prefill_start":0,"prefill_stop":3987,"scored_start":3987,"scored_stop":3987},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"8a6b31f4f399feceed15eefe68a896dcd67ad7e19c3225837e50ada41fc0a847","anchor_positions":[252,506,759,1013,1266,1520,1773,2027,2280,2534,2787,3041,3294,3548,3801,4055],"anchor_positions_sha256":"e3a8a2871a86f2cf789e76ba6df6ede1df61047b14b69480ff1217c0a911147f","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=4096:seed=12339:sample=0","config":"vt","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[252,505,759,1012,1265,1519,1772,2026,2279,2532,2786,3039,3292,3546,3799,4053],"fisher_boundary_sha256":"159f8ad58c26093c2f75744da0f9502f7a2906b033bd5fce2665cf525fefb38b","horizon":1,"input_positions":[253,506,760,1013,1266,1520,1773,2027,2280,2533,2787,3040,3293,3547,3800,4054],"input_token_ids_sha256":"e8a2ff60bb805b206d15a2e0bf12885fedc537e7082f756e2c445a6bf70c326d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[254,507,761,1014,1267,1521,1774,2028,2281,2534,2788,3041,3294,3548,3801,4055],"target_token_ids_sha256":"49fc01abb0da9ff437c1e5a9aefc9158a1f8db110f275f2946581b8be9491118"},"formatted_content_sha256":"946e2f3a8214ecc488a65888ebeb1dc5b42d3027357523816a5ce8d3414b899c","generator_receipt_sha256":"8ea9a21c91b1b5203697a120d21824bea8dc810455d228e58a7ae319dc72dd82","identity_record_sha256":"5556eeb12c7c154e817d9c46859eaeb64f6a54512b787f1b1f38bf9fcf284542","prompt_token_ids_sha256":"9973907d38c28c89e9a53bec04ecb4c695cc4c94f35050f705a22d2e6ab2c897","ruler_category":"multi_hop_tracing","seed":12339,"selection_rank":4,"selection_sha256":"5c5a0c61d80b5c3a76e5bbf1062b35594e4a4995d482c429634e381eaaadcf21","sequence_length":4056,"sequence_token_ids_sha256":"9973907d38c28c89e9a53bec04ecb4c695cc4c94f35050f705a22d2e6ab2c897","source_content_sha256":"8ea9a21c91b1b5203697a120d21824bea8dc810455d228e58a7ae319dc72dd82","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":4056,"cache_exposed_stop":4056,"prefill_start":0,"prefill_stop":4056,"scored_start":4056,"scored_stop":4056},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"e6965757239ac98f811d9377515f2e5110de3926f0390ca3416c9f2e3ef74bd3","anchor_positions":[79,160,240,321,401,482,562,643,724,804,885,965,1046,1126,1207,1288],"anchor_positions_sha256":"a02a7b889641d261dbd8927a54b0e9bda873159838da4973760162d67a2adbcd","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_multiquery:length=2048:seed=12339:sample=0","config":"niah_multiquery","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[79,159,240,320,401,481,562,642,722,803,883,964,1044,1125,1205,1286],"fisher_boundary_sha256":"57d7fe986325f9a24836dcf6f4afdba3cd327289b6aa8077b19ec4aaff566ce9","horizon":1,"input_positions":[80,160,241,321,402,482,563,643,723,804,884,965,1045,1126,1206,1287],"input_token_ids_sha256":"0a50d970537d979d05cfd20781db749fecbfb1eb42f1426875d990cadb4a07ee","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[81,161,242,322,403,483,564,644,724,805,885,966,1046,1127,1207,1288],"target_token_ids_sha256":"88866037b9a7485b03068ce42d898b0e5484251bc087329d421a95fea8993da1"},"formatted_content_sha256":"bfb4cc6fdedde1a2578a09daf8fa4e779725a810722080b59de46588e1d5ab2b","generator_receipt_sha256":"0e8034964c18899e91a435a5737bfa29f7791fc41c29f42e4f2d9d75fa3a98ff","identity_record_sha256":"6bb7156902c5f36fc7c3e3c92666c510e1a2d572d3c6c557cc98ab440dd94db2","prompt_token_ids_sha256":"c68672f5e746622c7ef9c8d45f9312057365f04182c7af0576723d8089244ca1","ruler_category":"retrieval","seed":12339,"selection_rank":5,"selection_sha256":"65f813250e251ff50bf38bf6b227abf56f6e43ea5dedef6b18e6cf94874e510d","sequence_length":1289,"sequence_token_ids_sha256":"c68672f5e746622c7ef9c8d45f9312057365f04182c7af0576723d8089244ca1","source_content_sha256":"0e8034964c18899e91a435a5737bfa29f7791fc41c29f42e4f2d9d75fa3a98ff","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1289,"cache_exposed_stop":1289,"prefill_start":0,"prefill_stop":1289,"scored_start":1289,"scored_stop":1289},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"128a207210316772af37d86a91c97448a5eec69418db26c573a15daa57fff7db","anchor_positions":[252,506,759,1013,1266,1520,1773,2027,2281,2534,2788,3041,3295,3548,3802,4056],"anchor_positions_sha256":"ab52ebab779359ace36255ddbc6dfccb71733c90923120903537310d833e1be5","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=4096:seed=12340:sample=0","config":"vt","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[252,505,759,1012,1266,1519,1773,2026,2279,2533,2786,3040,3293,3547,3800,4054],"fisher_boundary_sha256":"c02daef8e0847ca8ef8f6795ed064d750cbfb33070533ceaf84d3f199f95bdf4","horizon":1,"input_positions":[253,506,760,1013,1267,1520,1774,2027,2280,2534,2787,3041,3294,3548,3801,4055],"input_token_ids_sha256":"758c575c40dc855f729daac2fb4c427f22776d59517e67e7dcd8ad13b9015cd3","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[254,507,761,1014,1268,1521,1775,2028,2281,2535,2788,3042,3295,3549,3802,4056],"target_token_ids_sha256":"e7ee891d8494cf6174e6a03c1da26e912c52473297bcba994663141abda1e73e"},"formatted_content_sha256":"84db9b2d31c7c5cfb2df3227d24ed8fa03d50cc474dbaf560e4f7aee2e52bb97","generator_receipt_sha256":"5b6cd316a368db4420845994f9423aaeedc0fd75802aba9eafb7deb4f4eddc73","identity_record_sha256":"ce937620a8ea8bbf5af1e1b522055853a1ec44ac92520d74486cc8e8f0bae348","prompt_token_ids_sha256":"42167ea36f4f43f21c1eadf1bd9fb391cb7d16904f553e91eac1401c360e902c","ruler_category":"multi_hop_tracing","seed":12340,"selection_rank":6,"selection_sha256":"79c3cb58b42a4b44fb604b28cd22ae1aa71bc49e05a3eb8399ca698c0044ee51","sequence_length":4057,"sequence_token_ids_sha256":"42167ea36f4f43f21c1eadf1bd9fb391cb7d16904f553e91eac1401c360e902c","source_content_sha256":"5b6cd316a368db4420845994f9423aaeedc0fd75802aba9eafb7deb4f4eddc73","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":4057,"cache_exposed_stop":4057,"prefill_start":0,"prefill_stop":4057,"scored_start":4057,"scored_stop":4057},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"0fe15004ffd8e39b3cd00a623cb86c9ae2e0570216811c075442338f6c8c6963","anchor_positions":[141,284,426,569,711,854,996,1139,1282,1424,1567,1709,1852,1994,2137,2280],"anchor_positions_sha256":"9ea473fa5cae3aef9908a7ca4fa168866d469f3650f18f07e5dce4ee02714d7c","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_2:length=4096:seed=12340:sample=0","config":"qa_2","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[141,283,426,568,711,853,996,1138,1280,1423,1565,1708,1850,1993,2135,2278],"fisher_boundary_sha256":"12c58ae371c707d325c0b7db5868000c046d213a4b92b2344f6482266e812bec","horizon":1,"input_positions":[142,284,427,569,712,854,997,1139,1281,1424,1566,1709,1851,1994,2136,2279],"input_token_ids_sha256":"e6b8490ddd4e543ee4bbe0a2096b3a34397a5c6e4ee16c9efb438fba25a387d5","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[143,285,428,570,713,855,998,1140,1282,1425,1567,1710,1852,1995,2137,2280],"target_token_ids_sha256":"644343be1688f80c9846724beff4e6d47423e4ced93613951f85b480fb1aa6bf"},"formatted_content_sha256":"1d3faed8d004df01dbd7309b8dd1ed0e326c1bd1c5fa54306abe22855487372d","generator_receipt_sha256":"fba640bf322cb8c1f6175cc1a73e6267b18618e8555d9601e202268427ebabe8","identity_record_sha256":"1e32be0f57d7bfa1580c73b4f1c5e8a06ff8f429ff4611e766534940463ad32e","prompt_token_ids_sha256":"dab144649219fd1b15323997ba14f284492db0e725562192c789c0f766c8de3c","ruler_category":"question_answering","seed":12340,"selection_rank":7,"selection_sha256":"7a527495d7e22511d3058e2b2a4a846cd8b244d93a021ee51622ee975c2530ea","sequence_length":2281,"sequence_token_ids_sha256":"dab144649219fd1b15323997ba14f284492db0e725562192c789c0f766c8de3c","source_content_sha256":"fba640bf322cb8c1f6175cc1a73e6267b18618e8555d9601e202268427ebabe8","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2281,"cache_exposed_stop":2281,"prefill_start":0,"prefill_stop":2281,"scored_start":2281,"scored_stop":2281},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"4a35e619dc657f107bad1efd0356762a6276d17f1c44fe7c843454ce15350f14","anchor_positions":[84,170,255,341,426,512,597,683,768,854,939,1025,1110,1196,1281,1367],"anchor_positions_sha256":"52762b5ae990ebf341f9864370415ddc0be98c34b1415f0fb53c8f4bb5237780","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_1:length=2048:seed=12339:sample=0","config":"qa_1","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[84,169,255,340,425,511,596,682,767,852,938,1023,1108,1194,1279,1365],"fisher_boundary_sha256":"71d2d40de65a4fd2d9d9e691302e52d3a9fc059b1629b02c3b6543f8201e463f","horizon":1,"input_positions":[85,170,256,341,426,512,597,683,768,853,939,1024,1109,1195,1280,1366],"input_token_ids_sha256":"673327b5691ed16299c8dd34b55f88b87d1fe642535708588295ca0f00f89422","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[86,171,257,342,427,513,598,684,769,854,940,1025,1110,1196,1281,1367],"target_token_ids_sha256":"aefebb06df48a7ba94a6d5de230228ea289d46553d344edc66924e98dea4b336"},"formatted_content_sha256":"9463b4927878ab07df8a32bd0b0584b76f5d42182dc21d90193275757cadc022","generator_receipt_sha256":"8d5f10bad274c66452a465f737da20099210de1d161c27a36c6331783da5f4f2","identity_record_sha256":"9198d8dbcec2c86f25636e37bc394d9d761cea2bb93559b1786b527f69e7fd7c","prompt_token_ids_sha256":"1102e50da7352084f75b9fa23ab6cc2d70708530625fc296126f13ff696d77e9","ruler_category":"question_answering","seed":12339,"selection_rank":8,"selection_sha256":"8bfbb4cad87302a003026059836508ab6e2d89448865d0217704654cd5b9bc8e","sequence_length":1368,"sequence_token_ids_sha256":"1102e50da7352084f75b9fa23ab6cc2d70708530625fc296126f13ff696d77e9","source_content_sha256":"8d5f10bad274c66452a465f737da20099210de1d161c27a36c6331783da5f4f2","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1368,"cache_exposed_stop":1368,"prefill_start":0,"prefill_stop":1368,"scored_start":1368,"scored_stop":1368},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"56eb209f76f5b84c51f3a6dfa390051c3e6179a589094e42a630548e33c79ab6","anchor_positions":[117,235,353,472,590,708,827,945,1063,1182,1300,1418,1537,1655,1773,1892],"anchor_positions_sha256":"0f267c5222d70787fd8238bd3de2e5bcbdfa19d30f6519cbc9abe6eb49aff8f8","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_2:length=2048:seed=12340:sample=0","config":"qa_2","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[117,235,353,471,589,708,826,944,1062,1180,1299,1417,1535,1653,1771,1890],"fisher_boundary_sha256":"433027cb3aa35bbaaa74d9db1d137366f2b08b50e51181e868b602a2a55b6952","horizon":1,"input_positions":[118,236,354,472,590,709,827,945,1063,1181,1300,1418,1536,1654,1772,1891],"input_token_ids_sha256":"eef4188a93a14820c4b8529c64e375c3dc36f29c2314d289c81684b9e3d1db9d","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[119,237,355,473,591,710,828,946,1064,1182,1301,1419,1537,1655,1773,1892],"target_token_ids_sha256":"828ccb325b0d5b2cbe71be59f39c620de4093e298de9f0dda10d92926f2263fe"},"formatted_content_sha256":"696b7eb132f4a3d5f49052ac7680047c4a0a1ad1fc05f0bfa2bb25b4ddd67353","generator_receipt_sha256":"e58aa114cbdb8502b1dff17517a049edca17022eacfe90cd5afc0e2eda097c3e","identity_record_sha256":"67d8a257b752e3b5c57062870f7fde1fc0efbf2427ed7c5aaa4b1f1b94419ddc","prompt_token_ids_sha256":"ca3e9180e70700d36dc0d15f1ca4e7e4febee3a9058814b3f575e89f58f04daf","ruler_category":"question_answering","seed":12340,"selection_rank":9,"selection_sha256":"8fd992ea1ad818ee9cb171e5d9070ef59e24c95c1dfb3549cdc3963f904c96d3","sequence_length":1893,"sequence_token_ids_sha256":"ca3e9180e70700d36dc0d15f1ca4e7e4febee3a9058814b3f575e89f58f04daf","source_content_sha256":"e58aa114cbdb8502b1dff17517a049edca17022eacfe90cd5afc0e2eda097c3e","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1893,"cache_exposed_stop":1893,"prefill_start":0,"prefill_stop":1893,"scored_start":1893,"scored_stop":1893},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"cc7b46a7e50bc99f0cdb680c8631375304bb7983eedd6e6bec9dbbd2fbcfbdf5","anchor_positions":[251,504,756,1009,1261,1514,1766,2019,2271,2524,2776,3029,3281,3534,3786,4039],"anchor_positions_sha256":"7360c0b207ce24b236c422e4e664bff53fcafb7e524bd09aaebed6ea7c3ec7f8","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:question_answering:qa_1:length=4096:seed=12339:sample=0","config":"qa_1","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[251,503,756,1008,1260,1513,1765,2018,2270,2522,2775,3027,3279,3532,3784,4037],"fisher_boundary_sha256":"e551a79a205e6b53d8d809cc13da1c1d6f98faf520fcf9e40ed135aee7b9bb8e","horizon":1,"input_positions":[252,504,757,1009,1261,1514,1766,2019,2271,2523,2776,3028,3280,3533,3785,4038],"input_token_ids_sha256":"eb303aed866e00a6665d3e56e5ccdf9775fbd1e8a9bb7f5652c3fd7bd9707424","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[253,505,758,1010,1262,1515,1767,2020,2272,2524,2777,3029,3281,3534,3786,4039],"target_token_ids_sha256":"fdd7b7b8e4d53d09f5fa32c50c573deb2dbc0fe9474bed57416b8fac4ca54c1e"},"formatted_content_sha256":"9c7b681fe9e8d8f416aceaac08bdfd92d885e97dc140320d76ff0d9fefb803db","generator_receipt_sha256":"2dd908d09753092d8a358262f5073246919dd0069dfb0a60eca406aba764caa5","identity_record_sha256":"de00cb025c394a25eaa32bd222daf198e16413a112f3bccd5a72b628d593cd51","prompt_token_ids_sha256":"a089b295fbea580bdf8d050e97c37b242dfd30da2c3c7a3e620b76a54ed9c79f","ruler_category":"question_answering","seed":12339,"selection_rank":10,"selection_sha256":"a114fee36328bb536916462de6cc07e89d3dd7b3809c5295bc63e2663d7973f7","sequence_length":4040,"sequence_token_ids_sha256":"a089b295fbea580bdf8d050e97c37b242dfd30da2c3c7a3e620b76a54ed9c79f","source_content_sha256":"2dd908d09753092d8a358262f5073246919dd0069dfb0a60eca406aba764caa5","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":4040,"cache_exposed_stop":4040,"prefill_start":0,"prefill_stop":4040,"scored_start":4040,"scored_stop":4040},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"271a989874c4cc17ca1a739347bba23628a9e467e85d3219bb363f9ed7b07bf0","anchor_positions":[115,231,348,464,580,697,813,930,1046,1162,1279,1395,1511,1628,1744,1861],"anchor_positions_sha256":"cf6b395abd6929080fc35256e6179d2e3c657bc70c02f16093cb0533b0035bcc","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:cwe:length=2048:seed=12340:sample=0","config":"cwe","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[115,231,347,464,580,696,812,929,1045,1161,1277,1394,1510,1626,1742,1859],"fisher_boundary_sha256":"4015c686931b188bc2932b79e3bfa9ac30c8393959c5972c66fb8b82816a3bfd","horizon":1,"input_positions":[116,232,348,465,581,697,813,930,1046,1162,1278,1395,1511,1627,1743,1860],"input_token_ids_sha256":"7a7df1462084ce8ab589c85b605a1f4311c0ec91dc306a02673a84285b192131","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[117,233,349,466,582,698,814,931,1047,1163,1279,1396,1512,1628,1744,1861],"target_token_ids_sha256":"77743f96c0e111a7c8ad0cb12178be0abbe535b25edec949d8812bb2522c07a0"},"formatted_content_sha256":"3b5a450e8c283afa9103e3393bb8414b1d833a42c88cc5d7d4bd3f26a6120467","generator_receipt_sha256":"1c23f946536f8b05ad81d768edf4b1a5cc74ce39a3e9048124593635eb45ad57","identity_record_sha256":"21e8e431c773084606e1bb52807ebb59255d6a84e9bff884ebb333b51b0833f7","prompt_token_ids_sha256":"52bbc9b8942f395aaf5ad8a6b712b521a6a64d665cf085cb5fba76caccdf5b8f","ruler_category":"aggregation","seed":12340,"selection_rank":11,"selection_sha256":"a4f7cd62de6cc19b84e61c4bbedb2f095811b5b15e0045c070a67a003ff2110a","sequence_length":1862,"sequence_token_ids_sha256":"52bbc9b8942f395aaf5ad8a6b712b521a6a64d665cf085cb5fba76caccdf5b8f","source_content_sha256":"1c23f946536f8b05ad81d768edf4b1a5cc74ce39a3e9048124593635eb45ad57","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1862,"cache_exposed_stop":1862,"prefill_start":0,"prefill_stop":1862,"scored_start":1862,"scored_stop":1862},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"134c9730064b61e43d80673f0cb931058bd1b3199bca91ad233138c0226ce11c","anchor_positions":[120,242,364,486,608,730,852,974,1096,1218,1340,1462,1584,1706,1828,1950],"anchor_positions_sha256":"1e64ef366c8307e3a3410c30274175d81a4400831d9e53231d196d85e95f722d","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:fwe:length=2048:seed=12339:sample=0","config":"fwe","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[120,242,364,486,608,729,851,973,1095,1217,1338,1460,1582,1704,1826,1948],"fisher_boundary_sha256":"6f240a219eecca56c401dc3488a6be62114cf717fb5d3354698e9b9bd01cb16c","horizon":1,"input_positions":[121,243,365,487,609,730,852,974,1096,1218,1339,1461,1583,1705,1827,1949],"input_token_ids_sha256":"cb97fb7a28acb3307ed6ee8d8c153283de017a8ff009aa8791bb6bcb457ee5f6","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[122,244,366,488,610,731,853,975,1097,1219,1340,1462,1584,1706,1828,1950],"target_token_ids_sha256":"317374ba87916932cb7eb42091c9f2e9f86931a246150c2f199b1253ca4deba1"},"formatted_content_sha256":"6133db426215f641d319c7f7a251c50725f112b597bacfa38b5b397f559b9bc6","generator_receipt_sha256":"bb2518d8cd4ac6e41fa7aafc28a2951da762de64e8c2843d63b20f731b44ed69","identity_record_sha256":"fda1b358c53c26d14ec53f9f1307c3a894b7bd9b2265f34dc4835ad657df108d","prompt_token_ids_sha256":"5dc1507b9a08d033e3d20d61fa912a594f04959a9046d7a45a38e8828444ba30","ruler_category":"aggregation","seed":12339,"selection_rank":12,"selection_sha256":"a802c5ae87d368a5b0da4639128597266434ded4c54487021d47309dd1c7d305","sequence_length":1951,"sequence_token_ids_sha256":"5dc1507b9a08d033e3d20d61fa912a594f04959a9046d7a45a38e8828444ba30","source_content_sha256":"bb2518d8cd4ac6e41fa7aafc28a2951da762de64e8c2843d63b20f731b44ed69","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":1951,"cache_exposed_stop":1951,"prefill_start":0,"prefill_stop":1951,"scored_start":1951,"scored_stop":1951},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"3a9bc7a5fff3d80eb04fe1bfebb5c1e9543f99c72342a46af4069af857693b87","anchor_positions":[222,446,669,893,1116,1340,1563,1787,2010,2234,2457,2681,2904,3128,3351,3575],"anchor_positions_sha256":"f02061347ae4b281dab9fe2e3493b73bfb716ffe8fbf61cda899a3009674e03c","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:aggregation:cwe:length=4096:seed=12340:sample=0","config":"cwe","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[222,445,669,892,1115,1339,1562,1786,2009,2232,2456,2679,2902,3126,3349,3573],"fisher_boundary_sha256":"44f51387dfbe3723bf9be8b472cc53ae9a75aa3358aacbe82ce774bca098b167","horizon":1,"input_positions":[223,446,670,893,1116,1340,1563,1787,2010,2233,2457,2680,2903,3127,3350,3574],"input_token_ids_sha256":"7d7dec828b0e79f43e590036098693e354e401b432f63c491d9b74ef31439c57","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[224,447,671,894,1117,1341,1564,1788,2011,2234,2458,2681,2904,3128,3351,3575],"target_token_ids_sha256":"8a65c1ab28b83d19798e54204b1019dbe72fa0be387f3d6e56ba27d82d13e7fa"},"formatted_content_sha256":"bbb3633bd44e955d1f33e80a2c1f71f0ae0e778f671a8a282b12b48eb76e3e53","generator_receipt_sha256":"3a6698b2ea721d388d72f28e1c4d4646616d6952fe01ed8038c01dc53dee3a72","identity_record_sha256":"cad2c6009d9d8080acff02b0fc66babc81f232bac219039d597c2889f240a957","prompt_token_ids_sha256":"679f814d93763a3c0db75fbc0cbf70d684bfd0ccde3a62fe15ff9d09dbc1a014","ruler_category":"aggregation","seed":12340,"selection_rank":13,"selection_sha256":"e09c4998af87e1e33e97059c684d3b0558e6e1e107295e2ccf3655bc3d70e7c8","sequence_length":3576,"sequence_token_ids_sha256":"679f814d93763a3c0db75fbc0cbf70d684bfd0ccde3a62fe15ff9d09dbc1a014","source_content_sha256":"3a6698b2ea721d388d72f28e1c4d4646616d6952fe01ed8038c01dc53dee3a72","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":3576,"cache_exposed_stop":3576,"prefill_start":0,"prefill_stop":3576,"scored_start":3576,"scored_stop":3576},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"8046d74f592f47985b3070756d1e18a618db723abbd079cab7a5a30d4efcf596","anchor_positions":[124,250,375,501,626,752,877,1003,1129,1254,1380,1505,1631,1756,1882,2008],"anchor_positions_sha256":"d296c9d479bcd5abca26602f7c8ff6f9e1668dd7bcce7512dc2b7a9f6612803c","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:multi_hop_tracing:vt:length=2048:seed=12340:sample=0","config":"vt","configured_length":2048,"family":"ruler","fisher_boundary":{"boundary_positions":[124,249,375,500,626,751,877,1002,1127,1253,1378,1504,1629,1755,1880,2006],"fisher_boundary_sha256":"bddd9058cc4355edbe7a8c638de06d849c2134d8813b5401ee6957f2d201a918","horizon":1,"input_positions":[125,250,376,501,627,752,878,1003,1128,1254,1379,1505,1630,1756,1881,2007],"input_token_ids_sha256":"56e2b2e1f3b426b0a06e44e2cbb64564b09850f5d87d64bf62b5cea5466bcee1","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[126,251,377,502,628,753,879,1004,1129,1255,1380,1506,1631,1757,1882,2008],"target_token_ids_sha256":"bd6a1f6ec03f5d3adcfb37a0b1ac11dadaa147d56888994f667aa7841dc1257b"},"formatted_content_sha256":"b17100d16d1771ca1122c2b9937cacb984da6216ccf38bef7be0228979e43248","generator_receipt_sha256":"e8b50ab62299b2f7306b66c83a8c6f2b471e1198bc02aba090f1ab406da24110","identity_record_sha256":"ab943760500ed5247a613c9eb87ac98c621b16c8919c9b1bcf5bebb2f1da258f","prompt_token_ids_sha256":"c6dfdc419e73d825ef29b21a8e2369076e96964690bfab85b8262a2299d05ce2","ruler_category":"multi_hop_tracing","seed":12340,"selection_rank":14,"selection_sha256":"f833f67c79e0d9e87d7d872ac8220d32c02483b56c47dbcd2385976fff75fe5f","sequence_length":2009,"sequence_token_ids_sha256":"c6dfdc419e73d825ef29b21a8e2369076e96964690bfab85b8262a2299d05ce2","source_content_sha256":"e8b50ab62299b2f7306b66c83a8c6f2b471e1198bc02aba090f1ab406da24110","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":2009,"cache_exposed_stop":2009,"prefill_start":0,"prefill_stop":2009,"scored_start":2009,"scored_stop":2009},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"},{"anchor_manifest_sha256":"163e62428e8cb8f598c255e524c87ac6fe75f5451263fe61445eb8cafaa35b23","anchor_positions":[246,493,740,987,1234,1482,1729,1976,2223,2470,2718,2965,3212,3459,3706,3954],"anchor_positions_sha256":"c09190aec7245025981cb99d1fb2f1b133b5cd4db4f1f1baa5e107f0f2266b39","canonical_id":"recurquant.experiment013.ruler.sequence.v1:c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a:retrieval:niah_single_1:length=4096:seed=12339:sample=0","config":"niah_single_1","configured_length":4096,"family":"ruler","fisher_boundary":{"boundary_positions":[246,493,740,987,1234,1481,1728,1975,2222,2469,2716,2963,3210,3457,3704,3952],"fisher_boundary_sha256":"5d449cbf648406313b9dd1abfb6d827d86ea00767627eae59437398324207334","horizon":1,"input_positions":[247,494,741,988,1235,1482,1729,1976,2223,2470,2717,2964,3211,3458,3705,3953],"input_token_ids_sha256":"bc6efe8e70dc84d76b6d63607de536e9dad39e3e692c1153b17d01d1f948f5b4","schema":"recurquant.experiment013.fisher-boundary.v1","target_positions":[248,495,742,989,1236,1483,1730,1977,2224,2471,2718,2965,3212,3459,3706,3954],"target_token_ids_sha256":"8b9cd390ee896f79de83be50c222152af4aff55c3013e4e55bd32df875a3182e"},"formatted_content_sha256":"5044e6b8cb4a5c93592f73b69fc42041fad9c54b2ba7d53216fc486266f9f8ba","generator_receipt_sha256":"bd8f1387bc9aca3118ced5d9287c08f72c670bfbe6ef88729d36d6940cc2a22f","identity_record_sha256":"9209f85027779d40b0ad2a15080aef80e042f89c3c96760320f1648c2f18b461","prompt_token_ids_sha256":"786505e0d874cafdc0c5a41e3e0922fe96e048b0fb0ed0d4c098605d9bf54be8","ruler_category":"retrieval","seed":12339,"selection_rank":15,"selection_sha256":"ffbb6f30b667ab8eb83eff25cc9f46717b4a36965d1f9c15e08b6633fe46e13d","sequence_length":3955,"sequence_token_ids_sha256":"786505e0d874cafdc0c5a41e3e0922fe96e048b0fb0ed0d4c098605d9bf54be8","source_content_sha256":"bd8f1387bc9aca3118ced5d9287c08f72c670bfbe6ef88729d36d6940cc2a22f","target_token_ids_sha256":"37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570","token_span":{"cache_exposed_start":3955,"cache_exposed_stop":3955,"prefill_start":0,"prefill_stop":3955,"scored_start":3955,"scored_stop":3955},"tokenizer_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef"}],"resolver_version":6,"schema_version":5,"selection":{"humaneval_plus_stage_a_b_namespace":"recurquant.experiment013.humaneval-plus.stage-a-b.v1\u0000","humaneval_plus_stage_c_namespace":"recurquant.experiment013.humaneval-plus.stage-c.v1\u0000","pg19_test_namespace":"recurquant.experiment013.pg19.test.v1\u0000","pg19_train_namespace":"recurquant.experiment013.pg19.train.v1\u0000","pg19_validation_namespace":"recurquant.experiment013.pg19.validation.v1\u0000"},"source_manifest_sha256":"37652b1a540b34119c5d490e34fd84884e0b92ba23c4cc54ea88efe24dd2cd71","status":"frozen","tokenizer":{"class":"Qwen2Tokenizer","file_manifest_sha256":"e48bffe3aeaf5436b23f349a4517ebc8c8f965cd60b9566014191a7e7938f2ef","files":[{"name":"merges.txt","sha256":"a9d356d7bdf1ef4949e3e748e95b8e10ad9d4e2e838eddc38a0a7b6b94d1db8d","size_bytes":3353259},{"name":"tokenizer.json","sha256":"fe000e3ed39ed12b8d2481d527d44f93c65d37e87645d2dcc80d1bf9d50d2927","size_bytes":12807196},{"name":"tokenizer_config.json","sha256":"e611fbccc7c29ef3b1cafb1cb7ea548d189968632901d678fd62be68c47885de","size_bytes":16712},{"name":"vocab.json","sha256":"ce99b4cb2983d118806ce0a8b777a35b093e2000a503ebde25853284c9dfa003","size_bytes":6722759}],"revision":"dc7cdfe2ee4154fa7e30f5b51ca41bfa40174e68","source_id":"Qwen/Qwen3.5-0.8B-Base","transformers_version":"5.14.1"},"upstream_tool_contracts":{"evalplus_formatter":{"id":"evalplus/evalplus","revision":"26d6d00bb1fd0fa37f39c99d5290da67891d1c5e"},"ruler_generator":{"id":"NVIDIA/RULER","revision":"c3f5e3b4f87f97e048793bb510a3a6b19a46bf3a"}}}} From 3abaa502da47a0fa14f53e280868274f5ce17adf Mon Sep 17 00:00:00 2001 From: HiddenTrojan Date: Sat, 15 Aug 2026 06:53:03 +0800 Subject: [PATCH 24/24] Repair Experiment 013 RULER receipt preflight --- ...EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md | 144 +++++++++- scripts/launch_static_q468_calibration.py | 249 ++++++++++++++--- scripts/run_static_q468_calibration.py | 102 ++++++- tests/test_launch_static_q468_calibration.py | 251 +++++++++++++++++- tests/test_run_static_q468_calibration.py | 246 ++++++++++++++++- 5 files changed, 940 insertions(+), 52 deletions(-) diff --git a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md index 398beda..3ed97dc 100644 --- a/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md +++ b/research/EXPERIMENT_013_STATIC_RHT_Q468_PROTOCOL.md @@ -1,7 +1,8 @@ # Experiment 013: static RHT-Q468 packed-native adoption protocol -> **Status: third replacement-H0 candidate after a preserved post-authorization, -> pre-payload model-staging path failure; not yet re-preregistered.** +> **Status: fifth replacement-H0 candidate after a preserved pre-model, +> pre-dataset-row Fisher-smoke failure caused by an ambiguous RULER directory +> argument; not yet re-preregistered.** > > This replacement working copy becomes the next frozen Experiment 013 > preregistration only when its exact bytes and dependencies are committed in a @@ -19,7 +20,7 @@ Protocol draft initiated: 2026-08-02 -Current third replacement-H0 candidate amended: 2026-08-15 +Current fifth replacement-H0 candidate amended: 2026-08-15 Pre-resolution audit amendment: 2026-08-02. The amendment corrects a cache-exposed-span off-by-one, binds the Stage-A calibration chain by exact @@ -481,6 +482,130 @@ pointers may differ: /evidence/source_manifest_sha256 ``` +Sixteenth pre-resolution RULER-receipt-directory amendment: 2026-08-15. Under +source commit `475659ac8a0a98aaf38814e89f0f95d31392ec8b`, tag +`experiment013-h0-475659a`, and identity-only descendant +`fd67384944dc92abac0422960ea53fc973b64736`, the runner-v5 staging-path +preflight, model-staging authorization, and identity-bound model publication +completed. The frozen identity file SHA-256 was +`40c434d038879608093fc8f74b66893062e4f52a0e1db9d33b40ac9fa411be90`, +and the published model root was +`C:\tmp\recurquant-exp013-model-h1-fd673849`. + +The first sealed `--fisher-h1-smoke` invocation then supplied the external +RULER source checkout to runner v5's generically named `--ruler-root` option. +The reviewed adapter interprets that value only as the directory containing the +sealed RULER result receipts. The command therefore reached calibration +sequence materialization, authenticated source-head metadata and public +tokenizer material, and then rejected the shallow directory inventory: all 20 +frozen receipt filenames plus `generation-manifest.json` were missing, while +the source checkout's Git, environment, documentation, source, auxiliary, and +raw-receipt entries were unexpected. The immediate cause was a wrong command +argument; the execution-control defect was an ambiguous CLI name with no pure +early receipt-directory precondition. + +The inventory rejection occurred before any generation-manifest or RULER +receipt body was opened. It also preceded MBPP row access, PG19 projection or +row access, RULER record decoding or semantic replay, and HumanEval+ projection +or row access. Because complete sequence materialization precedes local model +authentication, the failed smoke did not traverse or hash the published model +root, read model configuration or weight bytes, deserialize parameters, enter +CUDA, execute a causal forward or Fisher step, or compute any calibration or +quality value. The declared smoke output directory remained absent; no report, +completion marker, score, policy, calibration binding, stability value, or +quality artifact was published. This is a pre-model and pre-dataset-row +execution-contract incident, not a Fisher result and not an infrastructure +interruption. + +The child additionally reported that its sealed scratch directory was nonempty +while unwinding the primary inventory exception, and the host repeated that +postcondition failure before deleting the owned scratch tree. No scratch path +survived, but its transient inventory was not preserved and therefore must not +be guessed. This secondary diagnostic does not replace or weaken the proven +primary cause. The repaired launcher preserves a child exception as primary and +adds aggregated child-postcondition failures as notes; when `sealed_main` or +the sealed child returns nonzero, it preserves that return code and reports +secondary postcondition or cleanup failures separately. Only an otherwise +successful child promotes a postcondition failure to the primary error. The +host records each temporary root's device/inode/type identity at creation, +refuses cleanup if that identity changed or the owned tree contains a link, +reparse point, or entry other than a regular file or directory, removes only +the authenticated owned tree, +and detects survival. Regressions must cover primary-exception and nonzero- +return preservation, aggregation of multiple secondary failures, successful- +child postcondition failure, identity replacement, reparse refusal, partial +temporary-root creation, and cleanup of an ordinary nonempty owned tree. + +Calibration-runner v6 removes `--ruler-root` from the official smoke and full- +calibration CLI without a compatibility alias. The sole public option is +`--ruler-receipt-dir`; its help and protocol meaning are exactly the sealed v7 +receipt directory, never the RULER source checkout or raw-output directory. At +sealed-runner entry, before that runner reads or authenticates the runtime +manifest, a standard-library-only precondition requires an absolute existing +directory whose +ancestors and root contain no link or reparse point; an exact shallow, +case-insensitively unique inventory of `generation-manifest.json` plus the 20 +frozen receipt filenames; and a regular non-link, non-reparse file for every +entry. It opens no file body or JSON value, performs no Git or artifact +authentication, imports no capture, tokenizer, dataset, adapter, Hub, model, or +CUDA code, and creates no artifact. `_official_main` repeats the same +precondition before reading the frozen identity or runtime, model, Parquet, or +source-manifest bytes. The existing phase-scoped semantic verifier remains +authoritative at point of use and still authenticates the permitted receipt +bodies and their complete manifest commitments. + +Runner revision advances from v5 to v6. The external CLI rename does not rename +the reviewed adapter's internal `AdapterConstructionContext.ruler_root` field; +that field remains an implementation detail populated only from the normalized +`--ruler-receipt-dir` result. A regression must prove that the retired +`--ruler-root` spelling is unrecognized, the RULER source checkout fails at the +pure precondition, the exact 21-file receipt directory passes, and every +missing, extra, case-colliding, non-regular, link, reparse, relative, or +ancestor-link variant fails before manifest authentication, materialization, +model access, or scratch population. No record selection, dataset revision, +calibration span, Fisher boundary, quantization policy, metric, gate, model +contract, or protected-stage rule changes. + +The incident H0, tag, H1, frozen identity, published model root, failed command, +and absent output remain unchanged as superseded evidence. They authorize no +same-command retry and no further official execution. In particular, the old +published model root may not be adopted, renamed, copied, or rebound as the +official model root of a replacement H1. + +The exact RULER v7 batch may be reused only if its complete 21-file receipt +inventory and the 100 raw producer files replay-authenticate unchanged and its +generation-manifest file SHA-256 remains +`979f91848b6c0692160419c3e5e9ee555aa94d9e7add3092067f003ea0543e80`. +The intended CLI value is the absolute resolution of repository-relative +`artifacts/experiment013/ruler-receipts-v7-h0-447295e`; the separate pinned +RULER source checkout is used only for replay authentication and must never be +passed as `--ruler-receipt-dir`. +The sealed calibration runtime, public model-metadata manifest, immutable- +Parquet materialization manifest, and external shared Hub cache may likewise be +reused only after all existing byte, inventory, path, version, and semantic +checks pass at every required point of use. Their currently expected manifest +file SHA-256 values are, respectively, +`80ca233a29af4facbb334fd4fb51a4f6e9a3d6815465cb79b9f3db63ef668d6a`, +`586d9c7e520f3bbd99ecef30663bf07d283eb14622475c58891becd8e033b05c`, +and `ee5628e50e5d3516fd79077542d355fd915455ac0e53128d372f4177ad63d39c`. +Any mismatch stops reuse; it is not an authorization to regenerate, substitute, +or weaken a frozen dependency. + +Before another smoke, the exact runner-v6, launcher cleanup repair, tests, and +this protocol must be committed in a fresh clean H0 and bound by a fresh +repository-source manifest. The exact promoted calibration identity must be +recaptured and reverified against that source binding, committed alone in a +fresh H1, and reauthorized. Model-staging paths and authorization must be +reverified, and the same authenticated three-file payload must be published by +a new no-overwrite `stage-model` execution under that H1 into a fresh model +root. Only then may the sealed Fisher H=1 smoke be attempted with the exact +receipt directory supplied through `--ruler-receipt-dir`. The replacement +identity must retain all 160 records and content manifest +`ee72483a8f8b4370c9e667e4287747e5bc358aeb0265a58167140f4e780a7b29`; +relative to the retired identity, only the five repository-source and +promotion-hash cascade pointers listed above may differ. Any other identity +difference stops the chain. + ## Question Can a calibration-frozen, static Q4/Q6/Q8 recurrent-state layout satisfy the @@ -876,7 +1001,7 @@ Before committing H1, the exact promoted identity bytes in their ignored, no-overwrite precommit location must pass `verify-frozen-identity-contract`. That read-only command authenticates H0 and its source manifest, loads the exact H0 resolver, and consumes the complete record inventory through calibration- -runner v5's identity view. It accepts no H1, model manifest, Hub, cache, or +runner v6's identity view. It accepts no H1, model manifest, Hub, cache, or output argument. Its non-persisted canonical JSON stdout document uses artifact kind `recurquant_experiment013_frozen_identity_contract_verification`, schema version one, and binds the H0/source contract, portable Git identity, all four @@ -945,6 +1070,17 @@ the published model root. Failure may clean only the exact owned staging- directory identity; it never overwrites the output or deletes the shared Hub cache. +Every official Fisher smoke and full calibration invocation uses the public +option `--ruler-receipt-dir` for the normalized exact 21-file v7 receipt +directory. The retired `--ruler-root` spelling is invalid and has no legacy +alias. After the outer sealed launcher has authenticated the source and runtime +needed to establish the child, but before the sealed runner reads its runtime +manifest or any frozen-identity, source-manifest, model-manifest, or Parquet- +manifest byte, runner v6 performs the shallow pure directory precondition +defined by the Sixteenth amendment and repeats it inside `_official_main`. +Passing that precondition does not authenticate any receipt body; phase-scoped +point-of-use verification remains mandatory. + Stage-A resolution additionally consumes one strictly decoded `experiment-013-stage-a-calibration-binding-v3` artifact. The resolved Stage-A identity binds these eight dependency files directly, not merely semantic IDs diff --git a/scripts/launch_static_q468_calibration.py b/scripts/launch_static_q468_calibration.py index f4b3e87..787f390 100644 --- a/scripts/launch_static_q468_calibration.py +++ b/scripts/launch_static_q468_calibration.py @@ -84,10 +84,12 @@ { "--frozen-identity", "--repository-root", + "--ruler-receipt-dir", *_BOUND_ARTIFACT_OPTIONS.values(), *_EXPECTED_BOUND_DIGEST_OPTIONS.values(), } ) +_FORBIDDEN_RUNNER_OPTIONS: Final = frozenset({"--ruler-root"}) _SMOKE_PREREQUISITE_OPTIONS: Final = frozenset( { "--prior-fisher-h1-smoke-report", @@ -839,6 +841,10 @@ def _extract_runner_options(arguments: Sequence[str]) -> dict[str, str]: index = 0 while index < len(arguments): option = arguments[index] + if option in _FORBIDDEN_RUNNER_OPTIONS or any( + option.startswith(f"{forbidden}=") for forbidden in _FORBIDDEN_RUNNER_OPTIONS + ): + raise SealedLaunchError(f"legacy runner option is forbidden: {option}") if option in value_options: if option in result or index + 1 >= len(arguments): raise SealedLaunchError(f"runner option is duplicated or incomplete: {option}") @@ -1083,20 +1089,99 @@ def _verify_empty_scratch(path: Path) -> Path: return root -def _assert_scratch_tree_has_no_reparse(path: Path) -> None: +def _temporary_directory_identity(path: Path, *, context: str) -> tuple[int, int, int]: + root = _absolute_directory(path, context=context) + try: + status = root.lstat() + except OSError as exc: + raise SealedLaunchError(f"cannot identify {context}") from exc + return (int(status.st_dev), int(status.st_ino), stat.S_IFMT(status.st_mode)) + + +def _assert_owned_temporary_tree_has_no_reparse(path: Path, *, context: str) -> None: stack = [path] while stack: directory = stack.pop() - for entry in os.scandir(directory): - status = entry.stat(follow_symlinks=False) + try: + entries = tuple(os.scandir(directory)) + except OSError as exc: + raise SealedLaunchError(f"cannot enumerate {context} during cleanup") from exc + for entry in entries: + try: + status = entry.stat(follow_symlinks=False) + except OSError as exc: + raise SealedLaunchError(f"cannot inspect {context} during cleanup") from exc if entry.is_symlink() or bool( getattr(status, "st_file_attributes", 0) & _WINDOWS_REPARSE_POINT ): - raise SealedLaunchError("sealed scratch contains a link or reparse point") + raise SealedLaunchError(f"{context} contains a link or reparse point") if stat.S_ISDIR(status.st_mode): stack.append(Path(entry.path)) elif not stat.S_ISREG(status.st_mode): - raise SealedLaunchError("sealed scratch contains a non-regular path") + raise SealedLaunchError(f"{context} contains a non-regular path") + + +def _cleanup_owned_temporary_directory( + path: Path, + *, + expected_identity: tuple[int, int, int], + context: str, +) -> None: + """Remove one launcher-owned tree without following redirected content.""" + + root = Path(os.path.abspath(path)) + if not os.path.lexists(root): + return + observed_identity = _temporary_directory_identity(root, context=context) + if observed_identity != expected_identity: + raise SealedLaunchError(f"{context} identity changed before cleanup") + _assert_owned_temporary_tree_has_no_reparse(root, context=context) + try: + shutil.rmtree(root, ignore_errors=False) + except OSError as exc: + raise SealedLaunchError(f"cannot remove {context}") from exc + if os.path.lexists(root): + raise SealedLaunchError(f"{context} survived cleanup") + + +def _failure_summary(failures: Sequence[tuple[str, BaseException]]) -> str: + return "; ".join( + f"{context}: {error.__class__.__name__}: {error}" for context, error in failures + ) + + +def _postcondition_error(failures: Sequence[tuple[str, BaseException]]) -> SealedLaunchError: + if not failures: + raise ValueError("postcondition failure inventory cannot be empty") + error = SealedLaunchError(f"sealed child postcondition failed: {_failure_summary(failures)}") + error.__cause__ = failures[0][1] + return error + + +def _surface_secondary_failures( + failures: Sequence[tuple[str, BaseException]], + *, + primary_error: BaseException | None, + child_returncode: int | None, +) -> None: + if not failures: + return + message = f"sealed launcher secondary failure: {_failure_summary(failures)}" + if primary_error is not None: + add_note = getattr(primary_error, "add_note", None) + if callable(add_note): + add_note(message) + else: # pragma: no cover - supported Python versions expose add_note + print(message, file=sys.stderr, flush=True) + return + if child_returncode is not None and child_returncode != 0: + print( + f"{message}; preserving child return code {child_returncode}", + file=sys.stderr, + flush=True, + ) + return + raise _postcondition_error(failures) def _sealed_environment(*, scratch_directory: Path) -> dict[str, str]: @@ -1261,11 +1346,30 @@ def _sealed_argv( "--prior-fisher-h1-smoke-report", "--prior-fisher-h1-smoke-complete-marker", } +_forbidden_runner_options = {"--ruler-root"} _smoke_marker = b"recurquant-experiment013-fisher-h1-smoke-complete-v1\n" def _fail(message): raise RuntimeError(message) +def _surface_postcondition_failures(primary, result, failures): + if not failures: + return + details = "; ".join( + label + ": " + error.__class__.__name__ + ": " + str(error) + for label, error in failures + ) + message = "sealed bootstrap secondary postcondition failure: " + details + if primary is not None: + primary.add_note(message) + elif type(result) is int and result != 0: + _s.stderr.write( + message + "; preserving sealed_main return code " + str(result) + "\n" + ) + _s.stderr.flush() + else: + _fail(message) + def _canonical(value): return (_j.dumps(value, ensure_ascii=False, allow_nan=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8") @@ -1461,12 +1565,15 @@ def _name(value): def _options(arguments): required = {"--frozen-identity", "--repository-root", *_binding_options.values(), - *_expected_digest_options.values()} + *_expected_digest_options.values(), "--ruler-receipt-dir"} value_options = required | _smoke_options result = {} index = 0 while index < len(arguments): item = arguments[index] + if (item in _forbidden_runner_options + or any(item.startswith(value + "=") for value in _forbidden_runner_options)): + _fail("legacy runner option is forbidden: " + item) if item in value_options: if ( item in result @@ -1903,6 +2010,7 @@ def _verify_runtime(manifest, base_raw, package_raw, git_raw, packages_appended= _module.__file__ = str(_runner_path) _module.__package__ = "" _s.modules[_module.__name__] = _module +_result = None try: try: _code = compile(_payload, str(_runner_path), "exec", dont_inherit=True) @@ -1925,19 +2033,34 @@ def _verify_runtime(manifest, base_raw, package_raw, git_raw, packages_appended= git_executable_path=_git_executable, pycache_prefix=_pycache, ) + if not isinstance(_result, int) or isinstance(_result, bool): + _fail("sealed_main returned a non-integer status") finally: - if any(_pycache.iterdir()): - _fail("sealed pycache prefix changed during calibration") - if any(_scratch.iterdir()): - _fail("sealed scratch directory was not cleaned by the runner") - _verify_source(_source_manifest, _repository_root) - _verify_runtime( - _runtime, _base, _packages, _git_executable, packages_appended=True - ) + _primary = _s.exception() + _postcondition_failures = [] + try: + if any(_pycache.iterdir()): + raise RuntimeError("sealed pycache prefix changed during calibration") + except Exception as error: + _postcondition_failures.append(("pycache", error)) + try: + if any(_scratch.iterdir()): + raise RuntimeError("sealed scratch directory was not cleaned by the runner") + except Exception as error: + _postcondition_failures.append(("scratch", error)) + try: + _verify_source(_source_manifest, _repository_root) + except Exception as error: + _postcondition_failures.append(("repository source reauthentication", error)) + try: + _verify_runtime( + _runtime, _base, _packages, _git_executable, packages_appended=True + ) + except Exception as error: + _postcondition_failures.append(("runtime reauthentication", error)) + _surface_postcondition_failures(_primary, _result, _postcondition_failures) finally: _s.modules.pop("_recurquant_experiment013_sealed_runner", None) -if not isinstance(_result, int) or isinstance(_result, bool): - _fail("sealed_main returned a non-integer status") raise SystemExit(_result) """.strip() @@ -2002,11 +2125,23 @@ def launch(argv: Sequence[str]) -> int: require_current_process=False, ) - pycache_parent = Path(tempfile.mkdtemp(prefix="recurquant-exp013-sealed-pycache-")) - pycache = _verify_empty_pycache(pycache_parent) - scratch_parent = Path(tempfile.mkdtemp(prefix="recurquant-exp013-sealed-scratch-")) - scratch = _verify_empty_scratch(scratch_parent) + pycache: Path | None = None + scratch: Path | None = None + pycache_identity: tuple[int, int, int] | None = None + scratch_identity: tuple[int, int, int] | None = None + completed: subprocess.CompletedProcess[bytes] | None = None + primary_error: BaseException | None = None + secondary_failures: list[tuple[str, BaseException]] = [] try: + pycache = Path(tempfile.mkdtemp(prefix="recurquant-exp013-sealed-pycache-")) + pycache_identity = _temporary_directory_identity(pycache, context="pycache prefix") + pycache = _verify_empty_pycache(pycache) + scratch = Path(tempfile.mkdtemp(prefix="recurquant-exp013-sealed-scratch-")) + scratch_identity = _temporary_directory_identity( + scratch, + context="sealed scratch directory", + ) + scratch = _verify_empty_scratch(scratch) command = _sealed_argv( interpreter=interpreter, runtime_manifest=args.runtime_manifest.resolve(strict=True), @@ -2024,29 +2159,61 @@ def launch(argv: Sequence[str]) -> int: env=_sealed_environment(scratch_directory=scratch), input=SEALED_BOOTSTRAP_BYTES, ) - _verify_empty_pycache(pycache) - _verify_empty_scratch(scratch) - _bindings, repeated_source, _runner_path = _verify_bound_artifacts( - runner_options, - runtime_manifest_path=args.runtime_manifest, - ) - if repeated_source["git_executable"] != source_manifest["git_executable"]: - raise SealedLaunchError("source Git executable binding changed during execution") - _verify_runtime( - runtime_manifest, - base_runtime_root=base, - package_roots=packages, - git_executable_path=git_executable, - require_current_process=False, - ) + try: + _verify_empty_pycache(pycache) + except Exception as exc: + secondary_failures.append(("pycache postcondition", exc)) + try: + _verify_empty_scratch(scratch) + except Exception as exc: + secondary_failures.append(("scratch postcondition", exc)) + try: + _bindings, repeated_source, _runner_path = _verify_bound_artifacts( + runner_options, + runtime_manifest_path=args.runtime_manifest, + ) + if repeated_source["git_executable"] != source_manifest["git_executable"]: + raise SealedLaunchError("source Git executable binding changed during execution") + except Exception as exc: + secondary_failures.append(("bound-artifact reauthentication", exc)) + try: + _verify_runtime( + runtime_manifest, + base_runtime_root=base, + package_roots=packages, + git_executable_path=git_executable, + require_current_process=False, + ) + except Exception as exc: + secondary_failures.append(("runtime reauthentication", exc)) + if completed.returncode == 0 and secondary_failures: + failures = tuple(secondary_failures) + secondary_failures.clear() + raise _postcondition_error(failures) return int(completed.returncode) + except BaseException as exc: + primary_error = exc + raise finally: - _verify_empty_pycache(pycache) - shutil.rmtree(pycache, ignore_errors=False) - _assert_scratch_tree_has_no_reparse(scratch) - shutil.rmtree(scratch, ignore_errors=False) - if os.path.lexists(scratch): - raise SealedLaunchError("sealed scratch directory survived cleanup") + for path, expected_identity, context in ( + (scratch, scratch_identity, "sealed scratch directory"), + (pycache, pycache_identity, "pycache prefix"), + ): + if path is None or expected_identity is None: + continue + try: + _cleanup_owned_temporary_directory( + path, + expected_identity=expected_identity, + context=context, + ) + except Exception as exc: + secondary_failures.append((f"{context} cleanup", exc)) + _surface_secondary_failures( + secondary_failures, + primary_error=primary_error, + child_returncode=None if completed is None else int(completed.returncode), + ) def main(argv: Sequence[str] | None = None) -> int: diff --git a/scripts/run_static_q468_calibration.py b/scripts/run_static_q468_calibration.py index 594b52f..adc7233 100644 --- a/scripts/run_static_q468_calibration.py +++ b/scripts/run_static_q468_calibration.py @@ -56,7 +56,7 @@ CANONICAL_ADAPTER_MODULE: Final = "recurquant.experiment013_qwen35_adapter" CANONICAL_ADAPTER_PATH: Final = "src/recurquant/experiment013_qwen35_adapter.py" -RUNNER_REVISION: Final = "experiment-013-static-q468-calibration-runner-v5" +RUNNER_REVISION: Final = "experiment-013-static-q468-calibration-runner-v6" FROZEN_IDENTITY_SCHEMA_VERSION: Final = 5 FISHER_BOUNDARY_SCHEMA: Final = "recurquant.experiment013.fisher-boundary.v1" FISHER_BOUNDARY_NAMESPACE: Final = b"recurquant.experiment013.fisher-boundary.v1\0" @@ -127,6 +127,29 @@ FISHER_SMOKE_REPORT_FILENAME: Final = "fisher-h1-smoke-report.json" FISHER_SMOKE_COMPLETE_FILENAME: Final = "FISHER_H1_SMOKE_COMPLETE" FISHER_SMOKE_COMPLETE_BYTES: Final = b"recurquant-experiment013-fisher-h1-smoke-complete-v1\n" +RULER_RECEIPT_DIRECTORY_FILENAMES: Final = ( + "aggregation__cwe__l2048__s12340.json", + "aggregation__cwe__l4096__s12340.json", + "aggregation__fwe__l2048__s12339.json", + "aggregation__fwe__l4096__s12339.json", + "aggregation__fwe__l4096__s2343.json", + "generation-manifest.json", + "multi_hop_tracing__vt__l2048__s12339.json", + "multi_hop_tracing__vt__l2048__s12340.json", + "multi_hop_tracing__vt__l4096__s12339.json", + "multi_hop_tracing__vt__l4096__s12340.json", + "multi_hop_tracing__vt__l4096__s2343.json", + "question_answering__qa_1__l2048__s12339.json", + "question_answering__qa_1__l4096__s12339.json", + "question_answering__qa_1__l4096__s2343.json", + "question_answering__qa_2__l2048__s12340.json", + "question_answering__qa_2__l4096__s12340.json", + "retrieval__niah_multikey_2__l2048__s12340.json", + "retrieval__niah_multiquery__l2048__s12339.json", + "retrieval__niah_multiquery__l4096__s2343.json", + "retrieval__niah_multivalue__l4096__s12340.json", + "retrieval__niah_single_1__l4096__s12339.json", +) PREPARED_RUNTIME_MANIFEST_FILENAME: Final = "calibration-runtime-manifest.json" PREPARED_RUNTIME_COMPLETE_FILENAME: Final = "RUNTIME_PREPARED" DEFAULT_PACKAGE_RUNTIME_ROOT_NAME: Final = "calibration-packages" @@ -1930,6 +1953,67 @@ def snapshot() -> tuple[DirectoryComponentIdentity, ...]: return resolved, after +def _verify_ruler_receipt_directory_precondition(path: Path) -> Path: + """Validate only the frozen receipt-directory shape without reading file bytes.""" + + raw = Path(path) + if not raw.is_absolute(): + raise CalibrationRunError("RULER receipt directory must be absolute") + root, component_identities = _require_existing_regular_directory( + raw, + context="RULER receipt directory", + ) + + def snapshot() -> tuple[tuple[str, int, int, int, int], ...]: + try: + with os.scandir(root) as iterator: + entries = sorted(iterator, key=lambda entry: entry.name) + except OSError as exc: + raise CalibrationRunError("RULER receipt directory is unavailable") from exc + names = [entry.name for entry in entries] + if len({name.casefold() for name in names}) != len(names): + raise CalibrationRunError("RULER receipt directory has case-colliding names") + expected = set(RULER_RECEIPT_DIRECTORY_FILENAMES) + observed = set(names) + if observed != expected: + missing = sorted(expected - observed) + unexpected = sorted(observed - expected) + raise CalibrationRunError( + "RULER receipt directory inventory drifted: " + f"missing={missing}, unexpected={unexpected}" + ) + identities: list[tuple[str, int, int, int, int]] = [] + for entry in entries: + try: + status = entry.stat(follow_symlinks=False) + except OSError as exc: + raise CalibrationRunError( + f"RULER receipt entry is unavailable: {entry.name}" + ) from exc + if ( + entry.is_symlink() + or bool(getattr(status, "st_file_attributes", 0) & _WINDOWS_REPARSE_POINT) + or not stat.S_ISREG(status.st_mode) + ): + raise CalibrationRunError( + f"RULER receipt entry must be a regular non-link file: {entry.name}" + ) + identities.append( + (entry.name, status.st_dev, status.st_ino, status.st_mode, status.st_size) + ) + return tuple(identities) + + before = snapshot() + repeated_root, repeated_components = _require_existing_regular_directory( + raw, + context="RULER receipt directory", + ) + after = snapshot() + if repeated_root != root or repeated_components != component_identities or after != before: + raise CalibrationRunError("RULER receipt directory changed while it was validated") + return root + + def _normalized_absolute_path_sha256(path: Path) -> str: if not path.is_absolute(): raise ValueError("path digest input must be absolute") @@ -5809,7 +5893,7 @@ def _load_adapter( def _parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description=__doc__) + parser = argparse.ArgumentParser(description=__doc__, allow_abbrev=False) parser.add_argument("--frozen-identity", required=True, type=Path) parser.add_argument("--repository-source-manifest", required=True, type=Path) parser.add_argument("--model-file-manifest", required=True, type=Path) @@ -5823,7 +5907,15 @@ def _parser() -> argparse.ArgumentParser: parser.add_argument("--expected-runtime-manifest-sha256", required=True) parser.add_argument("--model-root", required=True, type=Path) parser.add_argument("--cache-root", required=True, type=Path) - parser.add_argument("--ruler-root", required=True, type=Path) + parser.add_argument( + "--ruler-receipt-dir", + required=True, + type=Path, + help=( + "Absolute directory containing exactly the frozen RULER generation manifest " + "and 20 receipt JSON files; this is not the NVIDIA/RULER source checkout." + ), + ) parser.add_argument("--repository-root", default=REPOSITORY_ROOT, type=Path) parser.add_argument("--source-commit", required=True) parser.add_argument("--output-dir", required=True, type=Path) @@ -6096,6 +6188,7 @@ def _official_main( ): raise CalibrationRunError("authenticated runner modules were already loaded") args = _parser().parse_args(arguments) + ruler_receipt_dir = _verify_ruler_receipt_directory_precondition(args.ruler_receipt_dir) identity_bytes = args.frozen_identity.read_bytes() bindings = _bootstrap_identity_bindings(identity_bytes) source_manifest_bytes = args.repository_source_manifest.read_bytes() @@ -6239,7 +6332,7 @@ def _official_main( repository_root=args.repository_root, model_root=args.model_root, cache_root=args.cache_root, - ruler_root=args.ruler_root, + ruler_root=ruler_receipt_dir, repository_source_manifest_bytes=source_manifest_bytes, calibration_runtime_manifest_bytes=runtime_manifest_bytes, model_file_manifest_bytes=model_manifest_bytes, @@ -6297,6 +6390,7 @@ def sealed_main( arguments = list(argv) args = _parser().parse_args(arguments) + _verify_ruler_receipt_directory_precondition(args.ruler_receipt_dir) runtime_manifest_bytes = args.runtime_manifest.read_bytes() manifest, runtime_context, _authenticated = _authenticate_sealed_runtime_context( runtime_manifest_bytes, diff --git a/tests/test_launch_static_q468_calibration.py b/tests/test_launch_static_q468_calibration.py index 94a9640..66f0a07 100644 --- a/tests/test_launch_static_q468_calibration.py +++ b/tests/test_launch_static_q468_calibration.py @@ -1,5 +1,6 @@ from __future__ import annotations +import ast import hashlib import importlib.util import json @@ -133,6 +134,8 @@ def _sealed_fixture(tmp_path: Path) -> dict[str, Any]: ) model_path = _write(artifacts / "model.json", b'{"model":"fixture"}\n') parquet_path = _write(artifacts / "parquet.json", b'{"parquet":"fixture"}\n') + ruler_receipt_dir = tmp_path / "ruler-receipts" + ruler_receipt_dir.mkdir() bindings = { "calibration_runtime_manifest_file_sha256": _sha256(runtime_path.read_bytes()), "model_file_manifest_file_sha256": _sha256(model_path.read_bytes()), @@ -175,6 +178,8 @@ def _sealed_fixture(tmp_path: Path) -> dict[str, Any]: bindings["calibration_runtime_manifest_file_sha256"], "--repository-root", str(repository), + "--ruler-receipt-dir", + str(ruler_receipt_dir), ] host_arguments = [ "--base-runtime-root", @@ -197,6 +202,7 @@ def _sealed_fixture(tmp_path: Path) -> dict[str, Any]: "packages": packages, "parquet_path": parquet_path, "repository": repository, + "ruler_receipt_dir": ruler_receipt_dir, "runner_arguments": runner_arguments, "runtime_path": runtime_path, } @@ -301,7 +307,6 @@ def run_wrapper( "pycache", "bound", "runtime", - "pycache", ] assert len(commands) == 1 command, cwd, environment, stdin_payload = commands[0] @@ -327,6 +332,250 @@ def run_wrapper( assert stdin_payload == launcher.SEALED_BOOTSTRAP_BYTES +def test_failed_child_return_code_survives_residue_and_owned_roots_are_cleaned( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + fixture = _sealed_fixture(tmp_path) + temporary_roots: list[Path] = [] + + def run_wrapper(command: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[bytes]: + pycache = Path(command[13]) + scratch = Path(command[15]) + temporary_roots.extend((pycache, scratch)) + _write(pycache / "late.pyc", b"residue") + _write(scratch / "tokenizer" / "temporary.json", b"residue") + return subprocess.CompletedProcess(command, 37) + + monkeypatch.setattr(launcher.subprocess, "run", run_wrapper) + + assert launcher.launch(fixture["host_arguments"]) == 37 + + assert temporary_roots and all(not path.exists() for path in temporary_roots) + diagnostic = capsys.readouterr().err + assert "sealed launcher secondary failure" in diagnostic + assert "pycache postcondition" in diagnostic + assert "scratch postcondition" in diagnostic + assert "preserving child return code 37" in diagnostic + + +def test_successful_child_residue_fails_closed_after_owned_roots_are_cleaned( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fixture = _sealed_fixture(tmp_path) + temporary_roots: list[Path] = [] + + def run_wrapper(command: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[bytes]: + pycache = Path(command[13]) + scratch = Path(command[15]) + temporary_roots.extend((pycache, scratch)) + _write(pycache / "late.pyc", b"residue") + _write(scratch / "temporary.txt", b"residue") + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(launcher.subprocess, "run", run_wrapper) + + with pytest.raises( + launcher.SealedLaunchError, match="sealed child postcondition failed" + ) as caught: + launcher.launch(fixture["host_arguments"]) + + assert "pycache postcondition" in str(caught.value) + assert "scratch postcondition" in str(caught.value) + assert temporary_roots and all(not path.exists() for path in temporary_roots) + + +def test_subprocess_exception_remains_primary_while_owned_roots_are_cleaned( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fixture = _sealed_fixture(tmp_path) + temporary_roots: list[Path] = [] + + def run_wrapper(command: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[bytes]: + pycache = Path(command[13]) + scratch = Path(command[15]) + temporary_roots.extend((pycache, scratch)) + _write(scratch / "temporary.txt", b"residue") + raise OSError("primary subprocess failure") + + monkeypatch.setattr(launcher.subprocess, "run", run_wrapper) + + with pytest.raises(OSError, match="primary subprocess failure"): + launcher.launch(fixture["host_arguments"]) + + assert temporary_roots and all(not path.exists() for path in temporary_roots) + + +def test_cleanup_failure_is_secondary_note_on_primary_exception( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fixture = _sealed_fixture(tmp_path) + created: list[Path] = [] + original_cleanup = launcher._cleanup_owned_temporary_directory + + def mkdtemp(*, prefix: str) -> str: + path = tmp_path / f"{prefix}{len(created)}" + path.mkdir() + created.append(path) + return str(path) + + def cleanup_wrapper(path: Path, **kwargs: Any) -> None: + if kwargs["context"] == "sealed scratch directory": + raise launcher.SealedLaunchError("simulated scratch cleanup failure") + original_cleanup(path, **kwargs) + + def run_wrapper(_command: list[str], **_kwargs: Any) -> subprocess.CompletedProcess[bytes]: + raise OSError("primary subprocess failure") + + monkeypatch.setattr(launcher.tempfile, "mkdtemp", mkdtemp) + monkeypatch.setattr(launcher, "_cleanup_owned_temporary_directory", cleanup_wrapper) + monkeypatch.setattr(launcher.subprocess, "run", run_wrapper) + + with pytest.raises(OSError, match="primary subprocess failure") as caught: + launcher.launch(fixture["host_arguments"]) + + assert any("simulated scratch cleanup failure" in note for note in caught.value.__notes__) + scratch = next(path for path in created if "scratch" in path.name) + assert scratch.exists() + assert all(not path.exists() for path in created if path != scratch) + original_cleanup( + scratch, + expected_identity=launcher._temporary_directory_identity( + scratch, + context="sealed scratch directory", + ), + context="sealed scratch directory", + ) + + +def test_cleanup_refuses_replaced_temporary_root_identity(tmp_path: Path) -> None: + root = tmp_path / "owned-temporary-root" + root.mkdir() + (root / "replacement-sentinel.txt").write_bytes(b"replacement must survive\n") + observed = launcher._temporary_directory_identity(root, context="temporary root") + replaced_identity = (observed[0], observed[1] + 1, observed[2]) + + with pytest.raises(launcher.SealedLaunchError, match="identity changed before cleanup"): + launcher._cleanup_owned_temporary_directory( + root, + expected_identity=replaced_identity, + context="temporary root", + ) + + assert (root / "replacement-sentinel.txt").read_bytes() == b"replacement must survive\n" + + +def test_cleanup_refuses_link_entry_and_preserves_external_target(tmp_path: Path) -> None: + root = tmp_path / "owned-temporary-root" + root.mkdir() + outside = tmp_path / "outside-sentinel.txt" + outside.write_bytes(b"outside must survive\n") + link = root / "redirect" + try: + link.symlink_to(outside) + except OSError as error: + pytest.skip(f"symlink creation is unavailable: {type(error).__name__}") + identity = launcher._temporary_directory_identity(root, context="temporary root") + + with pytest.raises(launcher.SealedLaunchError, match="link or reparse point"): + launcher._cleanup_owned_temporary_directory( + root, + expected_identity=identity, + context="temporary root", + ) + + assert link.is_symlink() + assert outside.read_bytes() == b"outside must survive\n" + + +def test_partial_temporary_root_creation_preserves_error_and_cleans_first_root( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fixture = _sealed_fixture(tmp_path) + created: list[Path] = [] + + def mkdtemp(*, prefix: str) -> str: + if created: + raise OSError("second temporary-root creation failed") + path = tmp_path / f"{prefix}first" + path.mkdir() + created.append(path) + return str(path) + + monkeypatch.setattr(launcher.tempfile, "mkdtemp", mkdtemp) + monkeypatch.setattr( + launcher.subprocess, + "run", + lambda *_args, **_kwargs: pytest.fail("subprocess must not start"), + ) + + with pytest.raises(OSError, match="second temporary-root creation failed"): + launcher.launch(fixture["host_arguments"]) + + assert len(created) == 1 + assert not created[0].exists() + + +def test_launcher_requires_receipt_directory_and_rejects_legacy_flag( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + fixture = _sealed_fixture(tmp_path) + monkeypatch.setattr( + launcher.subprocess, + "run", + lambda *_args, **_kwargs: pytest.fail("subprocess must not start"), + ) + missing = list(fixture["host_arguments"]) + option_index = missing.index("--ruler-receipt-dir") + del missing[option_index : option_index + 2] + + with pytest.raises(launcher.SealedLaunchError, match="--ruler-receipt-dir"): + launcher.launch(missing) + + legacy = list(fixture["host_arguments"]) + legacy[legacy.index("--ruler-receipt-dir")] = "--ruler-root" + with pytest.raises(launcher.SealedLaunchError, match="legacy runner option is forbidden"): + launcher.launch(legacy) + + +def test_embedded_bootstrap_keeps_cleanup_failure_secondary_to_child_failure( + capsys: pytest.CaptureFixture[str], +) -> None: + parsed = ast.parse(launcher.SEALED_BOOTSTRAP) + handler_node = next( + node + for node in parsed.body + if isinstance(node, ast.FunctionDef) and node.name == "_surface_postcondition_failures" + ) + handler_module = ast.fix_missing_locations(ast.Module(body=[handler_node], type_ignores=[])) + + def fail(message: str) -> None: + raise RuntimeError(message) + + namespace: dict[str, Any] = {"_fail": fail, "_s": sys} + exec(compile(handler_module, "", "exec"), namespace) + handler = namespace["_surface_postcondition_failures"] + failures = [("scratch", RuntimeError("scratch residue"))] + + primary = ValueError("primary child failure") + handler(primary, None, failures) + assert any("scratch residue" in note for note in primary.__notes__) + + handler(None, 37, failures) + diagnostic = capsys.readouterr().err + assert "scratch residue" in diagnostic + assert "preserving sealed_main return code 37" in diagnostic + + with pytest.raises(RuntimeError, match="scratch residue"): + handler(None, 0, failures) + + def test_sealed_environment_omits_auth_network_and_compute_modifiers( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_run_static_q468_calibration.py b/tests/test_run_static_q468_calibration.py index 0acbd00..c023c19 100644 --- a/tests/test_run_static_q468_calibration.py +++ b/tests/test_run_static_q468_calibration.py @@ -9,6 +9,7 @@ from collections.abc import Mapping, Sequence from dataclasses import replace from pathlib import Path +from types import SimpleNamespace from typing import Any import pytest @@ -80,6 +81,47 @@ def model_staging_path_contract_sha256( ) +def ruler_receipt_directory(path: Path) -> Path: + path.mkdir(parents=True) + for filename in runner.RULER_RECEIPT_DIRECTORY_FILENAMES: + (path / filename).write_bytes(b"fixture\n") + return path + + +def official_cli_arguments(tmp_path: Path, *, ruler_receipts: Path) -> list[str]: + return [ + "--frozen-identity", + str(tmp_path / "identity.json"), + "--repository-source-manifest", + str(tmp_path / "source.json"), + "--model-file-manifest", + str(tmp_path / "model.json"), + "--expected-model-file-manifest-sha256", + "1" * 64, + "--parquet-materialization-manifest", + str(tmp_path / "parquet.json"), + "--expected-parquet-materialization-manifest-sha256", + "2" * 64, + "--runtime-manifest", + str(tmp_path / "runtime.json"), + "--expected-runtime-manifest-sha256", + "3" * 64, + "--model-root", + str(tmp_path / "model-root"), + "--cache-root", + str(tmp_path / "cache-root"), + "--ruler-receipt-dir", + str(ruler_receipts), + "--repository-root", + str(tmp_path / "repository"), + "--source-commit", + "4" * 40, + "--output-dir", + str(tmp_path / "output"), + "--fisher-h1-smoke", + ] + + def fisher_boundary_contract( token_ids: tuple[int, ...] = (1, 2, 3), ) -> dict[str, object]: @@ -4145,6 +4187,205 @@ def test_source_manifest_output_location_allows_only_external_or_ignored_paths( ) +def test_official_cli_uses_only_unambiguous_ruler_receipt_directory_option( + tmp_path: Path, +) -> None: + receipt_dir = ruler_receipt_directory(tmp_path / "ruler-receipts") + arguments = official_cli_arguments(tmp_path, ruler_receipts=receipt_dir) + + parsed = runner._parser().parse_args(arguments) + + assert parsed.ruler_receipt_dir == receipt_dir + assert "--ruler-receipt-dir" in runner._parser().format_help() + assert "--ruler-root" not in runner._parser().format_help() + legacy = list(arguments) + legacy[legacy.index("--ruler-receipt-dir")] = "--ruler-root" + with pytest.raises(SystemExit): + runner._parser().parse_args(legacy) + + +def test_ruler_receipt_directory_precondition_reads_no_file_bodies( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + receipt_dir = ruler_receipt_directory(tmp_path / "ruler-receipts") + monkeypatch.setattr( + Path, + "read_bytes", + lambda _path: pytest.fail("receipt precondition must not read file bytes"), + ) + + assert runner._verify_ruler_receipt_directory_precondition(receipt_dir) == ( + receipt_dir.resolve(strict=True) + ) + + +@pytest.mark.parametrize("mutation", ["missing", "extra"]) +def test_ruler_receipt_directory_precondition_rejects_inventory_drift( + tmp_path: Path, + mutation: str, +) -> None: + receipt_dir = ruler_receipt_directory(tmp_path / "ruler-receipts") + if mutation == "missing": + (receipt_dir / runner.RULER_RECEIPT_DIRECTORY_FILENAMES[-1]).unlink() + else: + (receipt_dir / "unexpected.json").write_bytes(b"unexpected\n") + + with pytest.raises(runner.CalibrationRunError, match="inventory drifted"): + runner._verify_ruler_receipt_directory_precondition(receipt_dir) + + +def test_ruler_receipt_directory_precondition_rejects_case_collisions( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + receipt_dir = ruler_receipt_directory(tmp_path / "ruler-receipts") + names = [*runner.RULER_RECEIPT_DIRECTORY_FILENAMES, "GENERATION-MANIFEST.JSON"] + + class FakeScandir: + def __enter__(self) -> list[SimpleNamespace]: + return [SimpleNamespace(name=name) for name in names] + + def __exit__(self, *_args: object) -> None: + return None + + monkeypatch.setattr(runner.os, "scandir", lambda _root: FakeScandir()) + + with pytest.raises(runner.CalibrationRunError, match="case-colliding"): + runner._verify_ruler_receipt_directory_precondition(receipt_dir) + + +def test_ruler_receipt_directory_precondition_rejects_nonregular_entry( + tmp_path: Path, +) -> None: + receipt_dir = ruler_receipt_directory(tmp_path / "ruler-receipts") + entry = receipt_dir / runner.RULER_RECEIPT_DIRECTORY_FILENAMES[-1] + entry.unlink() + entry.mkdir() + + with pytest.raises(runner.CalibrationRunError, match="regular non-link file"): + runner._verify_ruler_receipt_directory_precondition(receipt_dir) + + +def test_ruler_receipt_directory_precondition_rejects_symlink_entry( + tmp_path: Path, +) -> None: + receipt_dir = ruler_receipt_directory(tmp_path / "ruler-receipts") + entry = receipt_dir / runner.RULER_RECEIPT_DIRECTORY_FILENAMES[-1] + target = tmp_path / "outside.json" + target.write_bytes(b"outside\n") + entry.unlink() + try: + entry.symlink_to(target) + except OSError as error: + pytest.skip(f"symlink creation is unavailable: {type(error).__name__}") + + with pytest.raises(runner.CalibrationRunError, match="regular non-link file"): + runner._verify_ruler_receipt_directory_precondition(receipt_dir) + + +def test_ruler_receipt_directory_precondition_rejects_reparse_entry( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + receipt_dir = ruler_receipt_directory(tmp_path / "ruler-receipts") + target = runner.RULER_RECEIPT_DIRECTORY_FILENAMES[-1] + + class FakeEntry: + def __init__(self, name: str) -> None: + self.name = name + + @staticmethod + def is_symlink() -> bool: + return False + + def stat(self, *, follow_symlinks: bool) -> SimpleNamespace: + assert follow_symlinks is False + return SimpleNamespace( + st_dev=1, + st_file_attributes=(runner._WINDOWS_REPARSE_POINT if self.name == target else 0), + st_ino=runner.RULER_RECEIPT_DIRECTORY_FILENAMES.index(self.name) + 1, + st_mode=runner.stat.S_IFREG | 0o644, + st_size=8, + ) + + class FakeScandir: + def __enter__(self) -> list[FakeEntry]: + return [FakeEntry(name) for name in runner.RULER_RECEIPT_DIRECTORY_FILENAMES] + + def __exit__(self, *_args: object) -> None: + return None + + monkeypatch.setattr(runner.os, "scandir", lambda _root: FakeScandir()) + + with pytest.raises(runner.CalibrationRunError, match="regular non-link file"): + runner._verify_ruler_receipt_directory_precondition(receipt_dir) + + +def test_ruler_receipt_directory_precondition_rejects_linked_ancestor( + tmp_path: Path, +) -> None: + real_parent = tmp_path / "real-parent" + receipt_dir = ruler_receipt_directory(real_parent / "ruler-receipts") + linked_parent = tmp_path / "linked-parent" + try: + linked_parent.symlink_to(real_parent, target_is_directory=True) + except OSError as error: + pytest.skip(f"symlink creation is unavailable: {type(error).__name__}") + + with pytest.raises(runner.CalibrationRunError, match="link or non-directory"): + runner._verify_ruler_receipt_directory_precondition(linked_parent / receipt_dir.name) + + +@pytest.mark.parametrize("kind", ["relative", "missing"]) +def test_ruler_receipt_directory_precondition_rejects_invalid_path_before_access( + tmp_path: Path, + kind: str, +) -> None: + path = Path("relative-ruler-receipts") if kind == "relative" else tmp_path / "missing" + message = "must be absolute" if kind == "relative" else "must already exist" + + with pytest.raises(runner.CalibrationRunError, match=message): + runner._verify_ruler_receipt_directory_precondition(path) + + +def test_sealed_main_rejects_ruler_source_checkout_before_authentication( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source_checkout = tmp_path / "ruler-source-checkout" + (source_checkout / ".git").mkdir(parents=True) + (source_checkout / "scripts").mkdir() + (source_checkout / "README.md").write_bytes(b"RULER source checkout\n") + arguments = official_cli_arguments(tmp_path, ruler_receipts=source_checkout) + monkeypatch.setattr( + runner, + "_authenticate_sealed_runtime_context", + lambda *_args, **_kwargs: pytest.fail("runtime authentication must not begin"), + ) + monkeypatch.setattr( + runner, + "_official_main", + lambda *_args, **_kwargs: pytest.fail("materialization boundary must not be entered"), + ) + + with pytest.raises(runner.CalibrationRunError, match="receipt directory inventory drifted"): + runner.sealed_main( + arguments, + base_runtime_root=tmp_path / "unopened-base-runtime", + package_roots={"packages": tmp_path / "unopened-packages"}, + package_import_paths={"packages": "Lib/site-packages"}, + interpreter_path=tmp_path / "unopened-base-runtime" / "python.exe", + git_executable_path=tmp_path / "unopened-git.exe", + pycache_prefix=tmp_path / "unopened-pycache", + ) + + assert not (tmp_path / "runtime.json").exists() + assert not (tmp_path / "identity.json").exists() + assert not (tmp_path / "model-root").exists() + assert not (tmp_path / "output").exists() + + def test_public_main_binds_manifest_bytes_before_adapter_import( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -4171,6 +4412,7 @@ def test_public_main_binds_manifest_bytes_before_adapter_import( paths["runtime"].write_bytes(runtime_bytes) paths["model"].write_bytes(model_bytes) paths["parquet"].write_bytes(parquet_bytes) + receipt_dir = ruler_receipt_directory(tmp_path / "unopened-ruler-receipts") imported: list[bool] = [] monkeypatch.setattr(runner, "_load_adapter", lambda *args, **kwargs: imported.append(True)) @@ -4197,8 +4439,8 @@ def test_public_main_binds_manifest_bytes_before_adapter_import( str(tmp_path / "unopened-model"), "--cache-root", str(tmp_path / "unopened-cache"), - "--ruler-root", - str(tmp_path / "unopened-ruler"), + "--ruler-receipt-dir", + str(receipt_dir), "--repository-root", str(tmp_path / "unopened-repository"), "--source-commit",