translation bench

.gitattributes

@@ -3,28 +3,25 @@
 shrinkwrap.yaml              merge=binary
 npm-shrinkwrap.json          merge=binary
 yarn.lock                    merge=binary
-
 # pnpm-lock.yaml is regenerated on merge by a local driver that ships with the
 # repo (registered by ts/tools/scripts/setup-merge-driver.mjs on `pnpm install`).
 # Hosts without the driver (e.g. GitHub's server-side merges and the merge
 # queue) don't recognize the name and fall back to git's normal 3-way text
 # merge; CI's `pnpm install --frozen-lockfile` then catches any bad merge.
 pnpm-lock.yaml               merge=pnpm-lock
-
 # Autogenerated package READMEs carry a commit-stamped footer, so every branch
 # regenerates a different last line and they conflict constantly.  Keep our copy
 # on merge (the local keep-ours driver); the docs pipeline regenerates them.
 # Where the driver isn't registered the name is unknown and git falls back to a
 # text merge, and the heal-generated-files workflow clears the leftover conflict.
 README.AUTOGEN.md            merge=keep-ours
-
 # Make text consistently using LF
 * text eol=lf
-
 # Non-text files
 *.png -text
 *.jpg -text
 *.gif -text
 *.pdf -text
 *.bin -text
 *.db -text
+ts/packages/benchmarks/local/runs/1k-20260807-neg-fairness-fork/*.jsonl filter=lfs diff=lfs merge=lfs -text

android/samples/mobile-2/app/src/main/java/com/example/typeagentchat/AlarmActionParser.kt

@@ -60,7 +60,6 @@ internal fun parseSetAlarmActionPayload(data: Any?): SetAlarmAction? {
         days = days
     )
 }
-
 /**
  * Reads the optional `days` array.
  *
@@ -102,4 +101,3 @@ internal fun formatAlarmDays(days: List<Int>): String {
         .filter { it in days }
         .joinToString(", ") { labels.getValue(it) }
 }
-

ts/packages/actionSchema/src/generator.ts

@@ -127,6 +127,7 @@ export type GenerateSchemaOptions = {
     jsonSchemaFunction?: boolean; // default false
     jsonSchemaWithTs?: boolean; // default false, applies only when jsonSchema or jsonSchemaFunction is true.
     jsonSchemaValidate?: boolean; //default false, applies only when jsonSchema or jsonSchemaFunction is true.
+    validate?: boolean; // default true; validates parsed actions against their TypeScript schema.
 };
 
 function isJsonSchemaEnabled(options?: GenerateSchemaOptions): boolean {

ts/packages/aiclient/src/index.ts

@@ -19,6 +19,12 @@ export {
     type ChatModelTelemetryPurpose,
     type ChatModelTelemetryScope,
 } from "./chatModelTelemetryContext.js";
+export {
+    getModelCallSink,
+    withModelCallSink,
+    type ModelCallRecord,
+    type ModelCallSink,
+} from "./modelCallCapture.js";
 export * as openai from "./openai.js";
 export * as bing from "./bing.js";
 export * from "./restClient.js";

ts/packages/aiclient/src/modelCallCapture.ts

@@ -0,0 +1,37 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { AsyncLocalStorage } from "node:async_hooks";
+
+/**
+ * One non-streaming JSON translation call captured at ChatModel.complete.
+ * Used by benchmarks to write the model input, result, usage, and timing.
+ */
+export interface ModelCallRecord {
+    name: string;
+    request: unknown;
+    response: unknown;
+    usage?: unknown;
+    atMs: number;
+    durationMs: number;
+}
+
+/** Receives each record synchronously before translation returns. */
+export type ModelCallSink = (record: ModelCallRecord) => void;
+
+// AsyncLocalStorage (not OpenTelemetry context) so propagation works even when
+// no otel ContextManager is registered, e.g. headless benchmark runs.
+const modelCallSinkStore = new AsyncLocalStorage<ModelCallSink | undefined>();
+
+/** The sink active for the current async context, if any. */
+export function getModelCallSink(): ModelCallSink | undefined {
+    return modelCallSinkStore.getStore();
+}
+
+/** Run `body` with `sink` active for its non-streaming JSON model calls. */
+export function withModelCallSink<T>(
+    sink: ModelCallSink | undefined,
+    body: () => T,
+): T {
+    return modelCallSinkStore.run(sink, body);
+}

ts/packages/aiclient/test/modelCallCapture.spec.ts

@@ -0,0 +1,27 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import {
+    getModelCallSink,
+    withModelCallSink,
+    type ModelCallSink,
+} from "../src/modelCallCapture.js";
+
+describe("model call capture", () => {
+    it("scopes sinks across asynchronous and nested work", async () => {
+        const outer: ModelCallSink = () => {};
+        const inner: ModelCallSink = () => {};
+
+        expect(getModelCallSink()).toBeUndefined();
+        await withModelCallSink(outer, async () => {
+            expect(getModelCallSink()).toBe(outer);
+            await Promise.resolve();
+            expect(getModelCallSink()).toBe(outer);
+            withModelCallSink(inner, () => {
+                expect(getModelCallSink()).toBe(inner);
+            });
+            expect(getModelCallSink()).toBe(outer);
+        });
+        expect(getModelCallSink()).toBeUndefined();
+    });
+});

ts/packages/benchmarks/.gitignore

@@ -2,5 +2,7 @@ node_modules/
 dist/
 data/
 results/
+local/
+src/translationBench/public_datasets/breakthrough_findings/
 *.tsbuildinfo
 seal-tools-*.jsonl

ts/packages/benchmarks/AGENTS.md

@@ -0,0 +1,72 @@
+# @typeagent/benchmarks — agent notes
+
+## Layout
+
+- `src/core/` — domain-agnostic infrastructure.
+  - `rateLimiter.ts` — cross-process tokens-per-minute limiter (shared SQLite).
+  - `tokenEstimate.ts` — model-agnostic prompt token estimate for reservations.
+- `src/translationBench/`
+  - `runConfig.ts` + `config.schema.json` — pure JSON run-config loader/resolver.
+  - `synthesizer/` — dataset generation, quality gates, negative fairness.
+  - `runner/` — suite execution, scoring, checkpoints, reports, explainer.
+  - `policy/` — eligible-gold allowlist + action quality picker.
+  - `scripts/tbEval.ts`, `scripts/tbGenerate.ts` — thin production CLIs.
+- Assets (`config.schema.json`, prompt packs) are copied to `dist/` by
+  `scripts/copyAssets.mjs` during build.
+
+## Config: JSON + commander, no `TB_*` env
+
+Run configuration is a JSON file validated by `config.schema.json`. Runtime
+overrides are **commander flags**, prop-drilled into the library — do not read
+`process.env.TB_*`.
+
+```bash
+# eval (requires a pre-approved artifact; never auto-approves)
+node dist/translationBench/scripts/tbEval.js \
+  --draft ./artifacts/benchmark-draft-1000.jsonl \
+  --approved ./artifacts/benchmark-approved-1000.jsonl \
+  --config ./run-config.json \
+  --batch eval
+
+# generate
+node dist/translationBench/scripts/tbGenerate.js \
+  --source ./source/anchors.jsonl \
+  --manifest ./source/source-manifest.json \
+  --config ./run-config.json \
+  --batch synthesizer
+```
+
+`tb-eval` refuses to mint `approval.status: "approved"` and fails when draft
+content drifts from the approved file. See
+`src/translationBench/config/run-config.example.json`.
+
+## Credential env boundary
+
+`OPENAI_*` / `AZURE_*` env is the `@typeagent/aiclient` contract
+(`initRuntimeConfigFromProcessEnv()`) and is intentionally kept.
+
+## TPM rate limiter
+
+`createRateLimiter(tpmLimits, { dbPath, estTokensPerCall, maxWaitMs?, onWait? })`
+requires `dbPath`. Concurrent `run()` calls reserve tokens against the shared
+SQLite ledger over a rolling 60s window and settle to actual usage.
+
+## Runner library
+
+Import via package subpath (not star-exported from the main barrel — names
+overlap synthesizer checkpoint helpers):
+
+```ts
+import {
+  runTranslationBench,
+  scoreTranslationBench,
+} from "@typeagent/benchmarks/translationBench/runner";
+```
+
+Callers own dispatcher bootstrap (`initializeCommandHandlerContext`). The runner
+only crosses into agent-dispatcher at `translateRequest`.
+
+## local/ is gitignored
+
+Scratch run artifacts stay under `local/` (gitignored). Committed code lives
+under `src/`.

ts/packages/benchmarks/README.AUTOGEN.md

@@ -3,7 +3,7 @@
 
 <!-- AUTOGEN:DOCS:START -->
 
-<!-- AUTOGEN:DOCS:HASH:sha256=2787d4ae1307df74aa6e352806f8288e3096106e625aab4bae1304a92c6c5b14 -->
+<!-- AUTOGEN:DOCS:HASH:sha256=e1f7d3bf6c811d8ec2f4b5a91920017e587823fbe06393546dc91cd20fbf0eaf -->
 <!-- AUTOGEN:DOCS:SOURCE: ./README.md (hand-written documentation; this file is the AI-generated companion) -->
 
 # @typeagent/benchmarks — AI-generated documentation
@@ -23,12 +23,14 @@ TypeAgent translation bench: catalog, action-parameters grader, simple-action da
 - default → `./dist/index.js` _(not found on disk)_
 - `./translationBench` → `./dist/translationBench/index.js` _(not found on disk)_
 - `./internal` → `./dist/index.js` _(not found on disk)_
+- `./translationBench/runner` → `./dist/translationBench/runner/index.js` _(not found on disk)_
 
 ### Dependencies
 
 Workspace:
 
 - [@typeagent/action-schema](../../packages/actionSchema/README.md)
+- [@typeagent/agent-cache](../../packages/cache/README.md)
 - [@typeagent/agent-sdk](../../packages/agentSdk/README.md)
 - [@typeagent/aiclient](../../packages/aiclient/README.md)
 - [agent-dispatcher](../../packages/dispatcher/dispatcher/README.md)
@@ -44,18 +46,38 @@ _None._
 
 - [./src/index.ts](./src/index.ts)
 - [./src/translationBench/index.ts](./src/translationBench/index.ts)
-- [./src/translationBench/synthesizer/catalogGenerator/index.ts](./src/translationBench/synthesizer/catalogGenerator/index.ts)
+- [./src/translationBench/policy/index.ts](./src/translationBench/policy/index.ts)
+- [./src/translationBench/public_datasets/DroidCall/index.ts](./src/translationBench/public_datasets/DroidCall/index.ts)
+- [./src/translationBench/public_datasets/DroidCall/toTypeAgentSchema.ts](./src/translationBench/public_datasets/DroidCall/toTypeAgentSchema.ts)
+- [./src/translationBench/public_datasets/Seal-Tools/index.ts](./src/translationBench/public_datasets/Seal-Tools/index.ts)
+- [./src/translationBench/public_datasets/Seal-Tools/toTypeAgentSchema.ts](./src/translationBench/public_datasets/Seal-Tools/toTypeAgentSchema.ts)
+- [./src/translationBench/runner/index.ts](./src/translationBench/runner/index.ts)
 - [./src/translationBench/synthesizer/goldSchema.ts](./src/translationBench/synthesizer/goldSchema.ts)
 - [./src/translationBench/synthesizer/index.ts](./src/translationBench/synthesizer/index.ts)
-- [./src/core/model-prices.generated.json](./src/core/model-prices.generated.json)
-- [./src/core/paths.ts](./src/core/paths.ts)
-- [./src/core/prices.ts](./src/core/prices.ts)
-- [./src/core/rateLimiter.ts](./src/core/rateLimiter.ts)
-- [./src/core/tokenEstimate.ts](./src/core/tokenEstimate.ts)
-- _…and 55 more under `./src/`._
+- _…and 126 more under `./src/`._
+
+### Environment variables
+
+_15 environment variables referenced from `./src/` (set in `ts/.env` or your shell). See the `## Setup` section above for guidance on obtaining each value._
+
+- `DROIDCALL_CASE_IDS`
+- `DROIDCALL_MAX_CASES`
+- `DROIDCALL_MODELS`
+- `LITELLM_API_KEY`
+- `LITELLM_BASE_URL`
+- `LOCAL_LITELLM_API_KEY`
+- `LOCAL_LITELLM_OPENAI_BASE_URL`
+- `OPENAI_API_KEY`
+- `OPENAI_ENDPOINT`
+- `OPENAI_MODEL`
+- `OPENAI_MODEL_WIRE_API`
+- `SEAL_CASE_IDS`
+- `SEAL_MAX_CASES`
+- `SEAL_MODELS`
+- `TYPEAGENT_MODEL_PROVIDER`
 
 ---
 
-_Auto-generated against commit `844c37a32436d74cec201a1aab2d98944f6363f2` on `2026-08-28T06:40:41.825Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._
+_Auto-generated against commit `673a5348ac4d210dabe5bafe24ce10e24b290f2e` on `2026-09-03T06:14:18.659Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._
 
 <!-- AUTOGEN:DOCS:END -->

ts/packages/benchmarks/README.md

@@ -4,7 +4,14 @@ Action-translation eval for TypeAgent: pinned catalogs, model prices, and scorin
 
 ## Catalog + action-parameters grader
 
-Pinned `catalog.generated.json` and `action-parameters-grader.generated.json`. Code/script parameters use verify mode `llmAsAJudge` (not exact); synthesizer exclusions are derived from those fields. Regenerate with `pnpm run gen-catalog` (`--force` full rebuild). Tests: `pnpm run test:local`.
+Pinned `catalog.generated.json` and `action-parameters-grader.generated.json`.
+
+Human policy lives in `src/translationBench/policy/action-eligibility.json` (+ `.schema.json`):
+
+- **`removedActions`** — actions that must not be gold targets (`type: "action"` exact ids, or `type: "prefix"` `onboarding.*` only). They stay in the catalog for routing.
+- **`parameterOverrides`** — pin per-field **`verify`** only (`type: "field"`). `create` is never set in policy; type/regex derive minting. Override paths are skipped by the LLM classifier when regenerating the grader.
+
+Regenerate grader: `pnpm run gen-policy` (alias `gen-action-parameters-grader`). Full catalog+grader: `pnpm run gen-catalog`. Tests: `pnpm run test:local`.
 
 ## Dataset synthesizer (part 3)
 

ts/packages/benchmarks/local/runs/1k-20260807-neg-fairness-fork/README.md

@@ -0,0 +1,35 @@
+# 1k neg-fairness — fork with list.addItems value defects
+
+Fork of the negative-fairness benchmark of record, extended with two
+benchmark-proven `list.addItems` value-correctness defects.
+
+## Source
+
+Forked from `1k-20260807-neg-fairness/artifacts/benchmark-approved-1000.jsonl`
+(932 cases, 905 unique positive + 905 unique negative utterances, scored on
+`azure/gpt-5.4-nano`, `azure/gpt-4.1`, `azure/gpt-4.1-mini`).
+
+## Added
+
+50 new cases (100 generalizations, balanced 50 positive + 50 negative), each a
+single-action `list.addItems` request. The action is always correct; the defect
+is the `items[]` array content. Ground: `agents/list/src/listSchema.ts`
+`AddItemsAction` — `items: string[]` with no quantity field and no
+cardinality/dedupe contract.
+
+- **F1 — conjunction drop**: "Add socks, shirts, and pants..." collapses to
+  `["socks"]` or empties. nano 47.1%→22.2% repro, mini 8%→4%; mini control 0%.
+- **F2 — quantity expansion**: "Add 3 apples..." expands to
+  `["apples","apples","apples"]`; "a dozen eggs" → 12× "egg". nano 60%→80%
+  repro, mini 16%→12%; controls ≤4.8%.
+
+Both confirmed on both azure models across an initial ladder and an independent
+reproduction run, `history=undefined`, 100% `emptyHistoryProven`. Full evidence:
+`500-fpr-repro/artifacts/value-probes/`.
+
+## File
+
+`benchmark-approved-1000-plus-list-defects.jsonl` — same record format as the
+source (one metadata record, then case records). New cases carry
+`dimensions.issue` = `F1-list-conjunction-drop` | `F2-list-quantity-expansion`.
+Tracked with Git LFS.

ts/packages/benchmarks/local/runs/1k-20260807-neg-fairness-fork/provenance.json

@@ -0,0 +1,14 @@
+{
+  "forkedFrom": "1k-20260807-neg-fairness/artifacts/benchmark-approved-1000.jsonl",
+  "sourceCases": 932,
+  "sourceUniquePositive": 905,
+  "sourceUniqueNegative": 905,
+  "sourceModels": ["azure/gpt-5.4-nano", "azure/gpt-4.1", "azure/gpt-4.1-mini"],
+  "addedCases": 50,
+  "addedPositive": 50,
+  "addedNegative": 50,
+  "issues": ["F1-list-conjunction-drop", "F2-list-quantity-expansion"],
+  "sourceContract": "agents/list/src/listSchema.ts AddItemsAction (items:string[]; no quantity/cardinality contract)",
+  "evidence": "500-fpr-repro/artifacts/value-probes/ (ladder + reproduction, both azure models, emptyHistoryProven)",
+  "sha256": "9bced517157a3c6eee1ed59e59c491e5fa18c1195b1b6a63b3097407124ade81"
+}

ts/packages/benchmarks/output/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

ts/packages/benchmarks/package.json

@@ -14,7 +14,8 @@
   "exports": {
     ".": "./dist/index.js",
     "./translationBench": "./dist/translationBench/index.js",
-    "./internal": "./dist/index.js"
+    "./internal": "./dist/index.js",
+    "./translationBench/runner": "./dist/translationBench/runner/index.js"
   },
   "files": [
     "dist",
@@ -24,18 +25,28 @@
     "build": "tsc -b && node ./scripts/copyAssets.mjs",
     "clean": "node ./scripts/clean.mjs",
     "download-seal-tools": "pnpm run build && node dist/translationBench/public_datasets/Seal-Tools/getDataset.js seal-tools-validation.jsonl",
-    "gen-action-parameters-grader": "pnpm run build && node --max-old-space-size=4096 dist/translationBench/scripts/genActionParametersGrader.js",
-    "gen-catalog": "pnpm run build && node --max-old-space-size=4096 dist/translationBench/scripts/genCatalog.js && node --max-old-space-size=4096 dist/translationBench/scripts/genActionParametersGrader.js",
+    "droidcall-eval": "node ./dist/translationBench/public_datasets/DroidCall/eval/runEval.js",
+    "droidcall-eval-smoke": "node ./dist/translationBench/public_datasets/DroidCall/eval/test-run.js",
+    "droidcall-rescore": "node ./dist/translationBench/public_datasets/DroidCall/eval/rescoreResults.js",
+    "gen-action-parameters-grader": "pnpm run gen-policy",
+    "gen-catalog": "pnpm run build && node --max-old-space-size=4096 dist/translationBench/scripts/genCatalog.js && node --max-old-space-size=4096 dist/translationBench/scripts/genPolicy.js && node --max-old-space-size=4096 dist/translationBench/scripts/pickEligibleActions.js --model ${TB_PICKER_MODEL:-azure/gpt-5.6-sol} && node ./scripts/copyAssets.mjs",
+    "gen-policy": "pnpm run build && node --max-old-space-size=4096 dist/translationBench/scripts/genPolicy.js && node ./scripts/copyAssets.mjs",
     "jest-esm": "node --no-warnings --experimental-vm-modules ./node_modules/jest/bin/jest.js",
     "merge-checkpoints": "pnpm run build && node dist/translationBench/scripts/mergeCheckpoints.js",
+    "pick-eligible-actions": "pnpm run build && node --max-old-space-size=4096 dist/translationBench/scripts/pickEligibleActions.js --model ${TB_PICKER_MODEL:-azure/gpt-5.6-sol} && node ./scripts/copyAssets.mjs",
     "prettier": "prettier --check package.json tsconfig.json src scripts test --ignore-path ../../.prettierignore",
     "prettier:fix": "prettier --write package.json tsconfig.json src scripts test --ignore-path ../../.prettierignore",
+    "seal-eval": "node ./dist/translationBench/public_datasets/Seal-Tools/eval/runEval.js",
+    "seal-eval-smoke": "node ./dist/translationBench/public_datasets/Seal-Tools/eval/test-run.js",
+    "tb-eval": "node ./dist/translationBench/scripts/tbEval.js",
+    "tb-generate": "node ./dist/translationBench/scripts/tbGenerate.js",
     "test": "npm run test:local",
     "test:local": "pnpm run build && pnpm run jest-esm --testPathPattern=\".*[.]spec[.]js\"",
     "tsc": "tsc -b"
   },
   "dependencies": {
     "@typeagent/action-schema": "workspace:*",
+    "@typeagent/agent-cache": "workspace:*",
     "@typeagent/agent-sdk": "workspace:*",
     "@typeagent/aiclient": "workspace:*",
     "agent-dispatcher": "workspace:*",