diff --git a/ts/packages/benchmarks/README.AUTOGEN.md b/ts/packages/benchmarks/README.AUTOGEN.md index c9da3e3713..b1a1798ea8 100644 --- a/ts/packages/benchmarks/README.AUTOGEN.md +++ b/ts/packages/benchmarks/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # @typeagent/benchmarks — AI-generated documentation @@ -52,10 +52,10 @@ _None._ - [./src/core/prices.ts](./src/core/prices.ts) - [./src/core/rateLimiter.ts](./src/core/rateLimiter.ts) - [./src/core/tokenEstimate.ts](./src/core/tokenEstimate.ts) -- _…and 53 more under `./src/`._ +- _…and 54 more under `./src/`._ --- -_Auto-generated against commit `493931d64a0633ed8909c52b3e84ff536fdab1ce` on `2026-08-27T00:20:31.457Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ +_Auto-generated against commit `0acfc92bdba7c88a3026b17cc7048233458d67a4` on `2026-08-27T08:27:30.753Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ diff --git a/ts/packages/benchmarks/package.json b/ts/packages/benchmarks/package.json index 7b3b4a5b2c..ffac625195 100644 --- a/ts/packages/benchmarks/package.json +++ b/ts/packages/benchmarks/package.json @@ -27,6 +27,7 @@ "gen-action-parameters-grader": "pnpm run build && node --max-old-space-size=4096 dist/translationBench/scripts/genActionParametersGrader.js", "gen-catalog": "pnpm run build && node --max-old-space-size=4096 dist/translationBench/scripts/genCatalog.js && node --max-old-space-size=4096 dist/translationBench/scripts/genActionParametersGrader.js", "jest-esm": "node --no-warnings --experimental-vm-modules ./node_modules/jest/bin/jest.js", + "merge-checkpoints": "pnpm run build && node dist/translationBench/scripts/mergeCheckpoints.js", "prettier": "prettier --check package.json tsconfig.json src scripts test --ignore-path ../../.prettierignore", "prettier:fix": "prettier --write package.json tsconfig.json src scripts test --ignore-path ../../.prettierignore", "test": "npm run test:local", diff --git a/ts/packages/benchmarks/src/translationBench/scripts/mergeCheckpoints.ts b/ts/packages/benchmarks/src/translationBench/scripts/mergeCheckpoints.ts new file mode 100644 index 0000000000..7cfa7f913b --- /dev/null +++ b/ts/packages/benchmarks/src/translationBench/scripts/mergeCheckpoints.ts @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { Command } from "commander"; +import { + mergeTranslationBenchCheckpoints, + readTranslationBenchCheckpoint, +} from "../synthesizer/generationSupport.js"; + +const files = new Command().argument("").parse().args; +const merged = mergeTranslationBenchCheckpoints( + files.map((filePath) => readTranslationBenchCheckpoint(filePath)), +); +process.stdout.write(`${JSON.stringify(merged.rows, null, 2)}\n`); diff --git a/ts/packages/benchmarks/src/translationBench/synthesizer/generationSupport.ts b/ts/packages/benchmarks/src/translationBench/synthesizer/generationSupport.ts index 880fba5495..3cc93959c2 100644 --- a/ts/packages/benchmarks/src/translationBench/synthesizer/generationSupport.ts +++ b/ts/packages/benchmarks/src/translationBench/synthesizer/generationSupport.ts @@ -338,6 +338,60 @@ export function appendTranslationBenchCheckpointRows( }; } +export function mergeTranslationBenchCheckpoints( + checkpoints: readonly TranslationBenchCheckpoint[], +): TranslationBenchCheckpoint { + if (checkpoints.length === 0) { + throw new Error("No translation bench checkpoints to merge"); + } + const first = checkpoints[0]!.header; + const byShard = new Map>(); + for (const checkpoint of checkpoints) { + assertTranslationBenchCheckpointHeadersCompatible(checkpoint.header, { + ...first, + shardIndex: checkpoint.header.shardIndex, + }); + if (byShard.has(checkpoint.header.shardIndex)) { + throw new Error( + `Duplicate translation bench checkpoint shard ${checkpoint.header.shardIndex}`, + ); + } + byShard.set(checkpoint.header.shardIndex, checkpoint); + } + + const rows: TranslationBenchCheckpointRow[] = []; + const resumeKeys = new Set(); + for (let index = 0; index < first.shardCount; index++) { + const checkpoint = byShard.get(index); + if (checkpoint === undefined) { + throw new Error(`Missing checkpoint shard: ${index}`); + } + for (const row of checkpoint.rows) { + const normalized = parseTranslationBenchCheckpointRow(row); + validateTranslationBenchCheckpointRowShard( + normalized, + checkpoint.header, + ); + const key = translationBenchResumeKey(normalized); + if (resumeKeys.has(key)) { + throw new Error( + `Duplicate translation bench resume key '${key}'`, + ); + } + resumeKeys.add(key); + rows.push(normalized); + } + } + rows.sort((left, right) => + compareText( + translationBenchResumeKey(left), + translationBenchResumeKey(right), + ), + ); + + return { header: byShard.get(0)!.header, rows, resumeKeys }; +} + export function getTranslationBenchCatalogCensus( schemas: readonly TranslationBenchBenchmarkSchema[], ): TranslationBenchCatalogCensus {