From 161c38999a604be187b87525e239ff6982d8e443 Mon Sep 17 00:00:00 2001 From: Dominic Nguyen Date: Wed, 26 Aug 2026 16:14:12 -0700 Subject: [PATCH 01/10] feat(droidcall): add standalone contract scorer --- .../DroidCall/eval/officialDroidCallGrader.py | 308 ++++++++++++++++++ 1 file changed, 308 insertions(+) create mode 100644 ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/eval/officialDroidCallGrader.py diff --git a/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/eval/officialDroidCallGrader.py b/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/eval/officialDroidCallGrader.py new file mode 100644 index 0000000000..12078d1b2f --- /dev/null +++ b/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/eval/officialDroidCallGrader.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +"""DroidCall paper, released-code, and TypeAgent-adjusted scorers.""" + +import json +import re +import sys + + +_semantic_scorer = None +_semantic_scores = {} + + +def is_field_none(value): + return value is None or ( + isinstance(value, str) and value.strip().lower() == "none" + ) + + +def decode_number_lexemes(value): + if isinstance(value, list): + return [decode_number_lexemes(item) for item in value] + if isinstance(value, dict): + if set(value) == {"__pythonNumber"}: + source = value["__pythonNumber"] + return int(source) if re.fullmatch(r"[+-]?\d+", source) else float(source) + return {key: decode_number_lexemes(item) for key, item in value.items()} + return value + + +def collect_semantic_pairs(left, right, match_type, pairs): + if match_type != "semantic" or is_field_none(left) or is_field_none(right): + return + if type(left) is not type(right): + return + if isinstance(left, dict): + if len(left) != len(right): + return + for key, value in left.items(): + if key in right: + collect_semantic_pairs(value, right[key], match_type, pairs) + elif isinstance(left, list): + if len(left) != len(right): + return + for left_item in left: + for right_item in right: + collect_semantic_pairs(left_item, right_item, match_type, pairs) + elif isinstance(left, str): + pairs.add((left, right)) + + +def prepare_semantic_scores(pairs): + global _semantic_scorer + missing = [pair for pair in pairs if pair not in _semantic_scores] + if not missing: + return + if _semantic_scorer is None: + from bert_score import BERTScorer + + _semantic_scorer = BERTScorer(lang="en") + _, _, scores = _semantic_scorer.score( + [pair[0] for pair in missing], + [pair[1] for pair in missing], + ) + for pair, score in zip(missing, scores): + _semantic_scores[pair] = float(score) + + +def deep_compare(left, right, match_type="strict", semantic_threshold=0.85): + if match_type == "ignore": + return True + if is_field_none(left) and is_field_none(right): + return True + # Semantic pair collection skips one-sided sentinel values. + if is_field_none(left) or is_field_none(right): + return False + if type(left) is not type(right): + return False + if isinstance(left, dict): + if len(left) != len(right): + return False + return all( + key in right + and deep_compare(value, right[key], match_type, semantic_threshold) + for key, value in left.items() + ) + if isinstance(left, list): + if len(left) != len(right): + return False + # Consume each right-hand item once so duplicate values remain significant. + remaining = list(right) + for item in left: + match = next( + ( + index + for index, candidate in enumerate(remaining) + if deep_compare(item, candidate, match_type, semantic_threshold) + ), + None, + ) + if match is None: + return False + remaining.pop(match) + return True + if isinstance(left, str): + if match_type == "strict": + return left.strip().lower() == right.strip().lower() + return _semantic_scores[(left, right)] > semantic_threshold + if isinstance(left, (int, float)): + return left == right + return False + + +def resolved_arguments(answer, response, api): + for name, spec in api["arguments"].items(): + answer_has = name in answer["arguments"] + response_has = name in response["arguments"] + if not answer_has and not response_has: + continue + if spec.get("required", False) and not answer_has: + continue + default = spec.get("default") + yield ( + answer["arguments"].get(name, default), + response["arguments"].get(name, default), + spec.get("match_type", "strict"), + ) + + +def score_payload(payload, contract_name): + if contract_name == "paper-described": + semantic_threshold = 0.75 + aggregation = "function-call-mean" + mime_presence_only = False + elif contract_name == "released": + semantic_threshold = 0.85 + aggregation = "sample-mean" + mime_presence_only = False + elif contract_name == "typeagent-adjusted": + semantic_threshold = 0.85 + aggregation = "sample-mean" + mime_presence_only = True + else: + raise ValueError(f"Unknown DroidCall scoring contract: {contract_name}") + + apis = {item["name"]: item for item in payload["apis"]} + for row in payload["rows"]: + # Ignore malformed response entries; they cannot represent a scored call. + for response in row["response"]: + if isinstance(response, dict) and isinstance( + response.get("arguments"), dict + ): + response["arguments"] = decode_number_lexemes( + response["arguments"] + ) + pairs = set() + prepared = [] + for row in payload["rows"]: + response_map = { + item.get("name", ""): item + for item in row["response"] + if ( + isinstance(item, dict) + and isinstance(item.get("name", ""), str) + and isinstance(item.get("arguments"), dict) + ) + } + prepared.append(response_map) + for answer in row["answers"]: + response = response_map.get(answer["name"]) + if response is None: + continue + for left, right, match_type in resolved_arguments( + answer, response, apis[answer["name"]] + ): + collect_semantic_pairs(left, right, match_type, pairs) + prepare_semantic_scores(pairs) + + row_soft_total = 0.0 + call_soft_total = 0.0 + call_count = 0 + perfect_rows = 0 + correct_arguments = 0 + total_arguments = 0 + for row, response_map in zip(payload["rows"], prepared): + row_correct = 0 + row_total = 0 + missing_calls = 0 + for answer in row["answers"]: + api = apis[answer["name"]] + response = response_map.get(answer["name"]) + call_correct = 0 + call_total = 0 + if response is None: + missing_calls += 1 + call_total = len(api["arguments"]) + row_total += call_total + call_soft_total += 0.0 + call_count += 1 + continue + for name, spec in api["arguments"].items(): + answer_has = name in answer["arguments"] + response_has = name in response["arguments"] + if ( + mime_presence_only + and api["name"] == "ACTION_OPEN_DOCUMENT" + and name == "mime_types" + ): + if answer_has and response_has: + row_correct += 1 + call_correct += 1 + row_total += 1 + call_total += 1 + continue + if not answer_has and not response_has: + row_correct += 1 + call_correct += 1 + row_total += 1 + call_total += 1 + continue + if spec.get("required", False) and not answer_has: + row_total += 1 + call_total += 1 + continue + default = spec.get("default") + if deep_compare( + answer["arguments"].get(name, default), + response["arguments"].get(name, default), + spec.get("match_type", "strict"), + semantic_threshold, + ): + row_correct += 1 + call_correct += 1 + row_total += 1 + call_total += 1 + call_soft_total += ( + 1.0 if call_total == 0 else call_correct / call_total + ) + call_count += 1 + row_score = ( + 0.0 + if missing_calls > 0 + else 1.0 + if row_total == 0 + else row_correct / row_total + ) + row_soft_total += row_score + if abs(row_score - 1.0) < 1e-6: + perfect_rows += 1 + correct_arguments += row_correct + total_arguments += row_total + row_count = len(payload["rows"]) + soft_accuracy = ( + call_soft_total / call_count + if aggregation == "function-call-mean" and call_count > 0 + else row_soft_total / row_count + if row_count > 0 + else 0.0 + ) + overrides = [] + if mime_presence_only: + overrides.append( + { + "tool": "ACTION_OPEN_DOCUMENT", + "argument": "mime_types", + "comparison": "presence-only", + } + ) + return { + "softAccuracy": soft_accuracy, + "accuracy": perfect_rows / row_count if row_count > 0 else 0.0, + "counts": { + "rows": row_count, + "perfectRows": perfect_rows, + "correctArguments": correct_arguments, + "totalArguments": total_arguments, + "functionCalls": call_count, + }, + "contract": { + "name": contract_name, + "scorerRevision": "3f7ba458bee480a86c602edff6cc7ec9cfd555db", + "bertScore": "0.3.13", + "transformers": "4.48.1", + "semanticThreshold": semantic_threshold, + "softAccuracyAggregation": aggregation, + "overrides": overrides, + }, + } + + +def main(): + if "--jsonl" in sys.argv: + for line in sys.stdin: + try: + payload = json.loads(line) + contract_name = payload.pop("contract", "released") + print( + json.dumps(score_payload(payload, contract_name)), flush=True + ) + except Exception as error: + print(json.dumps({"error": str(error)}), flush=True) + return + payload = json.load(sys.stdin) + contract_name = payload.pop("contract", "released") + print(json.dumps(score_payload(payload, contract_name))) + + +if __name__ == "__main__": + main() From 5bfb17ba813e6eaaf624fcb58e1090ffa117ca9c Mon Sep 17 00:00:00 2001 From: typeagent-bot Date: Wed, 26 Aug 2026 23:19:47 +0000 Subject: [PATCH 02/10] style: apply prettier formatting and policy fixes --- .../public_datasets/DroidCall/eval/officialDroidCallGrader.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/eval/officialDroidCallGrader.py b/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/eval/officialDroidCallGrader.py index 12078d1b2f..e2b67aa5b4 100644 --- a/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/eval/officialDroidCallGrader.py +++ b/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/eval/officialDroidCallGrader.py @@ -1,4 +1,7 @@ #!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + """DroidCall paper, released-code, and TypeAgent-adjusted scorers.""" import json From c2b6b4ffc29ad962c28c4a5c605ed4774df58a60 Mon Sep 17 00:00:00 2001 From: Dominic Nguyen Date: Wed, 26 Aug 2026 16:42:53 -0700 Subject: [PATCH 03/10] feat(droidcall): add dataset analysis command --- .../public_datasets/DroidCall/analyze.ts | 264 +++++++++++++++ .../DroidCall/eval/officialDroidCallGrader.py | 311 ------------------ .../public_datasets/DroidCall/index.ts | 35 ++ 3 files changed, 299 insertions(+), 311 deletions(-) create mode 100644 ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts delete mode 100644 ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/eval/officialDroidCallGrader.py create mode 100644 ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/index.ts diff --git a/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts b/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts new file mode 100644 index 0000000000..226fb2ef91 --- /dev/null +++ b/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts @@ -0,0 +1,264 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { createHash } from "node:crypto"; +import { mkdir, readFile, stat, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { + classifyDroidCalls, + hasDroidCallResultReference, + parseDroidCallCode, + type DroidCall, + type DroidCallShape, +} from "../pythonLiteral.js"; +import { DROIDCALL_SOURCE, readDroidCallJsonl } from "../huggingFaceRows.js"; + +interface CanonicalRow { + query: string; + answers: DroidCall[]; + tools: unknown[]; +} + +interface ChatRow { + messages: { role: string; content: string }[]; +} + +interface Bucket { + rows: number; + percent: number; +} + +interface SplitAnalysis { + rows: number; + calls: number; + buckets: Record; + callCountDistribution: Record; + nestedConsumerTools: Record; +} + +interface SourceFileInfo { + bytes: number; + sha256: string; +} + +const DROIDCALL_SOURCE_SHA256 = { + "DroidCall_code_short.jsonl": + "263e79dbc060fa5c228dbeb835b89e04087c0a723904b5704a82c86001feb7b1", + "DroidCall_train.jsonl": + "4cb2d5691c1b95b0908c59efcb361c8a9c9b12f0a4d182acfc2df0ccc92e6d3b", + "DroidCall_test.jsonl": + "d7e40ce794c98984befb872d2d71ee28511938c0baef881b14098575cda151f2", + "annotated_api.jsonl": + "29c4791f7a496e587af74b1a1398864bd6c6de9db767cb97606525d62ec05951", + "README.md": + "08a6c5cfa655e1ecd774a41a9a815e845ad0afd47b289378bc1d6ee66c81dda9", + ".gitattributes": + "74a8a09003e5506f7f0d9f5571d9ec05fba960e14e08f8eeb20aab0c429d2303", + "figures/data_generation.png": + "a126a0caebabfb48b80815a81e2d23ccc80a82c13adba62d23ab339ba5061313", + "figures/intent.png": + "8d67eb492ed2c05498581fb9a6842ed899155530f25c18ddf8f5929bc63e157b", +} satisfies Record<(typeof DROIDCALL_SOURCE.files)[number], string>; + +const percent = (part: number, total: number): number => + total === 0 ? 0 : Number(((part * 100) / total).toFixed(2)); + +function analyzeSplit(rows: CanonicalRow[]): SplitAnalysis { + const bucketCounts: Record = { + noCall: 0, + singleTool: 0, + multiCallNested: 0, + multiCallWithoutNested: 0, + }; + const distribution = new Map(); + const nestedConsumers = new Map(); + let calls = 0; + for (const row of rows) { + calls += row.answers.length; + bucketCounts[classifyDroidCalls(row.answers)]++; + distribution.set( + row.answers.length, + (distribution.get(row.answers.length) ?? 0) + 1, + ); + for (const call of row.answers) { + if (hasDroidCallResultReference(call.arguments)) { + nestedConsumers.set( + call.name, + (nestedConsumers.get(call.name) ?? 0) + 1, + ); + } + } + } + return { + rows: rows.length, + calls, + buckets: Object.fromEntries( + Object.entries(bucketCounts).map(([key, count]) => [ + key, + { rows: count, percent: percent(count, rows.length) }, + ]), + ) as Record, + callCountDistribution: Object.fromEntries( + [...distribution].sort(([left], [right]) => left - right), + ), + nestedConsumerTools: Object.fromEntries( + [...nestedConsumers].sort((left, right) => right[1] - left[1]), + ), + }; +} + +function canonicalize(value: unknown): unknown { + if (Array.isArray(value)) return value.map(canonicalize); + if (typeof value === "object" && value !== null) { + return Object.fromEntries( + Object.entries(value) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, item]) => [key, canonicalize(item)]), + ); + } + return value; +} + +function sameCalls(left: DroidCall[], right: DroidCall[]): boolean { + return ( + JSON.stringify(canonicalize(left)) === + JSON.stringify(canonicalize(right)) + ); +} + +async function sourceFileInfo( + rawDir: string, +): Promise> { + const result: Record = {}; + for (const relativePath of DROIDCALL_SOURCE.files) { + const path = join(rawDir, relativePath); + const [contents, fileStat] = await Promise.all([ + readFile(path), + stat(path), + ]); + const sha256 = createHash("sha256").update(contents).digest("hex"); + if (sha256 !== DROIDCALL_SOURCE_SHA256[relativePath]) { + throw new Error( + `${relativePath} does not match DroidCall revision ${DROIDCALL_SOURCE.revision}`, + ); + } + result[relativePath] = { bytes: fileStat.size, sha256 }; + } + return result; +} + +function markdown(report: DroidCallAnalysis): string { + const row = (name: string, bucket: Bucket) => + `| ${name} | ${bucket.rows.toLocaleString()} | ${bucket.percent.toFixed(2)}% |`; + const sections = Object.entries(report.splits) + .map(([name, split]) => { + const heading = + name === "full" + ? "Full dataset" + : name === "train" + ? "Training split" + : "Test split"; + const multiCallRows = + split.buckets.multiCallNested.rows + + split.buckets.multiCallWithoutNested.rows; + const nestedShareOfMultiCall = percent( + split.buckets.multiCallNested.rows, + multiCallRows, + ); + const distribution = Object.entries(split.callCountDistribution) + .map(([calls, rows]) => `${calls}: ${rows.toLocaleString()}`) + .join(", "); + return `## ${heading}\n\n${split.rows.toLocaleString()} rows contain ${split.calls.toLocaleString()} calls. ${multiCallRows.toLocaleString()} rows (${percent(multiCallRows, split.rows).toFixed(2)}%) have more than one call. Of those multi-call rows, ${nestedShareOfMultiCall.toFixed(2)}% pass a prior result into a later call.\n\n| Shape | Rows | Share of rows |\n| --- | ---: | ---: |\n${row("No call", split.buckets.noCall)} +${row("Single tool", split.buckets.singleTool)}\n${row("Multi-call, nested", split.buckets.multiCallNested)}\n${row("Multi-call, without nesting", split.buckets.multiCallWithoutNested)}\n\nRows by call count: ${distribution}.`; + }) + .join("\n\n"); + const snapshotBytes = Object.values(report.source.files).reduce( + (sum, file) => sum + file.bytes, + 0, + ); + const full = report.splits.full!; + const fullMultiCallRows = + full.buckets.multiCallNested.rows + + full.buckets.multiCallWithoutNested.rows; + return `# DroidCall data analysis\n\nThe full DroidCall dataset has ${full.rows.toLocaleString()} rows and ${full.calls.toLocaleString()} gold calls. Single-tool requests account for ${full.buckets.singleTool.percent.toFixed(2)}% of rows. The other ${percent(fullMultiCallRows, full.rows).toFixed(2)}% are multi-call requests; ${percent(full.buckets.multiCallNested.rows, fullMultiCallRows).toFixed(2)}% of those pass a prior result into a later call.\n\nSource: [mllmTeam/DroidCall](https://huggingface.co/datasets/mllmTeam/DroidCall), revision \`${report.source.revision}\`. The local snapshot has ${Object.keys(report.source.files).length} files (${(snapshotBytes / 1024 / 1024).toFixed(2)} MiB). It includes every file listed by the HuggingFace repository at that revision.\n\n## Classification\n\nThe analysis reads the structured \`answers\` in \`DroidCall_train.jsonl\` and \`DroidCall_test.jsonl\`. The buckets are mutually exclusive:\n\n- No call: no gold calls. +- Single tool: exactly one gold call.\n- Multi-call, nested: at least two calls and an argument contains a \`#N\` result reference. The reference can occur inside an array or object.\n- Multi-call, without nesting: at least two calls and no argument contains a result reference.\n\n${sections}\n\n## Parser reuse and validation\n\nDroidCall's assistant output uses Python-like function calls. \`parseDroidCallCode()\` handles the assignment and call syntax, then delegates strings, numbers, booleans, nulls, arrays, and objects to Seal-Tools' existing \`parsePythonLiteral()\`. This keeps one literal parser for both datasets.\n\nThe code-format file covers the ${report.parserValidation.rows.toLocaleString()} training rows. Parsed calls exactly match the canonical structured answers for ${report.parserValidation.exactMatches.toLocaleString()} rows (${percent(report.parserValidation.exactMatches, report.parserValidation.rows).toFixed(2)}%). There are ${report.parserValidation.parseFailures} parse failures and ${report.parserValidation.mismatches} source mismatches. Source mismatches include values that the code syntax cannot reproduce, such as a sentence-like function name or an argument key with leading whitespace.\n`; +} + +export interface DroidCallAnalysis { + source: { + dataset: string; + revision: string; + files: Record; + }; + splits: Record; + parserValidation: { + rows: number; + exactMatches: number; + parseFailures: number; + mismatches: number; + }; +} + +export async function analyzeDroidCall( + outputDir: string, +): Promise { + const rawDir = join(outputDir, "raw"); + const [train, test, chat] = await Promise.all([ + readDroidCallJsonl(join(rawDir, "DroidCall_train.jsonl")), + readDroidCallJsonl(join(rawDir, "DroidCall_test.jsonl")), + readDroidCallJsonl(join(rawDir, "DroidCall_code_short.jsonl")), + ]); + if (chat.length !== train.length) { + throw new Error( + `DroidCall code and train row counts differ: ${chat.length} !== ${train.length}`, + ); + } + let exactMatches = 0; + let parseFailures = 0; + let mismatches = 0; + for (let index = 0; index < chat.length; index++) { + const assistant = [...chat[index]!.messages] + .reverse() + .find((message) => message.role === "assistant"); + if (assistant === undefined) { + parseFailures++; + continue; + } + try { + const parsed = parseDroidCallCode(assistant.content); + if (sameCalls(parsed, train[index]!.answers)) exactMatches++; + else mismatches++; + } catch { + parseFailures++; + } + } + const report: DroidCallAnalysis = { + source: { + dataset: DROIDCALL_SOURCE.dataset, + revision: DROIDCALL_SOURCE.revision, + files: await sourceFileInfo(rawDir), + }, + splits: { + full: analyzeSplit([...train, ...test]), + train: analyzeSplit(train), + test: analyzeSplit(test), + }, + parserValidation: { + rows: chat.length, + exactMatches, + parseFailures, + mismatches, + }, + }; + const docsDir = join(outputDir, "docs"); + await mkdir(docsDir, { recursive: true }); + await Promise.all([ + writeFile( + join(outputDir, "analysis.json"), + JSON.stringify(report, null, 2) + "\n", + ), + writeFile(join(docsDir, "DroidCall.md"), markdown(report)), + ]); + return report; +} diff --git a/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/eval/officialDroidCallGrader.py b/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/eval/officialDroidCallGrader.py deleted file mode 100644 index e2b67aa5b4..0000000000 --- a/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/eval/officialDroidCallGrader.py +++ /dev/null @@ -1,311 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -"""DroidCall paper, released-code, and TypeAgent-adjusted scorers.""" - -import json -import re -import sys - - -_semantic_scorer = None -_semantic_scores = {} - - -def is_field_none(value): - return value is None or ( - isinstance(value, str) and value.strip().lower() == "none" - ) - - -def decode_number_lexemes(value): - if isinstance(value, list): - return [decode_number_lexemes(item) for item in value] - if isinstance(value, dict): - if set(value) == {"__pythonNumber"}: - source = value["__pythonNumber"] - return int(source) if re.fullmatch(r"[+-]?\d+", source) else float(source) - return {key: decode_number_lexemes(item) for key, item in value.items()} - return value - - -def collect_semantic_pairs(left, right, match_type, pairs): - if match_type != "semantic" or is_field_none(left) or is_field_none(right): - return - if type(left) is not type(right): - return - if isinstance(left, dict): - if len(left) != len(right): - return - for key, value in left.items(): - if key in right: - collect_semantic_pairs(value, right[key], match_type, pairs) - elif isinstance(left, list): - if len(left) != len(right): - return - for left_item in left: - for right_item in right: - collect_semantic_pairs(left_item, right_item, match_type, pairs) - elif isinstance(left, str): - pairs.add((left, right)) - - -def prepare_semantic_scores(pairs): - global _semantic_scorer - missing = [pair for pair in pairs if pair not in _semantic_scores] - if not missing: - return - if _semantic_scorer is None: - from bert_score import BERTScorer - - _semantic_scorer = BERTScorer(lang="en") - _, _, scores = _semantic_scorer.score( - [pair[0] for pair in missing], - [pair[1] for pair in missing], - ) - for pair, score in zip(missing, scores): - _semantic_scores[pair] = float(score) - - -def deep_compare(left, right, match_type="strict", semantic_threshold=0.85): - if match_type == "ignore": - return True - if is_field_none(left) and is_field_none(right): - return True - # Semantic pair collection skips one-sided sentinel values. - if is_field_none(left) or is_field_none(right): - return False - if type(left) is not type(right): - return False - if isinstance(left, dict): - if len(left) != len(right): - return False - return all( - key in right - and deep_compare(value, right[key], match_type, semantic_threshold) - for key, value in left.items() - ) - if isinstance(left, list): - if len(left) != len(right): - return False - # Consume each right-hand item once so duplicate values remain significant. - remaining = list(right) - for item in left: - match = next( - ( - index - for index, candidate in enumerate(remaining) - if deep_compare(item, candidate, match_type, semantic_threshold) - ), - None, - ) - if match is None: - return False - remaining.pop(match) - return True - if isinstance(left, str): - if match_type == "strict": - return left.strip().lower() == right.strip().lower() - return _semantic_scores[(left, right)] > semantic_threshold - if isinstance(left, (int, float)): - return left == right - return False - - -def resolved_arguments(answer, response, api): - for name, spec in api["arguments"].items(): - answer_has = name in answer["arguments"] - response_has = name in response["arguments"] - if not answer_has and not response_has: - continue - if spec.get("required", False) and not answer_has: - continue - default = spec.get("default") - yield ( - answer["arguments"].get(name, default), - response["arguments"].get(name, default), - spec.get("match_type", "strict"), - ) - - -def score_payload(payload, contract_name): - if contract_name == "paper-described": - semantic_threshold = 0.75 - aggregation = "function-call-mean" - mime_presence_only = False - elif contract_name == "released": - semantic_threshold = 0.85 - aggregation = "sample-mean" - mime_presence_only = False - elif contract_name == "typeagent-adjusted": - semantic_threshold = 0.85 - aggregation = "sample-mean" - mime_presence_only = True - else: - raise ValueError(f"Unknown DroidCall scoring contract: {contract_name}") - - apis = {item["name"]: item for item in payload["apis"]} - for row in payload["rows"]: - # Ignore malformed response entries; they cannot represent a scored call. - for response in row["response"]: - if isinstance(response, dict) and isinstance( - response.get("arguments"), dict - ): - response["arguments"] = decode_number_lexemes( - response["arguments"] - ) - pairs = set() - prepared = [] - for row in payload["rows"]: - response_map = { - item.get("name", ""): item - for item in row["response"] - if ( - isinstance(item, dict) - and isinstance(item.get("name", ""), str) - and isinstance(item.get("arguments"), dict) - ) - } - prepared.append(response_map) - for answer in row["answers"]: - response = response_map.get(answer["name"]) - if response is None: - continue - for left, right, match_type in resolved_arguments( - answer, response, apis[answer["name"]] - ): - collect_semantic_pairs(left, right, match_type, pairs) - prepare_semantic_scores(pairs) - - row_soft_total = 0.0 - call_soft_total = 0.0 - call_count = 0 - perfect_rows = 0 - correct_arguments = 0 - total_arguments = 0 - for row, response_map in zip(payload["rows"], prepared): - row_correct = 0 - row_total = 0 - missing_calls = 0 - for answer in row["answers"]: - api = apis[answer["name"]] - response = response_map.get(answer["name"]) - call_correct = 0 - call_total = 0 - if response is None: - missing_calls += 1 - call_total = len(api["arguments"]) - row_total += call_total - call_soft_total += 0.0 - call_count += 1 - continue - for name, spec in api["arguments"].items(): - answer_has = name in answer["arguments"] - response_has = name in response["arguments"] - if ( - mime_presence_only - and api["name"] == "ACTION_OPEN_DOCUMENT" - and name == "mime_types" - ): - if answer_has and response_has: - row_correct += 1 - call_correct += 1 - row_total += 1 - call_total += 1 - continue - if not answer_has and not response_has: - row_correct += 1 - call_correct += 1 - row_total += 1 - call_total += 1 - continue - if spec.get("required", False) and not answer_has: - row_total += 1 - call_total += 1 - continue - default = spec.get("default") - if deep_compare( - answer["arguments"].get(name, default), - response["arguments"].get(name, default), - spec.get("match_type", "strict"), - semantic_threshold, - ): - row_correct += 1 - call_correct += 1 - row_total += 1 - call_total += 1 - call_soft_total += ( - 1.0 if call_total == 0 else call_correct / call_total - ) - call_count += 1 - row_score = ( - 0.0 - if missing_calls > 0 - else 1.0 - if row_total == 0 - else row_correct / row_total - ) - row_soft_total += row_score - if abs(row_score - 1.0) < 1e-6: - perfect_rows += 1 - correct_arguments += row_correct - total_arguments += row_total - row_count = len(payload["rows"]) - soft_accuracy = ( - call_soft_total / call_count - if aggregation == "function-call-mean" and call_count > 0 - else row_soft_total / row_count - if row_count > 0 - else 0.0 - ) - overrides = [] - if mime_presence_only: - overrides.append( - { - "tool": "ACTION_OPEN_DOCUMENT", - "argument": "mime_types", - "comparison": "presence-only", - } - ) - return { - "softAccuracy": soft_accuracy, - "accuracy": perfect_rows / row_count if row_count > 0 else 0.0, - "counts": { - "rows": row_count, - "perfectRows": perfect_rows, - "correctArguments": correct_arguments, - "totalArguments": total_arguments, - "functionCalls": call_count, - }, - "contract": { - "name": contract_name, - "scorerRevision": "3f7ba458bee480a86c602edff6cc7ec9cfd555db", - "bertScore": "0.3.13", - "transformers": "4.48.1", - "semanticThreshold": semantic_threshold, - "softAccuracyAggregation": aggregation, - "overrides": overrides, - }, - } - - -def main(): - if "--jsonl" in sys.argv: - for line in sys.stdin: - try: - payload = json.loads(line) - contract_name = payload.pop("contract", "released") - print( - json.dumps(score_payload(payload, contract_name)), flush=True - ) - except Exception as error: - print(json.dumps({"error": str(error)}), flush=True) - return - payload = json.load(sys.stdin) - contract_name = payload.pop("contract", "released") - print(json.dumps(score_payload(payload, contract_name))) - - -if __name__ == "__main__": - main() diff --git a/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/index.ts b/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/index.ts new file mode 100644 index 0000000000..334f9af789 --- /dev/null +++ b/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/index.ts @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { realpathSync } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { downloadDroidCall } from "../huggingFaceRows.js"; +import { analyzeDroidCall } from "./analyze.js"; + +const DEFAULT_OUTPUT_DIR = join( + process.cwd(), + "src/translationBench/public_datasets/DroidCall", +); + +async function main(): Promise { + const args = new Set(process.argv.slice(2)); + const outputArg = process.argv + .slice(2) + .find((arg) => !arg.startsWith("--")); + const outputDir = outputArg ?? DEFAULT_OUTPUT_DIR; + if (args.has("--download")) await downloadDroidCall(outputDir); + const report = await analyzeDroidCall(outputDir); + console.log(JSON.stringify(report.splits, null, 2)); +} + +if ( + process.argv[1] !== undefined && + realpathSync(process.argv[1]) === fileURLToPath(import.meta.url) +) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} From 70feb26e5209498e6c55f2b1811858d65ba689b6 Mon Sep 17 00:00:00 2001 From: typeagent-bot Date: Wed, 26 Aug 2026 23:52:55 +0000 Subject: [PATCH 04/10] docs: regenerate README.AUTOGEN.md, command reference, and action browser --- ts/packages/benchmarks/README.AUTOGEN.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ts/packages/benchmarks/README.AUTOGEN.md b/ts/packages/benchmarks/README.AUTOGEN.md index 3a554ac12b..a233966fba 100644 --- a/ts/packages/benchmarks/README.AUTOGEN.md +++ b/ts/packages/benchmarks/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # @typeagent/benchmarks — AI-generated documentation @@ -44,6 +44,7 @@ _None._ - [./src/index.ts](./src/index.ts) - [./src/translationBench/index.ts](./src/translationBench/index.ts) +- [./src/translationBench/public_datasets/DroidCall/index.ts](./src/translationBench/public_datasets/DroidCall/index.ts) - [./src/translationBench/synthesizer/catalogGenerator/index.ts](./src/translationBench/synthesizer/catalogGenerator/index.ts) - [./src/translationBench/synthesizer/goldSchema.ts](./src/translationBench/synthesizer/goldSchema.ts) - [./src/translationBench/synthesizer/index.ts](./src/translationBench/synthesizer/index.ts) @@ -51,11 +52,10 @@ _None._ - [./src/core/paths.ts](./src/core/paths.ts) - [./src/core/prices.ts](./src/core/prices.ts) - [./src/core/rateLimiter.ts](./src/core/rateLimiter.ts) -- [./src/core/tokenEstimate.ts](./src/core/tokenEstimate.ts) -- _…and 52 more under `./src/`._ +- _…and 54 more under `./src/`._ --- -_Auto-generated against commit `cd45ef0980b0c41742fe01efb9f16b942a172335` on `2026-08-25T22:29:19.411Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ +_Auto-generated against commit `c2b6b4ffc29ad962c28c4a5c605ed4774df58a60` on `2026-08-26T23:51:00.019Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ From 1b0cfc556c3f7522be6e6baca8e9cf6a36c026a1 Mon Sep 17 00:00:00 2001 From: Dominic Nguyen Date: Wed, 26 Aug 2026 16:55:19 -0700 Subject: [PATCH 05/10] refactor(droidcall): keep analyzer self-contained --- ts/packages/benchmarks/README.AUTOGEN.md | 8 ++--- .../public_datasets/DroidCall/analyze.ts | 34 +++++++++++++++++- .../public_datasets/DroidCall/index.ts | 35 ------------------- 3 files changed, 37 insertions(+), 40 deletions(-) delete mode 100644 ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/index.ts diff --git a/ts/packages/benchmarks/README.AUTOGEN.md b/ts/packages/benchmarks/README.AUTOGEN.md index a233966fba..3a554ac12b 100644 --- a/ts/packages/benchmarks/README.AUTOGEN.md +++ b/ts/packages/benchmarks/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # @typeagent/benchmarks — AI-generated documentation @@ -44,7 +44,6 @@ _None._ - [./src/index.ts](./src/index.ts) - [./src/translationBench/index.ts](./src/translationBench/index.ts) -- [./src/translationBench/public_datasets/DroidCall/index.ts](./src/translationBench/public_datasets/DroidCall/index.ts) - [./src/translationBench/synthesizer/catalogGenerator/index.ts](./src/translationBench/synthesizer/catalogGenerator/index.ts) - [./src/translationBench/synthesizer/goldSchema.ts](./src/translationBench/synthesizer/goldSchema.ts) - [./src/translationBench/synthesizer/index.ts](./src/translationBench/synthesizer/index.ts) @@ -52,10 +51,11 @@ _None._ - [./src/core/paths.ts](./src/core/paths.ts) - [./src/core/prices.ts](./src/core/prices.ts) - [./src/core/rateLimiter.ts](./src/core/rateLimiter.ts) -- _…and 54 more under `./src/`._ +- [./src/core/tokenEstimate.ts](./src/core/tokenEstimate.ts) +- _…and 52 more under `./src/`._ --- -_Auto-generated against commit `c2b6b4ffc29ad962c28c4a5c605ed4774df58a60` on `2026-08-26T23:51:00.019Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ +_Auto-generated against commit `cd45ef0980b0c41742fe01efb9f16b942a172335` on `2026-08-25T22:29:19.411Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ diff --git a/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts b/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts index 226fb2ef91..399006bf5e 100644 --- a/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts +++ b/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts @@ -2,8 +2,10 @@ // Licensed under the MIT License. import { createHash } from "node:crypto"; +import { realpathSync } from "node:fs"; import { mkdir, readFile, stat, writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { fileURLToPath } from "node:url"; import { classifyDroidCalls, @@ -12,7 +14,11 @@ import { type DroidCall, type DroidCallShape, } from "../pythonLiteral.js"; -import { DROIDCALL_SOURCE, readDroidCallJsonl } from "../huggingFaceRows.js"; +import { + DROIDCALL_SOURCE, + downloadDroidCall, + readDroidCallJsonl, +} from "../huggingFaceRows.js"; interface CanonicalRow { query: string; @@ -262,3 +268,29 @@ export async function analyzeDroidCall( ]); return report; } + +const DEFAULT_OUTPUT_DIR = join( + process.cwd(), + "src/translationBench/public_datasets/DroidCall", +); + +async function main(): Promise { + const args = new Set(process.argv.slice(2)); + const outputArg = process.argv + .slice(2) + .find((arg) => !arg.startsWith("--")); + const outputDir = outputArg ?? DEFAULT_OUTPUT_DIR; + if (args.has("--download")) await downloadDroidCall(outputDir); + const report = await analyzeDroidCall(outputDir); + console.log(JSON.stringify(report.splits, null, 2)); +} + +if ( + process.argv[1] !== undefined && + realpathSync(process.argv[1]) === fileURLToPath(import.meta.url) +) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +} diff --git a/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/index.ts b/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/index.ts deleted file mode 100644 index 334f9af789..0000000000 --- a/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/index.ts +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { realpathSync } from "node:fs"; -import { join } from "node:path"; -import { fileURLToPath } from "node:url"; - -import { downloadDroidCall } from "../huggingFaceRows.js"; -import { analyzeDroidCall } from "./analyze.js"; - -const DEFAULT_OUTPUT_DIR = join( - process.cwd(), - "src/translationBench/public_datasets/DroidCall", -); - -async function main(): Promise { - const args = new Set(process.argv.slice(2)); - const outputArg = process.argv - .slice(2) - .find((arg) => !arg.startsWith("--")); - const outputDir = outputArg ?? DEFAULT_OUTPUT_DIR; - if (args.has("--download")) await downloadDroidCall(outputDir); - const report = await analyzeDroidCall(outputDir); - console.log(JSON.stringify(report.splits, null, 2)); -} - -if ( - process.argv[1] !== undefined && - realpathSync(process.argv[1]) === fileURLToPath(import.meta.url) -) { - main().catch((error: unknown) => { - console.error(error instanceof Error ? error.message : error); - process.exitCode = 1; - }); -} From 63c8ffcf8e023b7b598234f299e2f9e833aff031 Mon Sep 17 00:00:00 2001 From: Dominic Nguyen Date: Wed, 26 Aug 2026 17:08:51 -0700 Subject: [PATCH 06/10] fix(droidcall): validate analysis inputs --- .../public_datasets/DroidCall/analyze.ts | 192 ++++++++++-------- 1 file changed, 102 insertions(+), 90 deletions(-) diff --git a/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts b/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts index 399006bf5e..53589e0b8f 100644 --- a/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts +++ b/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts @@ -3,22 +3,16 @@ import { createHash } from "node:crypto"; import { realpathSync } from "node:fs"; -import { mkdir, readFile, stat, writeFile } from "node:fs/promises"; +import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { - classifyDroidCalls, - hasDroidCallResultReference, parseDroidCallCode, type DroidCall, type DroidCallShape, } from "../pythonLiteral.js"; -import { - DROIDCALL_SOURCE, - downloadDroidCall, - readDroidCallJsonl, -} from "../huggingFaceRows.js"; +import { DROIDCALL_SOURCE, downloadDroidCall } from "../huggingFaceRows.js"; interface CanonicalRow { query: string; @@ -70,6 +64,49 @@ const DROIDCALL_SOURCE_SHA256 = { const percent = (part: number, total: number): number => total === 0 ? 0 : Number(((part * 100) / total).toFixed(2)); +const RESULT_REFERENCE = /^#(\d+)$/; + +function hasPriorResultReference( + value: unknown, + priorResultIds: ReadonlySet, +): boolean { + if (typeof value === "string") { + const match = RESULT_REFERENCE.exec(value); + return match !== null && priorResultIds.has(Number(match[1])); + } + if (Array.isArray(value)) { + return value.some((item) => + hasPriorResultReference(item, priorResultIds), + ); + } + if (typeof value === "object" && value !== null) { + return Object.values(value).some((item) => + hasPriorResultReference(item, priorResultIds), + ); + } + return false; +} + +function nestedConsumerNames(calls: readonly DroidCall[]): string[] { + const priorResultIds = new Set(); + const names: string[] = []; + for (const call of calls) { + if (hasPriorResultReference(call.arguments, priorResultIds)) { + names.push(call.name); + } + priorResultIds.add(call.id); + } + return names; +} + +function classifyCalls(calls: readonly DroidCall[]): DroidCallShape { + if (calls.length === 0) return "noCall"; + if (calls.length === 1) return "singleTool"; + return nestedConsumerNames(calls).length > 0 + ? "multiCallNested" + : "multiCallWithoutNested"; +} + function analyzeSplit(rows: CanonicalRow[]): SplitAnalysis { const bucketCounts: Record = { noCall: 0, @@ -82,18 +119,13 @@ function analyzeSplit(rows: CanonicalRow[]): SplitAnalysis { let calls = 0; for (const row of rows) { calls += row.answers.length; - bucketCounts[classifyDroidCalls(row.answers)]++; + bucketCounts[classifyCalls(row.answers)]++; distribution.set( row.answers.length, (distribution.get(row.answers.length) ?? 0) + 1, ); - for (const call of row.answers) { - if (hasDroidCallResultReference(call.arguments)) { - nestedConsumers.set( - call.name, - (nestedConsumers.get(call.name) ?? 0) + 1, - ); - } + for (const name of nestedConsumerNames(row.answers)) { + nestedConsumers.set(name, (nestedConsumers.get(name) ?? 0) + 1); } } return { @@ -133,62 +165,44 @@ function sameCalls(left: DroidCall[], right: DroidCall[]): boolean { ); } -async function sourceFileInfo( - rawDir: string, -): Promise> { - const result: Record = {}; +type SourceFileName = (typeof DROIDCALL_SOURCE.files)[number]; + +interface SourceSnapshot { + files: Record; + contents: Map; +} + +async function readSourceSnapshot(rawDir: string): Promise { + const files: Record = {}; + const contents = new Map(); for (const relativePath of DROIDCALL_SOURCE.files) { - const path = join(rawDir, relativePath); - const [contents, fileStat] = await Promise.all([ - readFile(path), - stat(path), - ]); - const sha256 = createHash("sha256").update(contents).digest("hex"); + const content = await readFile(join(rawDir, relativePath)); + const sha256 = createHash("sha256").update(content).digest("hex"); if (sha256 !== DROIDCALL_SOURCE_SHA256[relativePath]) { throw new Error( `${relativePath} does not match DroidCall revision ${DROIDCALL_SOURCE.revision}`, ); } - result[relativePath] = { bytes: fileStat.size, sha256 }; + files[relativePath] = { bytes: content.byteLength, sha256 }; + contents.set(relativePath, content); } - return result; + return { files, contents }; } -function markdown(report: DroidCallAnalysis): string { - const row = (name: string, bucket: Bucket) => - `| ${name} | ${bucket.rows.toLocaleString()} | ${bucket.percent.toFixed(2)}% |`; - const sections = Object.entries(report.splits) - .map(([name, split]) => { - const heading = - name === "full" - ? "Full dataset" - : name === "train" - ? "Training split" - : "Test split"; - const multiCallRows = - split.buckets.multiCallNested.rows + - split.buckets.multiCallWithoutNested.rows; - const nestedShareOfMultiCall = percent( - split.buckets.multiCallNested.rows, - multiCallRows, - ); - const distribution = Object.entries(split.callCountDistribution) - .map(([calls, rows]) => `${calls}: ${rows.toLocaleString()}`) - .join(", "); - return `## ${heading}\n\n${split.rows.toLocaleString()} rows contain ${split.calls.toLocaleString()} calls. ${multiCallRows.toLocaleString()} rows (${percent(multiCallRows, split.rows).toFixed(2)}%) have more than one call. Of those multi-call rows, ${nestedShareOfMultiCall.toFixed(2)}% pass a prior result into a later call.\n\n| Shape | Rows | Share of rows |\n| --- | ---: | ---: |\n${row("No call", split.buckets.noCall)} -${row("Single tool", split.buckets.singleTool)}\n${row("Multi-call, nested", split.buckets.multiCallNested)}\n${row("Multi-call, without nesting", split.buckets.multiCallWithoutNested)}\n\nRows by call count: ${distribution}.`; - }) - .join("\n\n"); - const snapshotBytes = Object.values(report.source.files).reduce( - (sum, file) => sum + file.bytes, - 0, - ); - const full = report.splits.full!; - const fullMultiCallRows = - full.buckets.multiCallNested.rows + - full.buckets.multiCallWithoutNested.rows; - return `# DroidCall data analysis\n\nThe full DroidCall dataset has ${full.rows.toLocaleString()} rows and ${full.calls.toLocaleString()} gold calls. Single-tool requests account for ${full.buckets.singleTool.percent.toFixed(2)}% of rows. The other ${percent(fullMultiCallRows, full.rows).toFixed(2)}% are multi-call requests; ${percent(full.buckets.multiCallNested.rows, fullMultiCallRows).toFixed(2)}% of those pass a prior result into a later call.\n\nSource: [mllmTeam/DroidCall](https://huggingface.co/datasets/mllmTeam/DroidCall), revision \`${report.source.revision}\`. The local snapshot has ${Object.keys(report.source.files).length} files (${(snapshotBytes / 1024 / 1024).toFixed(2)} MiB). It includes every file listed by the HuggingFace repository at that revision.\n\n## Classification\n\nThe analysis reads the structured \`answers\` in \`DroidCall_train.jsonl\` and \`DroidCall_test.jsonl\`. The buckets are mutually exclusive:\n\n- No call: no gold calls. -- Single tool: exactly one gold call.\n- Multi-call, nested: at least two calls and an argument contains a \`#N\` result reference. The reference can occur inside an array or object.\n- Multi-call, without nesting: at least two calls and no argument contains a result reference.\n\n${sections}\n\n## Parser reuse and validation\n\nDroidCall's assistant output uses Python-like function calls. \`parseDroidCallCode()\` handles the assignment and call syntax, then delegates strings, numbers, booleans, nulls, arrays, and objects to Seal-Tools' existing \`parsePythonLiteral()\`. This keeps one literal parser for both datasets.\n\nThe code-format file covers the ${report.parserValidation.rows.toLocaleString()} training rows. Parsed calls exactly match the canonical structured answers for ${report.parserValidation.exactMatches.toLocaleString()} rows (${percent(report.parserValidation.exactMatches, report.parserValidation.rows).toFixed(2)}%). There are ${report.parserValidation.parseFailures} parse failures and ${report.parserValidation.mismatches} source mismatches. Source mismatches include values that the code syntax cannot reproduce, such as a sentence-like function name or an argument key with leading whitespace.\n`; +function parseJsonl(fileName: SourceFileName, contents: Buffer): T[] { + const rows: T[] = []; + for (const [index, line] of contents + .toString("utf8") + .split("\n") + .entries()) { + if (line.trim().length === 0) continue; + try { + rows.push(JSON.parse(line) as T); + } catch (error) { + throw new Error(`${fileName}:${index + 1}: ${String(error)}`); + } + } + return rows; } export interface DroidCallAnalysis { @@ -209,12 +223,19 @@ export interface DroidCallAnalysis { export async function analyzeDroidCall( outputDir: string, ): Promise { - const rawDir = join(outputDir, "raw"); - const [train, test, chat] = await Promise.all([ - readDroidCallJsonl(join(rawDir, "DroidCall_train.jsonl")), - readDroidCallJsonl(join(rawDir, "DroidCall_test.jsonl")), - readDroidCallJsonl(join(rawDir, "DroidCall_code_short.jsonl")), - ]); + const source = await readSourceSnapshot(join(outputDir, "raw")); + const train = parseJsonl( + "DroidCall_train.jsonl", + source.contents.get("DroidCall_train.jsonl")!, + ); + const test = parseJsonl( + "DroidCall_test.jsonl", + source.contents.get("DroidCall_test.jsonl")!, + ); + const chat = parseJsonl( + "DroidCall_code_short.jsonl", + source.contents.get("DroidCall_code_short.jsonl")!, + ); if (chat.length !== train.length) { throw new Error( `DroidCall code and train row counts differ: ${chat.length} !== ${train.length}`, @@ -243,7 +264,7 @@ export async function analyzeDroidCall( source: { dataset: DROIDCALL_SOURCE.dataset, revision: DROIDCALL_SOURCE.revision, - files: await sourceFileInfo(rawDir), + files: source.files, }, splits: { full: analyzeSplit([...train, ...test]), @@ -257,30 +278,21 @@ export async function analyzeDroidCall( mismatches, }, }; - const docsDir = join(outputDir, "docs"); - await mkdir(docsDir, { recursive: true }); - await Promise.all([ - writeFile( - join(outputDir, "analysis.json"), - JSON.stringify(report, null, 2) + "\n", - ), - writeFile(join(docsDir, "DroidCall.md"), markdown(report)), - ]); + await writeFile( + join(outputDir, "analysis.json"), + JSON.stringify(report, null, 2) + "\n", + ); return report; } -const DEFAULT_OUTPUT_DIR = join( - process.cwd(), - "src/translationBench/public_datasets/DroidCall", -); - async function main(): Promise { - const args = new Set(process.argv.slice(2)); - const outputArg = process.argv - .slice(2) - .find((arg) => !arg.startsWith("--")); - const outputDir = outputArg ?? DEFAULT_OUTPUT_DIR; - if (args.has("--download")) await downloadDroidCall(outputDir); + const args = process.argv.slice(2); + const outputArgs = args.filter((arg) => !arg.startsWith("--")); + if (outputArgs.length !== 1) { + throw new Error("Usage: analyzeDroidCall [--download] "); + } + const outputDir = outputArgs[0]!; + if (args.includes("--download")) await downloadDroidCall(outputDir); const report = await analyzeDroidCall(outputDir); console.log(JSON.stringify(report.splits, null, 2)); } From 493931d64a0633ed8909c52b3e84ff536fdab1ce Mon Sep 17 00:00:00 2001 From: Dominic Nguyen Date: Wed, 26 Aug 2026 17:14:25 -0700 Subject: [PATCH 07/10] fix(droidcall): reject unknown analyzer flags --- .../public_datasets/DroidCall/analyze.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts b/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts index 53589e0b8f..6de0e85018 100644 --- a/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts +++ b/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts @@ -65,6 +65,7 @@ const percent = (part: number, total: number): number => total === 0 ? 0 : Number(((part * 100) / total).toFixed(2)); const RESULT_REFERENCE = /^#(\d+)$/; +const DOWNLOAD_OPTION = "--download"; function hasPriorResultReference( value: unknown, @@ -288,11 +289,16 @@ export async function analyzeDroidCall( async function main(): Promise { const args = process.argv.slice(2); const outputArgs = args.filter((arg) => !arg.startsWith("--")); - if (outputArgs.length !== 1) { - throw new Error("Usage: analyzeDroidCall [--download] "); + const unknownOptions = args.filter( + (arg) => arg.startsWith("--") && arg !== DOWNLOAD_OPTION, + ); + if (outputArgs.length !== 1 || unknownOptions.length > 0) { + throw new Error( + `Usage: analyzeDroidCall [${DOWNLOAD_OPTION}] `, + ); } const outputDir = outputArgs[0]!; - if (args.includes("--download")) await downloadDroidCall(outputDir); + if (args.includes(DOWNLOAD_OPTION)) await downloadDroidCall(outputDir); const report = await analyzeDroidCall(outputDir); console.log(JSON.stringify(report.splits, null, 2)); } From bc640f89765af07ae41a6eaf3fe7efcccbceca81 Mon Sep 17 00:00:00 2001 From: typeagent-bot Date: Thu, 27 Aug 2026 00:22:20 +0000 Subject: [PATCH 08/10] docs: regenerate README.AUTOGEN.md, command reference, and action browser --- ts/packages/benchmarks/README.AUTOGEN.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ts/packages/benchmarks/README.AUTOGEN.md b/ts/packages/benchmarks/README.AUTOGEN.md index 3a554ac12b..c9da3e3713 100644 --- a/ts/packages/benchmarks/README.AUTOGEN.md +++ b/ts/packages/benchmarks/README.AUTOGEN.md @@ -3,7 +3,7 @@ - + # @typeagent/benchmarks — AI-generated documentation @@ -52,10 +52,10 @@ _None._ - [./src/core/prices.ts](./src/core/prices.ts) - [./src/core/rateLimiter.ts](./src/core/rateLimiter.ts) - [./src/core/tokenEstimate.ts](./src/core/tokenEstimate.ts) -- _…and 52 more under `./src/`._ +- _…and 53 more under `./src/`._ --- -_Auto-generated against commit `cd45ef0980b0c41742fe01efb9f16b942a172335` on `2026-08-25T22:29:19.411Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ +_Auto-generated against commit `493931d64a0633ed8909c52b3e84ff536fdab1ce` on `2026-08-27T00:20:31.457Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._ From 79aaba417c375785b48e91dcbf0d1ebc8160e361 Mon Sep 17 00:00:00 2001 From: Dominic Nguyen Date: Wed, 26 Aug 2026 17:23:19 -0700 Subject: [PATCH 09/10] fix(droidcall): write analyzer output directly --- .../src/translationBench/public_datasets/DroidCall/analyze.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts b/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts index 6de0e85018..2414bbeb62 100644 --- a/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts +++ b/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts @@ -300,7 +300,7 @@ async function main(): Promise { const outputDir = outputArgs[0]!; if (args.includes(DOWNLOAD_OPTION)) await downloadDroidCall(outputDir); const report = await analyzeDroidCall(outputDir); - console.log(JSON.stringify(report.splits, null, 2)); + process.stdout.write(`${JSON.stringify(report.splits, null, 2)}\n`); } if ( From aee72d8c89ed514e0677dbd77be256e0a347319b Mon Sep 17 00:00:00 2001 From: Dominic Nguyen Date: Wed, 26 Aug 2026 23:26:52 -0700 Subject: [PATCH 10/10] fix(droidcall): count embedded result references --- .../translationBench/public_datasets/DroidCall/analyze.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts b/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts index 2414bbeb62..c06d71e0d1 100644 --- a/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts +++ b/ts/packages/benchmarks/src/translationBench/public_datasets/DroidCall/analyze.ts @@ -64,7 +64,7 @@ const DROIDCALL_SOURCE_SHA256 = { const percent = (part: number, total: number): number => total === 0 ? 0 : Number(((part * 100) / total).toFixed(2)); -const RESULT_REFERENCE = /^#(\d+)$/; +const RESULT_REFERENCE = /#(\d+)\b/g; const DOWNLOAD_OPTION = "--download"; function hasPriorResultReference( @@ -72,8 +72,9 @@ function hasPriorResultReference( priorResultIds: ReadonlySet, ): boolean { if (typeof value === "string") { - const match = RESULT_REFERENCE.exec(value); - return match !== null && priorResultIds.has(Number(match[1])); + return [...value.matchAll(RESULT_REFERENCE)].some((match) => + priorResultIds.has(Number(match[1])), + ); } if (Array.isArray(value)) { return value.some((item) =>