From e2687940915a032aee1734e0925bb9fa450fe96c Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:50:59 -0500 Subject: [PATCH 01/24] feat: add stock kimi tool-use eval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：添加基于 Kimi Vendor Verifier 原生实现的工具调用评估 --- .github/workflows/benchmark-tmpl.yml | 2 + benchmarks/benchmark_lib.sh | 257 ++++++++- utils/collect_eval_results.py | 4 + utils/evals/EVALS.md | 87 ++- utils/evals/kimi_vendor_eval.py | 255 +++++++++ utils/evals/test_kimi_vendor_eval.py | 287 ++++++++++ utils/evals/test_run_eval_dispatch.py | 506 +++++++++++++++++- utils/evals/thresholds.yaml | 1 + utils/test_collect_eval_results.py | 19 + .../test_validate_reusable_sweep_artifacts.py | 70 ++- utils/validate_reusable_sweep_artifacts.py | 12 + 11 files changed, 1480 insertions(+), 20 deletions(-) create mode 100755 utils/evals/kimi_vendor_eval.py create mode 100644 utils/evals/test_kimi_vendor_eval.py diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 6c4fe50fe5..4dc036a06d 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -392,6 +392,7 @@ jobs: path: | meta_env.json results*.json + kimi_vendor_report.json sample*.jsonl agent_preds.json predictions.jsonl @@ -409,6 +410,7 @@ jobs: rm -f meta_env.json || true # Remove any eval results JSONs that were moved into workspace rm -f results*.json || true + rm -f kimi_vendor_report.json || true rm -f sample*.jsonl || true rm -f agent_preds.json predictions.jsonl swebench_report_*.json *.traj* || true diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 1617e1190e..5519682c8f 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -8,6 +8,9 @@ export PYTHONDONTWRITEBYTECODE=1 export PYTHONPYCACHEPREFIX="${PYTHONPYCACHEPREFIX:-/tmp/inferencex-pycache}" mkdir -p "$PYTHONPYCACHEPREFIX" 2>/dev/null || true +INFERENCEX_BENCHMARK_LIB_DIR="$( + cd "$(dirname "${BASH_SOURCE[0]}")" && pwd +)" # Inference server port shared by every benchmark recipe. Launchers that need # a non-default value (e.g. launch_mi355x-amds.sh derives PORT from RUNNER_NAME @@ -816,6 +819,230 @@ _install_lm_eval_deps() { fi } +_require_tool_use_python() { + if python3 -c 'import sys; raise SystemExit(sys.version_info < (3, 12))'; then + return 0 + fi + + local python_version + python_version="$(python3 -c 'import platform; print(platform.python_version())' 2>/dev/null || printf 'unavailable')" + echo "ERROR: tool-use requires Python >=3.12 (python3 is ${python_version})" >&2 + return 2 +} + +_install_tool_use_eval_deps() { + python3 -m pip install -q --no-cache-dir --break-system-packages \ + "httpx[http2]==0.28.1" \ + "openai==2.14.0" \ + "jsonschema==4.25.1" \ + "pytest==8.4.2" +} + +_kimi_vendor_checkout_is_valid() { + local checkout_dir="$1" + local expected_ref="$2" + local checkout_ref checkout_status tracked_status untracked_files ignored_files + + [ -f "${checkout_dir}/LICENSE" ] \ + && [ -f "${checkout_dir}/pyproject.toml" ] \ + && [ -f "${checkout_dir}/tests/conftest.py" ] \ + && [ -f "${checkout_dir}/tests/__init__.py" ] \ + && [ -f "${checkout_dir}/tests/tool_call_json_schema/conftest.py" ] \ + && [ -f "${checkout_dir}/tests/tool_call_json_schema/validator.py" ] \ + && [ -f "${checkout_dir}/tests/tool_call_json_schema/test_tool_call_json_schema.py" ] \ + && [ -d "${checkout_dir}/testdata/walle_validator_cases/validator_cases" ] \ + || return 1 + checkout_ref="$(git -C "$checkout_dir" rev-parse HEAD 2>/dev/null)" \ + || return 1 + [ "$checkout_ref" = "$expected_ref" ] || return 1 + checkout_status="$( + git -C "$checkout_dir" status --porcelain --untracked-files=all -- \ + LICENSE \ + pyproject.toml \ + tests/conftest.py \ + tests/__init__.py \ + tests/tool_call_json_schema \ + testdata/walle_validator_cases + )" || return 1 + [ -z "$checkout_status" ] || return 1 + tracked_status="$( + git -C "$checkout_dir" status --porcelain --untracked-files=no + )" || return 1 + [ -z "$tracked_status" ] || return 1 + untracked_files="$( + git -C "$checkout_dir" ls-files --others --exclude-standard -- \ + . ':(exclude,top,glob).pytest_cache/**' + )" || return 1 + [ -z "$untracked_files" ] || return 1 + ignored_files="$( + git -C "$checkout_dir" ls-files --others --ignored --exclude-standard -- \ + . ':(exclude,top,glob).pytest_cache/**' + )" || return 1 + [ -z "$ignored_files" ] +} + +_prepare_kimi_vendor_verifier() { + local repo_url="$1" + local verifier_ref="$2" + local checkout_dir + + if [ -n "${KIMI_VENDOR_VERIFIER_DIR:-}" ]; then + checkout_dir="$KIMI_VENDOR_VERIFIER_DIR" + if ! _kimi_vendor_checkout_is_valid "$checkout_dir" "$verifier_ref"; then + echo "ERROR: KIMI_VENDOR_VERIFIER_DIR must be at ${verifier_ref}" >&2 + echo "ERROR: required verifier sources must be present and unmodified" >&2 + return 2 + fi + KIMI_VENDOR_VERIFIER_CHECKOUT_DIR="$checkout_dir" + return 0 + fi + + checkout_dir="/tmp/kimi-vendor-verifier-${verifier_ref}" + if _kimi_vendor_checkout_is_valid "$checkout_dir" "$verifier_ref"; then + KIMI_VENDOR_VERIFIER_CHECKOUT_DIR="$checkout_dir" + return 0 + fi + + command -v git >/dev/null 2>&1 || { + echo "ERROR: git is required to fetch Kimi-Vendor-Verifier" >&2 + return 1 + } + rm -rf "$checkout_dir" + mkdir -p "$(dirname "$checkout_dir")" || return $? + if ! ( + git init -q "$checkout_dir" \ + && git -C "$checkout_dir" remote add origin "$repo_url" \ + && git -C "$checkout_dir" config remote.origin.promisor true \ + && git -C "$checkout_dir" config remote.origin.partialclonefilter blob:none \ + && git -C "$checkout_dir" fetch -q --filter=blob:none --depth=1 \ + origin "$verifier_ref" \ + && git -C "$checkout_dir" update-ref HEAD FETCH_HEAD \ + && git -C "$checkout_dir" sparse-checkout set --no-cone \ + /LICENSE \ + /pyproject.toml \ + /tests/conftest.py \ + /tests/__init__.py \ + /tests/tool_call_json_schema/ \ + /testdata/walle_validator_cases/ \ + && git -C "$checkout_dir" checkout -q --detach HEAD + ); then + rm -rf "$checkout_dir" + echo "ERROR: failed to fetch Kimi-Vendor-Verifier at ${verifier_ref}" >&2 + return 1 + fi + if ! _kimi_vendor_checkout_is_valid "$checkout_dir" "$verifier_ref"; then + rm -rf "$checkout_dir" + echo "ERROR: fetched Kimi-Vendor-Verifier checkout is incomplete" >&2 + return 1 + fi + KIMI_VENDOR_VERIFIER_CHECKOUT_DIR="$checkout_dir" +} + +_write_tool_use_integration_error() { + local adapter_path="$1" + local model_name="$2" + local results_dir="$3" + local message="$4" + + python3 "$adapter_path" \ + --model "$model_name" \ + --output-dir "$results_dir" \ + --integration-error "$message" \ + || true +} + +run_tool_use_eval() { + local port="${PORT:-8888}" + local results_dir="${EVAL_RESULT_DIR:-$(mktemp -d /tmp/eval_out-XXXXXX)}" + local verifier_repo="https://github.com/MoonshotAI/Kimi-Vendor-Verifier.git" + local verifier_ref="b9ed3a6665bdff2c943246f7d2903cd003d6ddd6" + + while [[ $# -gt 0 ]]; do + case "$1" in + --port|--results-dir) + if [[ $# -lt 2 || -z "${2:-}" || "${2:-}" == --* ]]; then + echo "ERROR: $1 requires a value" >&2 + return 2 + fi + case "$1" in + --port) port="$2" ;; + --results-dir) results_dir="$2" ;; + esac + shift 2 + ;; + *) + echo "Unknown parameter: $1" >&2 + return 2 + ;; + esac + done + + local eval_suite="${EVAL_SUITE:-kimi_tool_call_schema}" + if [ "$eval_suite" != "kimi_tool_call_schema" ]; then + echo "ERROR: tool-use supports only EVAL_SUITE=kimi_tool_call_schema" >&2 + export EVAL_RESULT_DIR="" + return 2 + fi + case "${IS_MULTINODE:-false}" in + true|1) + echo "ERROR: tool-use Phase 1 supports single-node evals only" >&2 + export EVAL_RESULT_DIR="" + return 2 + ;; + esac + export EVAL_FRAMEWORK=tool-use + export EVAL_SUITE="$eval_suite" + + local _repo_root + _repo_root="$(cd "$INFERENCEX_BENCHMARK_LIB_DIR/.." && pwd)" + local model_name="${MODEL_NAME:-${MODEL:-}}" + local adapter_path="${_repo_root}/utils/evals/kimi_vendor_eval.py" + + mkdir -p "$results_dir" || return $? + export EVAL_RESULT_DIR="$results_dir" + + local setup_rc integration_error + if _require_tool_use_python; then + : + else + setup_rc=$? + integration_error="tool-use Python version check failed with exit code ${setup_rc}" + echo "ERROR: ${integration_error}" >&2 + _write_tool_use_integration_error \ + "$adapter_path" "$model_name" "$results_dir" "$integration_error" + return "$setup_rc" + fi + if [ "${INFERENCEX_TOOL_USE_EVAL_RUNTIME_READY:-false}" != "true" ]; then + if _install_tool_use_eval_deps; then + export INFERENCEX_TOOL_USE_EVAL_RUNTIME_READY=true + else + setup_rc=$? + integration_error="tool-use dependency installation failed with exit code ${setup_rc}" + echo "ERROR: ${integration_error}" >&2 + _write_tool_use_integration_error \ + "$adapter_path" "$model_name" "$results_dir" "$integration_error" + return "$setup_rc" + fi + fi + if _prepare_kimi_vendor_verifier "$verifier_repo" "$verifier_ref"; then + : + else + setup_rc=$? + integration_error="tool-use verifier checkout failed with exit code ${setup_rc}" + echo "ERROR: ${integration_error}" >&2 + _write_tool_use_integration_error \ + "$adapter_path" "$model_name" "$results_dir" "$integration_error" + return "$setup_rc" + fi + + python3 "$adapter_path" \ + --verifier-dir "$KIMI_VENDOR_VERIFIER_CHECKOUT_DIR" \ + --base-url "http://127.0.0.1:${port}/v1" \ + --api-key EMPTY \ + --model "$model_name" \ + --output-dir "$results_dir" +} + _eval_patches_dir() { cd "$(dirname "${BASH_SOURCE[0]}")/../utils/evals/patches" && pwd } @@ -934,6 +1161,15 @@ run_lm_eval() { tasks_dir="$_repo_root/$tasks_dir" fi + local effective_suite="${EVAL_SUITE:-}" + local task_basename + if [ -z "$effective_suite" ]; then + task_basename="${tasks_dir##*/}" + effective_suite="${task_basename%.yaml}" + effective_suite="${effective_suite%.yml}" + fi + export EVAL_SUITE="$effective_suite" + if [ "${INFERENCEX_LM_EVAL_RUNTIME_READY:-false}" != "true" ]; then _install_lm_eval_deps _patch_lm_eval @@ -1141,12 +1377,22 @@ append_lm_eval_summary() { fi fi fi + local eval_framework="${EVAL_FRAMEWORK:-lm-eval}" + local eval_suite="${EVAL_SUITE:-}" + if [ -z "$eval_suite" ] && [ -n "${EVAL_TASKS_DIR:-}" ]; then + eval_suite="$(basename "${EVAL_TASKS_DIR}")" + eval_suite="${eval_suite%.yaml}" + eval_suite="${eval_suite%.yml}" + fi + eval_suite="${eval_suite:-gsm8k}" cat > "${meta_json}" < /dev/null fi @@ -1635,6 +1887,7 @@ run_eval() { case "$framework" in lm-eval|lm_eval) run_lm_eval "${forwarded[@]}" || eval_rc=$? ;; swebench) run_swebench_eval "${forwarded[@]}" || eval_rc=$? ;; + tool-use) run_tool_use_eval "${forwarded[@]}" || eval_rc=$? ;; *) echo "Unknown framework '${framework}'"; eval_rc=1 ;; esac diff --git a/utils/collect_eval_results.py b/utils/collect_eval_results.py index 667e60bc6f..7bc49c5d8c 100644 --- a/utils/collect_eval_results.py +++ b/utils/collect_eval_results.py @@ -284,6 +284,10 @@ def build_row(meta: Dict[str, Any], m: Dict[str, Any]) -> Dict[str, Any]: 'source': m.get('source'), } + for metadata_field in ('eval_framework', 'eval_suite'): + if metadata_field in meta: + row[metadata_field] = meta[metadata_field] + # Add universal score field (primary metric for unified comparison) if m.get('strict') is not None: row['score'] = m.get('strict') diff --git a/utils/evals/EVALS.md b/utils/evals/EVALS.md index 7320795431..07aa48b4de 100644 --- a/utils/evals/EVALS.md +++ b/utils/evals/EVALS.md @@ -39,9 +39,85 @@ malformed metadata, duplicates, or raw/aggregate mismatches are not. See [workflow reuse](../../.github/workflows/README.md#reusing-an-approved-pr-full-sweep). ## How? -`run_eval` in `benchmarks/benchmark_lib.sh` runs EleutherAI/lm-evaluation-harness against the server's OpenAI-compatible endpoint. Concurrency is set via `EVAL_CONCURRENT_REQUESTS` env var (not a CLI flag). Results are collected by `utils/collect_eval_results.py` and published as a summary table. +`run_eval` in `benchmarks/benchmark_lib.sh` dispatches to the selected eval +framework against the server's OpenAI-compatible endpoint. The default is +[lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) +(`lm-eval`) with GSM8K. Existing fixed-sequence and agentic paths preserve that +default, and explicit agentic runs can still select SWE-bench. -The default eval framework is [lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) (`lm-eval`). Agentic eval-only matrix jobs inherit this default and therefore run the same GSM8K task as 8k1k; explicit agentic runs can still select SWE-bench. +The Phase 1 tool-use suite is opt-in. The Kimi-K3 B300 vLLM agentic launcher, +like every existing launcher, continues to select lm-eval/GSM8K by default. To +run the suite after its server is ready, use the existing entrypoint: + +```bash +EVAL_FRAMEWORK=tool-use EVAL_SUITE=kimi_tool_call_schema \ + run_eval --port "$PORT" +``` + +`run_tool_use_eval` supplies `kimi_tool_call_schema` when `EVAL_SUITE` is unset +for a manual `run_eval --framework tool-use` call and rejects every other suite. +The compatibility result continues through the existing collector, suite-aware +artifact identity, and strict `1.0` threshold. +Phase 1 is single-node only and rejects `IS_MULTINODE=true` or `1`; the +multi-node workflow does not yet preserve the stock native report. + +### Stock Kimi tool-call schema smoke + +This suite runs the unmodified +[MoonshotAI/Kimi-Vendor-Verifier](https://github.com/MoonshotAI/Kimi-Vendor-Verifier) +at commit `b9ed3a6665bdff2c943246f7d2903cd003d6ddd6`. Its bundled Walle +schema corpus is sourced from MoonshotAI/walle commit +`cc1c6b7dab5496d5184677ecf4c3b95fc1bd1606` (`v0.1.10`). The upstream +prompt, schema loading and selection, request construction, non-stream and +stream assembly, argument validation, and report generation are all stock. +InferenceX owns only the subprocess invocation and compatibility projection. + +Python 3.12 or newer is required; the runner fails with a version error before +installing or checking out anything on older Python. At runtime it installs only +`httpx[http2]==0.28.1`, `openai==2.14.0`, `jsonschema==4.25.1`, and +`pytest==8.4.2`. It then makes a network checkout from GitHub using a sparse, +detached checkout of the pinned verifier commit containing only: + +- `LICENSE` and `pyproject.toml`; +- `tests/__init__.py`, `tests/conftest.py`, and + `tests/tool_call_json_schema/`; +- `testdata/walle_validator_cases/`. + +An explicitly supplied `KIMI_VENDOR_VERIFIER_DIR` is reused only when it is at +that exact commit, required sources are unmodified, and no extra checkout files +can override the verifier (root `.pytest_cache/` is ignored). The verifier +project and its unrelated benchmark dependencies are not installed. + +The thin `utils/evals/kimi_vendor_eval.py` wrapper runs upstream +`tests/tool_call_json_schema/test_tool_call_json_schema.py` with: + +- base URL `http://127.0.0.1:${PORT}/v1`, API key `EMPTY`, and model + `${MODEL_NAME:-$MODEL}`; +- `--case-dir testdata/walle_validator_cases/validator_cases`, + `--think-mode none --selection object --max-cases 1 --max-tokens 2048`; +- `--tool-json-report /kimi_vendor_report.json`. + +That stock selection chooses Walle case `TestAdditionalProperties:1` and +upstream parametrizes it in both `non-stream` and `stream` modes, for two +results. Requests use upstream's `openai.Client(timeout=120)` unchanged, so +OpenAI SDK 2.14.0's stock retry policy remains in effect, including its default +two retries for eligible connection, timeout, 408, 409, 429, and 5xx failures. +InferenceX does not add request retries or make the two modes concurrent. +`EVAL_CONCURRENT_REQUESTS` remains matrix metadata; multi-value batched +concurrency remains supported only by `lm-eval`. + +The unchanged native `kimi_vendor_report.json` is uploaded alongside the +collector-compatible `results_kimi_vendor_.json`. The +compatibility score is `passed / 2` for task `kimi_tool_call_schema`, primary metric +`exact_match,strict-match`, and effective sample count two. Success requires +pytest to exit zero and exactly two upstream mode results to pass. A setup or +collection failure still produces a zero-score compatibility result with +integration error metadata; the native report can be absent when upstream +cannot collect. + +Phase 1 intentionally covers one stock object-schema case only. It does not +measure broader schema coverage, tool selection among multiple tools, parallel +tool calls, multi-turn tool execution, or general agent quality. ### Benchmark script flow @@ -72,8 +148,11 @@ Key eval functions in `benchmarks/benchmark_lib.sh`: |----------|-------------| | `run_eval` | Unified entrypoint - dispatches to framework-specific runner | | `run_lm_eval` | Runs lm-eval harness against the OpenAI-compatible endpoint | +| `run_tool_use_eval` | Runs the pinned stock verifier in non-stream and stream modes | | `append_lm_eval_summary` | Writes `meta_env.json` and moves eval artifacts to workspace | | `_install_lm_eval_deps` | Installs lm-eval dependencies | +| `_install_tool_use_eval_deps` | Installs the minimal pinned stock-verifier runtime | +| `_prepare_kimi_vendor_verifier` | Prepares or validates the pinned sparse checkout | | `_patch_lm_eval` | Patches lm-eval for reasoning tokens and TRT compatibility | | `compute_eval_context_length` | Computes eval context length (requested benchmark context, capped at model native max) | | `get_native_max_context_length` | Extracts model's native max context length from HF config | @@ -141,6 +220,8 @@ cat ./evals/agg_eval_all.json | jq '[.[] | select(.hw == "B200")]' | `em_flexible` | Flexible extraction (looser number matching) | | `n_eff` | Number of samples evaluated | | `task` | Eval task name (e.g., `gsm8k`) | +| `eval_framework` | Eval runner identity (for example, `lm-eval` or `tool-use`) | +| `eval_suite` | Explicit suite identity used for collection and artifact reuse | ### Environment variables @@ -149,8 +230,10 @@ cat ./evals/agg_eval_all.json | jq '[.[] | select(.hw == "B200")]' | `RUN_EVAL` | `false` | Enable eval after throughput benchmark | | `EVAL_ONLY` | `false` | Skip throughput, only run evals (set by workflow) | | `EVAL_FRAMEWORK` | `lm-eval` | Eval framework to use | +| `EVAL_SUITE` | basename of `EVAL_TASKS_DIR`, else `gsm8k` | Eval suite metadata; explicit values take precedence | | `EVAL_TASKS_DIR` | `utils/evals/gsm8k.yaml` | Path to lm-eval task YAML | | `EVAL_RESULT_DIR` | `/tmp/eval_out-*` | Output directory for eval results | +| `KIMI_VENDOR_VERIFIER_DIR` | generated pinned checkout | Optional pre-existing verifier checkout; exact ref and required paths are validated | | `EVAL_MAX_MODEL_LEN` | `16384` | Max context for eval (set by `compute_eval_context_length`) | | `EVAL_CONCURRENT_REQUESTS` | `64` | Concurrent requests during eval; a space-separated list enables sequential batched evals against one live engine | | `EVAL_LIMIT` | empty | Limit eval to first N instances (smoke tests); empty = full set | diff --git a/utils/evals/kimi_vendor_eval.py b/utils/evals/kimi_vendor_eval.py new file mode 100755 index 0000000000..c01343fe15 --- /dev/null +++ b/utils/evals/kimi_vendor_eval.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +"""Run the stock Kimi Vendor Verifier and project its native report.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from collections.abc import Mapping, Sequence +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +TASK_NAME = "kimi_tool_call_schema" +NATIVE_REPORT_FILENAME = "kimi_vendor_report.json" +COMPATIBILITY_GLOB = "results_kimi_vendor_*.json" +EXPECTED_MODES = {"non-stream", "stream"} + + +def prepare_compatibility_path(output_dir: Path) -> Path: + """Remove stale projections and return a timestamped collector artifact path.""" + for stale_path in output_dir.glob(COMPATIBILITY_GLOB): + stale_path.unlink() + timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S.%f") + return output_dir / f"results_kimi_vendor_{timestamp}.json" + + +def build_pytest_command( + *, base_url: str, api_key: str, model: str, report_path: Path +) -> list[str]: + """Build the fixed Phase 1 invocation of the upstream verifier.""" + return [ + sys.executable, + "-m", + "pytest", + "tests/tool_call_json_schema/test_tool_call_json_schema.py", + "--base-url", + base_url, + "--api-key", + api_key, + "--smoke-model", + model, + "--think-mode", + "none", + "--selection", + "object", + "--max-cases", + "1", + "--case-dir", + "testdata/walle_validator_cases/validator_cases", + "--max-tokens", + "2048", + "--tool-json-report", + str(report_path), + ] + + +def _mapping(value: Any, name: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError(f"{name} must be an object") + return value + + +def _project_report(model: str, report: Any) -> tuple[dict[str, Any], bool]: + root = _mapping(report, "report") + summary = _mapping(root.get("summary"), "report.summary") + results = root.get("results") + if not isinstance(results, list): + raise ValueError("report.results must be an array") + + total = summary.get("total") + by_status = _mapping(summary.get("by_status"), "report.summary.by_status") + passed = by_status.get("passed", 0) + if ( + not isinstance(total, int) + or isinstance(total, bool) + or not isinstance(passed, int) + or isinstance(passed, bool) + or passed < 0 + or passed > 2 + ): + raise ValueError("report summary contains invalid counts") + + modes: list[str] = [] + result_passes = 0 + for index, result in enumerate(results): + record = _mapping(result, f"report.results[{index}]") + mode = record.get("mode") + status = record.get("status") + if not isinstance(mode, str) or not isinstance(status, str): + raise ValueError(f"report.results[{index}] has invalid mode or status") + modes.append(mode) + result_passes += status == "passed" + + if total != len(results) or passed != result_passes: + raise ValueError("report summary does not match result records") + + score = passed / 2.0 + compatibility = _compatibility_result(model, score) + complete_pass = ( + total == 2 + and passed == 2 + and len(results) == 2 + and set(modes) == EXPECTED_MODES + and len(modes) == len(set(modes)) + ) + return compatibility, complete_pass + + +def _compatibility_result( + model: str, score: float, integration_error: BaseException | None = None +) -> dict[str, Any]: + result: dict[str, Any] = { + "lm_eval_version": "kimi-vendor-verifier", + "model_name": model, + "model_args": f"pretrained={model}", + "results": { + TASK_NAME: { + "exact_match,strict-match": score, + "exact_match_stderr,strict-match": 0.0, + } + }, + "configs": { + TASK_NAME: { + "task": TASK_NAME, + "output_type": "generate_until", + "num_fewshot": 0, + "repeats": 1, + "metric_list": [ + { + "metric": "exact_match", + "aggregation": "mean", + "higher_is_better": True, + } + ], + "filter_list": [ + {"name": "strict-match", "filter": [{"function": "identity"}]} + ], + } + }, + "versions": {TASK_NAME: 1}, + "n-shot": {TASK_NAME: 0}, + "higher_is_better": {TASK_NAME: {"exact_match": True}}, + "n-samples": {TASK_NAME: {"original": 2, "effective": 2}}, + } + if integration_error is not None: + result["integration_error"] = { + "type": type(integration_error).__name__, + "message": str(integration_error), + } + return result + + +def _write_compatibility(path: Path, result: Mapping[str, Any]) -> None: + path.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + + +def run_evaluation( + *, + verifier_dir: Path, + base_url: str, + api_key: str, + model: str, + output_dir: Path, +) -> bool: + """Run upstream pytest and always attempt to publish a compatibility result.""" + output_dir.mkdir(parents=True, exist_ok=True) + native_report = output_dir / NATIVE_REPORT_FILENAME + compatibility_path = prepare_compatibility_path(output_dir) + subprocess_rc: int | None = None + integration_error: BaseException | None = None + compatibility = _compatibility_result(model, 0.0) + complete_pass = False + + try: + native_report.unlink(missing_ok=True) + completed = subprocess.run( + build_pytest_command( + base_url=base_url, + api_key=api_key, + model=model, + report_path=native_report.resolve(), + ), + cwd=verifier_dir, + check=False, + ) + subprocess_rc = completed.returncode + report = json.loads(native_report.read_text(encoding="utf-8")) + compatibility, complete_pass = _project_report(model, report) + except (OSError, ValueError, json.JSONDecodeError) as exc: + integration_error = exc + compatibility = _compatibility_result(model, 0.0, exc) + finally: + try: + _write_compatibility(compatibility_path, compatibility) + except OSError as exc: + if integration_error is not None: + exc.add_note(f"Earlier integration error: {integration_error}") + raise + + return subprocess_rc == 0 and complete_pass and integration_error is None + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run the pinned stock Kimi Vendor Verifier tool-schema smoke test." + ) + parser.add_argument("--verifier-dir", type=Path) + parser.add_argument("--base-url") + parser.add_argument("--api-key", default="EMPTY") + parser.add_argument("--model", required=True) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument("--integration-error") + args = parser.parse_args(argv) + if args.integration_error is None: + missing = [ + option + for option, value in ( + ("--verifier-dir", args.verifier_dir), + ("--base-url", args.base_url), + ) + if value is None + ] + if missing: + parser.error( + f"{', '.join(missing)} required unless --integration-error is provided" + ) + return args + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + if args.integration_error is not None: + args.output_dir.mkdir(parents=True, exist_ok=True) + (args.output_dir / NATIVE_REPORT_FILENAME).unlink(missing_ok=True) + _write_compatibility( + prepare_compatibility_path(args.output_dir), + _compatibility_result( + args.model, 0.0, RuntimeError(args.integration_error) + ), + ) + return 1 + passed = run_evaluation( + verifier_dir=args.verifier_dir, + base_url=args.base_url, + api_key=args.api_key, + model=args.model, + output_dir=args.output_dir, + ) + return 0 if passed else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/utils/evals/test_kimi_vendor_eval.py b/utils/evals/test_kimi_vendor_eval.py new file mode 100644 index 0000000000..bfeb49819d --- /dev/null +++ b/utils/evals/test_kimi_vendor_eval.py @@ -0,0 +1,287 @@ +import json +import re +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import kimi_vendor_eval as kve + + +def _native_report(*, stream_status: str = "passed") -> dict: + statuses = ["passed", stream_status] + by_status: dict[str, int] = {} + for status in statuses: + by_status[status] = by_status.get(status, 0) + 1 + return { + "summary": { + "total": 2, + "by_status": by_status, + "by_selection_reason": {"object_schema": 2}, + "by_mode": { + "non-stream": {"passed": 1}, + "stream": {stream_status: 1}, + }, + }, + "results": [ + {"mode": "non-stream", "status": "passed"}, + {"mode": "stream", "status": stream_status}, + ], + } + + +def _compatibility_file(output_dir: Path) -> Path: + matches = list(output_dir.glob(kve.COMPATIBILITY_GLOB)) + assert len(matches) == 1 + assert re.fullmatch( + r"results_kimi_vendor_\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}\.\d{6}\.json", + matches[0].name, + ) + return matches[0] + + +def _projected(output_dir: Path) -> dict: + return json.loads(_compatibility_file(output_dir).read_text(encoding="utf-8")) + + +def _score(output_dir: Path) -> float: + return _projected(output_dir)["results"][kve.TASK_NAME][ + "exact_match,strict-match" + ] + + +def test_builds_exact_upstream_pytest_command(tmp_path: Path) -> None: + report = tmp_path / kve.NATIVE_REPORT_FILENAME + + command = kve.build_pytest_command( + base_url="http://127.0.0.1:8000/v1", + api_key="EMPTY", + model="test-model", + report_path=report, + ) + + assert command == [ + sys.executable, + "-m", + "pytest", + "tests/tool_call_json_schema/test_tool_call_json_schema.py", + "--base-url", + "http://127.0.0.1:8000/v1", + "--api-key", + "EMPTY", + "--smoke-model", + "test-model", + "--think-mode", + "none", + "--selection", + "object", + "--max-cases", + "1", + "--case-dir", + "testdata/walle_validator_cases/validator_cases", + "--max-tokens", + "2048", + "--tool-json-report", + str(report), + ] + + +def test_full_pass_projects_score_and_preserves_native_report( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + verifier_dir = tmp_path / "verifier" + verifier_dir.mkdir() + output_dir = tmp_path / "output" + output_dir.mkdir() + (output_dir / "results_kimi_vendor_2000-01-01T00-00-00.000000.json").write_text( + "stale", encoding="utf-8" + ) + native_bytes = (json.dumps(_native_report(), indent=2) + "\n").encode() + invocation: dict[str, object] = {} + + def fake_run(command: list[str], *, cwd: Path, check: bool) -> SimpleNamespace: + invocation.update(command=command, cwd=cwd, check=check) + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / kve.NATIVE_REPORT_FILENAME).write_bytes(native_bytes) + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(kve.subprocess, "run", fake_run) + + passed = kve.run_evaluation( + verifier_dir=verifier_dir, + base_url="http://localhost:8000/v1", + api_key="EMPTY", + model="model-a", + output_dir=output_dir, + ) + + assert passed + assert invocation["cwd"] == verifier_dir + assert invocation["check"] is False + assert invocation["command"] == kve.build_pytest_command( + base_url="http://localhost:8000/v1", + api_key="EMPTY", + model="model-a", + report_path=(output_dir / kve.NATIVE_REPORT_FILENAME).resolve(), + ) + assert (output_dir / kve.NATIVE_REPORT_FILENAME).read_bytes() == native_bytes + projected = _projected(output_dir) + assert _score(output_dir) == 1.0 + assert projected["n-samples"][kve.TASK_NAME] == {"original": 2, "effective": 2} + assert "integration_error" not in projected + + +def test_one_mode_failure_projects_partial_score_and_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + output_dir = tmp_path / "output" + + def fake_run(command: list[str], *, cwd: Path, check: bool) -> SimpleNamespace: + report_path = Path(command[command.index("--tool-json-report") + 1]) + report_path.write_text(json.dumps(_native_report(stream_status="failed"))) + return SimpleNamespace(returncode=1) + + monkeypatch.setattr(kve.subprocess, "run", fake_run) + + passed = kve.run_evaluation( + verifier_dir=tmp_path, + base_url="http://localhost/v1", + api_key="EMPTY", + model="model-a", + output_dir=output_dir, + ) + + assert not passed + assert _score(output_dir) == 0.5 + + +@pytest.mark.parametrize("native_contents", [None, "{not-json"]) +def test_missing_or_malformed_report_writes_zero_score_with_error( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + native_contents: str | None, +) -> None: + output_dir = tmp_path / "output" + + def fake_run(command: list[str], *, cwd: Path, check: bool) -> SimpleNamespace: + if native_contents is not None: + report_path = Path(command[command.index("--tool-json-report") + 1]) + report_path.write_text(native_contents, encoding="utf-8") + return SimpleNamespace(returncode=1) + + monkeypatch.setattr(kve.subprocess, "run", fake_run) + + passed = kve.run_evaluation( + verifier_dir=tmp_path, + base_url="http://localhost/v1", + api_key="EMPTY", + model="model-a", + output_dir=output_dir, + ) + + projected = _projected(output_dir) + assert not passed + assert _score(output_dir) == 0.0 + assert projected["integration_error"]["type"] in { + "FileNotFoundError", + "JSONDecodeError", + } + assert projected["integration_error"]["message"] + + +def test_collection_failure_cannot_project_a_stale_passing_report( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + output_dir = tmp_path / "output" + output_dir.mkdir() + native_report = output_dir / kve.NATIVE_REPORT_FILENAME + native_report.write_text(json.dumps(_native_report()), encoding="utf-8") + + def collection_failure( + command: list[str], *, cwd: Path, check: bool + ) -> SimpleNamespace: + assert not native_report.exists() + return SimpleNamespace(returncode=2) + + monkeypatch.setattr(kve.subprocess, "run", collection_failure) + + passed = kve.run_evaluation( + verifier_dir=tmp_path, + base_url="http://localhost/v1", + api_key="EMPTY", + model="model-a", + output_dir=output_dir, + ) + + projected = _projected(output_dir) + assert not passed + assert _score(output_dir) == 0.0 + assert projected["integration_error"]["type"] == "FileNotFoundError" + + +def test_subprocess_launch_failure_writes_zero_score_with_error( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def fail_to_launch(*args: object, **kwargs: object) -> subprocess.CompletedProcess: + raise OSError("pytest could not launch") + + monkeypatch.setattr(kve.subprocess, "run", fail_to_launch) + output_dir = tmp_path / "output" + + passed = kve.run_evaluation( + verifier_dir=tmp_path, + base_url="http://localhost/v1", + api_key="EMPTY", + model="model-a", + output_dir=output_dir, + ) + + projected = _projected(output_dir) + assert not passed + assert _score(output_dir) == 0.0 + assert projected["integration_error"] == { + "type": "OSError", + "message": "pytest could not launch", + } + + + +def test_cli_integration_error_writes_failure_without_running_pytest( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def unexpected_run(*args: object, **kwargs: object) -> None: + pytest.fail("integration-error mode must not launch pytest") + + monkeypatch.setattr(kve.subprocess, "run", unexpected_run) + output_dir = tmp_path / "output" + output_dir.mkdir() + (output_dir / "results_kimi_vendor_2000-01-01T00-00-00.000000.json").write_text( + "stale", encoding="utf-8" + ) + (output_dir / kve.NATIVE_REPORT_FILENAME).write_text( + json.dumps(_native_report()), encoding="utf-8" + ) + + return_code = kve.main( + [ + "--model", + "model-a", + "--output-dir", + str(output_dir), + "--integration-error", + "pinned verifier checkout failed", + ] + ) + + projected = _projected(output_dir) + assert return_code == 1 + assert not (output_dir / kve.NATIVE_REPORT_FILENAME).exists() + assert _score(output_dir) == 0.0 + assert projected["integration_error"] == { + "type": "RuntimeError", + "message": "pinned verifier checkout failed", + } \ No newline at end of file diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index 45be3a6e4d..067f2165e0 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -1,25 +1,36 @@ from __future__ import annotations +import json import os import stat import subprocess from pathlib import Path +import pytest + BENCHMARK_LIB = Path(__file__).resolve().parents[2] / "benchmarks" / "benchmark_lib.sh" _SCRIPT = r''' source "$BENCHMARK_LIB" run_lm_eval() { echo "DISPATCH=lm-eval"; } run_swebench_eval() { echo "DISPATCH=swebench"; } -append_lm_eval_summary() { echo "STAGED=summary"; } +run_tool_use_eval() { echo "DISPATCH=tool-use"; } +append_lm_eval_summary() { echo "STAGED=summary FRAMEWORK=$EVAL_FRAMEWORK"; } export EVAL_MAX_MODEL_LEN=16384 -unset EVAL_CONCURRENT_REQUESTS +export EVAL_CONCURRENT_REQUESTS="${REQUESTED_CONC:-}" run_eval ${CLI_FW:+--framework "$CLI_FW"} --port 8888 ''' -def _dispatch(*, is_agentic: str = "0", eval_only: str = "false", cli_fw=None, env_fw=None) -> str: +def _dispatch( + *, + is_agentic: str = "0", + eval_only: str = "false", + cli_fw=None, + env_fw=None, + requested_conc=None, +) -> str: env = { **os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB), @@ -29,11 +40,14 @@ def _dispatch(*, is_agentic: str = "0", eval_only: str = "false", cli_fw=None, e } env.pop("EVAL_FRAMEWORK", None) env.pop("CLI_FW", None) + env.pop("REQUESTED_CONC", None) env.pop("KV_OFFLOAD_BACKEND", None) if cli_fw is not None: env["CLI_FW"] = cli_fw if env_fw is not None: env["EVAL_FRAMEWORK"] = env_fw + if requested_conc is not None: + env["REQUESTED_CONC"] = str(requested_conc) res = subprocess.run( ["bash", "-c", _SCRIPT], env=env, text=True, capture_output=True, check=True ) @@ -52,6 +66,7 @@ def test_agentic_eval_only_stages_summary(): output = _dispatch(is_agentic="1", eval_only="true") assert "DISPATCH=lm-eval" in output assert "STAGED=summary" in output + assert "FRAMEWORK=lm-eval" in output def test_fixed_seqlen_eval_only_leaves_staging_to_recipe(): @@ -71,10 +86,73 @@ def test_env_can_force_swebench_on_fixed_seqlen(): assert "DISPATCH=swebench" in _dispatch(is_agentic="0", env_fw="swebench") +def test_cli_swebench_framework_is_canonical_in_metadata() -> None: + output = _dispatch( + is_agentic="1", + eval_only="true", + cli_fw="swebench", + ) + assert "DISPATCH=swebench" in output + assert "FRAMEWORK=swebench" in output + + +def test_env_can_force_tool_use_on_agentic_eval() -> None: + output = _dispatch( + is_agentic="1", + eval_only="true", + env_fw="tool-use", + ) + assert "DISPATCH=tool-use" in output + assert "FRAMEWORK=tool-use" in output + + +def test_tool_use_skips_unused_model_context_loading() -> None: + script = r''' +source "$BENCHMARK_LIB" +unset EVAL_MAX_MODEL_LEN +compute_eval_context_length() { echo "UNEXPECTED_CONTEXT_LOAD"; return 99; } +run_tool_use_eval() { echo "DISPATCH=tool-use"; } +export EVAL_FRAMEWORK=tool-use +export EVAL_CONCURRENT_REQUESTS="" +export EVAL_ONLY=false +export IS_AGENTIC=0 +run_eval --port 8888 +''' + result = subprocess.run( + ["bash", "-c", script], + env={**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB)}, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert "DISPATCH=tool-use" in result.stdout + assert "UNEXPECTED_CONTEXT_LOAD" not in result.stdout + + +def test_tool_use_accepts_single_matrix_concurrency_identity() -> None: + assert "DISPATCH=tool-use" in _dispatch( + is_agentic="1", + env_fw="tool-use", + requested_conc=64, + ) + + def test_recipe_lm_eval_arg_still_lm_eval_on_fixed_seqlen(): assert "DISPATCH=lm-eval" in _dispatch(is_agentic="0", cli_fw="lm-eval") +def test_lm_eval_alias_is_canonicalized_in_metadata() -> None: + output = _dispatch( + is_agentic="1", + eval_only="true", + cli_fw="lm_eval", + ) + assert "DISPATCH=lm-eval" in output + assert "FRAMEWORK=lm-eval" in output + + def _run_invalid_call(call: str) -> subprocess.CompletedProcess: env = { **os.environ, @@ -95,6 +173,298 @@ def test_run_eval_rejects_missing_framework_value(): assert "--framework requires a value" in result.stderr +def test_tool_use_rejects_batched_concurrency() -> None: + result = _run_invalid_call( + "EVAL_MAX_MODEL_LEN=16384 " + "EVAL_CONCURRENT_REQUESTS='1 4' " + "run_eval --framework tool-use" + ) + assert result.returncode == 1 + assert "batched eval concurrency is only supported for lm-eval" in result.stderr + + +def test_tool_use_rejects_unsupported_suite() -> None: + result = _run_invalid_call( + "EVAL_SUITE=gsm8k run_tool_use_eval" + ) + assert result.returncode == 2 + assert "supports only EVAL_SUITE=kimi_tool_call_schema" in result.stderr + + +def test_tool_use_rejects_multinode() -> None: + for value in ("true", "1"): + result = _run_invalid_call( + f"EVAL_SUITE=kimi_tool_call_schema IS_MULTINODE={value} " + "run_tool_use_eval" + ) + assert result.returncode == 2 + assert "supports single-node evals only" in result.stderr + + +@pytest.mark.parametrize( + ("failure_stage", "failure_rc", "message"), + ( + ("python", 11, "tool-use Python version check failed"), + ("dependencies", 12, "tool-use dependency installation failed"), + ("checkout", 13, "tool-use verifier checkout failed"), + ), +) +def test_tool_use_setup_failure_writes_compatibility_result( + tmp_path: Path, + failure_stage: str, + failure_rc: int, + message: str, +) -> None: + results_dir = tmp_path / "results" + script = r''' +source "$BENCHMARK_LIB" +_require_tool_use_python() { + [ "$FAILURE_STAGE" = python ] && return "$FAILURE_RC" + return 0 +} +_install_tool_use_eval_deps() { + [ "$FAILURE_STAGE" = dependencies ] && return "$FAILURE_RC" + return 0 +} +_prepare_kimi_vendor_verifier() { + [ "$FAILURE_STAGE" = checkout ] && return "$FAILURE_RC" + KIMI_VENDOR_VERIFIER_CHECKOUT_DIR="$FAKE_VERIFIER_DIR" + return 0 +} +run_tool_use_eval --results-dir "$RESULTS_DIR" +printf 'SETUP_RC=%s\n' "$?" +''' + env = { + **os.environ, + "BENCHMARK_LIB": str(BENCHMARK_LIB), + "RESULTS_DIR": str(results_dir), + "FAKE_VERIFIER_DIR": str(tmp_path / "verifier"), + "FAILURE_STAGE": failure_stage, + "FAILURE_RC": str(failure_rc), + "MODEL": "test-model", + "IS_MULTINODE": "false", + "KV_OFFLOADING": "none", + } + for key in ( + "EVAL_FRAMEWORK", + "EVAL_SUITE", + "EVAL_RESULT_DIR", + "INFERENCEX_TOOL_USE_EVAL_RUNTIME_READY", + "KIMI_VENDOR_VERIFIER_DIR", + "KIMI_VENDOR_VERIFIER_CHECKOUT_DIR", + "MODEL_NAME", + ): + env.pop(key, None) + + result = subprocess.run( + ["bash", "-c", script], + env=env, + text=True, + capture_output=True, + check=True, + ) + expected_message = f"{message} with exit code {failure_rc}" + + assert f"SETUP_RC={failure_rc}" in result.stdout + assert expected_message in result.stderr + score_files = list(results_dir.glob("results*.json")) + assert len(score_files) == 1 + score_result = json.loads(score_files[0].read_text()) + assert ( + score_result["results"]["kimi_tool_call_schema"][ + "exact_match,strict-match" + ] + == 0.0 + ) + assert score_result["integration_error"]["message"] == expected_message + assert not (results_dir / "kimi_vendor_report.json").exists() + + +def test_kimi_vendor_checkout_rejects_source_changes(tmp_path: Path) -> None: + checkout = tmp_path / "verifier" + required_files = ( + "LICENSE", + "pyproject.toml", + "tests/conftest.py", + "tests/__init__.py", + "tests/tool_call_json_schema/conftest.py", + "tests/tool_call_json_schema/validator.py", + "tests/tool_call_json_schema/test_tool_call_json_schema.py", + "testdata/walle_validator_cases/validator_cases/case.jsonl", + ) + for relative_path in required_files: + path = checkout / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"{relative_path}\n") + + subprocess.run(["git", "init", "-q", str(checkout)], check=True) + subprocess.run(["git", "-C", str(checkout), "add", "."], check=True) + subprocess.run( + [ + "git", + "-C", + str(checkout), + "-c", + "user.name=InferenceX Tests", + "-c", + "user.email=tests@inferencex.invalid", + "commit", + "-qm", + "fixture", + ], + check=True, + ) + verifier_ref = subprocess.run( + ["git", "-C", str(checkout), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + def checkout_is_valid() -> bool: + result = subprocess.run( + [ + "bash", + "-c", + 'source "$BENCHMARK_LIB"; ' + '_kimi_vendor_checkout_is_valid "$CHECKOUT" "$VERIFIER_REF"', + ], + env={ + **os.environ, + "BENCHMARK_LIB": str(BENCHMARK_LIB), + "CHECKOUT": str(checkout), + "VERIFIER_REF": verifier_ref, + "KV_OFFLOADING": "none", + }, + ) + return result.returncode == 0 + + assert checkout_is_valid() + pytest_cache = checkout / ".pytest_cache" / "v" / "cache" / "nodeids" + pytest_cache.parent.mkdir(parents=True) + pytest_cache.write_text("[]\n") + assert checkout_is_valid() + + with (checkout / ".git/info/exclude").open("a") as exclude_file: + exclude_file.write("\n/conftest.py\n") + root_override = checkout / "conftest.py" + root_override.write_text("# ignored root override\n") + assert not checkout_is_valid() + root_override.unlink() + + root_override.write_text("# staged root override\n") + subprocess.run( + ["git", "-C", str(checkout), "add", "-f", "conftest.py"], + check=True, + ) + assert not checkout_is_valid() + subprocess.run( + ["git", "-C", str(checkout), "reset", "-q", "HEAD", "--", "conftest.py"], + check=True, + ) + root_override.unlink() + + override = checkout / "tests/tool_call_json_schema/local_override.py" + override.write_text("# untracked override\n") + assert not checkout_is_valid() + override.unlink() + + validator = checkout / "tests/tool_call_json_schema/validator.py" + validator.write_text("# modified verifier\n") + assert not checkout_is_valid() + + +def test_tool_use_runner_uses_fixed_upstream_contract(tmp_path: Path) -> None: + shim_dir = tmp_path / "bin" + shim_dir.mkdir() + python_shim = shim_dir / "python3" + python_shim.write_text( + "#!/usr/bin/env bash\n" + "printf 'PYTHON_ARG=<%s>\\n' \"$@\"\n" + ) + python_shim.chmod( + python_shim.stat().st_mode | stat.S_IXUSR + ) + results_dir = tmp_path / "results" + verifier_dir = tmp_path / "verifier" + script = r''' +source "$BENCHMARK_LIB" +_require_tool_use_python() { echo "PYTHON_VERSION=OK"; } +_install_tool_use_eval_deps() { echo "INSTALL=UPSTREAM_MINIMAL"; } +_prepare_kimi_vendor_verifier() { + printf 'CHECKOUT_REPO=%s\n' "$1" + printf 'CHECKOUT_REF=%s\n' "$2" + KIMI_VENDOR_VERIFIER_CHECKOUT_DIR="$VERIFIER_DIR" +} +run_tool_use_eval --port 9999 --results-dir "$RESULTS_DIR" +printf 'EVAL_FRAMEWORK=%s\n' "$EVAL_FRAMEWORK" +printf 'EVAL_SUITE=%s\n' "$EVAL_SUITE" +printf 'EVAL_RESULT_DIR=%s\n' "$EVAL_RESULT_DIR" +printf 'RUNTIME_READY=%s\n' "$INFERENCEX_TOOL_USE_EVAL_RUNTIME_READY" +''' + env = { + **os.environ, + "BENCHMARK_LIB": str(BENCHMARK_LIB), + "RESULTS_DIR": str(results_dir), + "VERIFIER_DIR": str(verifier_dir), + "MODEL": "test-model", + "OPENAI_API_KEY": "must-not-be-forwarded", + "PATH": f"{shim_dir}:{os.environ['PATH']}", + "KV_OFFLOADING": "none", + "IS_MULTINODE": "false", + } + for key in ( + "EVAL_FRAMEWORK", + "EVAL_SUITE", + "EVAL_RESULT_DIR", + "INFERENCEX_TOOL_USE_EVAL_RUNTIME_READY", + "KIMI_VENDOR_VERIFIER_DIR", + "KIMI_VENDOR_VERIFIER_CHECKOUT_DIR", + "MODEL_NAME", + ): + env.pop(key, None) + result = subprocess.run( + ["bash", "-c", script], + env=env, + text=True, + capture_output=True, + check=True, + ) + output = result.stdout + expected_adapter = ( + BENCHMARK_LIB.parents[1] / "utils/evals/kimi_vendor_eval.py" + ) + + assert "PYTHON_VERSION=OK" in output + assert output.count("INSTALL=UPSTREAM_MINIMAL") == 1 + assert ( + "CHECKOUT_REPO=https://github.com/MoonshotAI/Kimi-Vendor-Verifier.git" + in output + ) + assert ( + "CHECKOUT_REF=b9ed3a6665bdff2c943246f7d2903cd003d6ddd6" + in output + ) + assert f"PYTHON_ARG=<{expected_adapter}>" in output + for flag in ( + "--verifier-dir", + "--base-url", + "--api-key", + "--model", + "--output-dir", + ): + assert f"PYTHON_ARG=<{flag}>" in output + assert f"PYTHON_ARG=<{verifier_dir}>" in output + assert "PYTHON_ARG=" in output + assert "PYTHON_ARG=" in output + assert "PYTHON_ARG=" in output + assert f"PYTHON_ARG=<{results_dir}>" in output + assert "EVAL_FRAMEWORK=tool-use" in output + assert "EVAL_SUITE=kimi_tool_call_schema" in output + assert f"EVAL_RESULT_DIR={results_dir}" in output + assert "RUNTIME_READY=true" in output + + def test_run_lm_eval_rejects_missing_option_value(): result = _run_invalid_call("run_lm_eval --port") assert result.returncode == 2 @@ -177,6 +547,94 @@ def test_lm_eval_defaults_to_gsm8k(): assert "utils/evals/gsm8k.yaml" in out +def _summary_metadata(tmp_path: Path, **overrides: str) -> dict: + work_dir = tmp_path / "work" + results_dir = tmp_path / "results" + work_dir.mkdir(parents=True) + results_dir.mkdir() + script = r''' +source "$BENCHMARK_LIB" +cd "$WORK_DIR" +append_lm_eval_summary >/dev/null +''' + env = { + **os.environ, + "BENCHMARK_LIB": str(BENCHMARK_LIB), + "WORK_DIR": str(work_dir), + "EVAL_RESULT_DIR": str(results_dir), + "MODEL": "test-model", + "CONC": "7", + "KV_OFFLOADING": "none", + } + for key in ("EVAL_FRAMEWORK", "EVAL_SUITE", "EVAL_TASKS_DIR"): + env.pop(key, None) + env.update(overrides) + subprocess.run(["bash", "-c", script], env=env, check=True) + return json.loads((work_dir / "meta_env.json").read_text()) + + +def test_summary_metadata_preserves_lm_eval_gsm8k_defaults(tmp_path: Path) -> None: + meta = _summary_metadata(tmp_path) + + assert meta["eval_framework"] == "lm-eval" + assert meta["eval_suite"] == "gsm8k" + assert meta["conc"] == 7 + + +def test_run_lm_eval_exports_cli_task_suite_to_metadata(tmp_path: Path) -> None: + work_dir = tmp_path / "work" + results_dir = tmp_path / "results" + work_dir.mkdir() + results_dir.mkdir() + script = r''' +set -e +source "$BENCHMARK_LIB" +cd "$WORK_DIR" +python3() { :; } +export EVAL_MAX_MODEL_LEN=16384 +export INFERENCEX_LM_EVAL_RUNTIME_READY=true +run_lm_eval --task custom.yaml --results-dir "$RESULTS_DIR" +append_lm_eval_summary >/dev/null +''' + env = { + **os.environ, + "BENCHMARK_LIB": str(BENCHMARK_LIB), + "WORK_DIR": str(work_dir), + "RESULTS_DIR": str(results_dir), + "MODEL": "test-model", + "MODEL_NAME": "test-model", + "OPENAI_API_KEY": "EMPTY", + "KV_OFFLOADING": "none", + } + for key in ("EVAL_FRAMEWORK", "EVAL_SUITE", "EVAL_TASKS_DIR"): + env.pop(key, None) + + subprocess.run(["bash", "-c", script], env=env, check=True) + meta = json.loads((work_dir / "meta_env.json").read_text()) + + assert meta["eval_framework"] == "lm-eval" + assert meta["eval_suite"] == "custom" + + +def test_summary_metadata_prefers_explicit_suite_then_task_basename( + tmp_path: Path, +) -> None: + from_task = _summary_metadata( + tmp_path / "task", + EVAL_TASKS_DIR="/tmp/custom_reasoning.yaml", + ) + explicit = _summary_metadata( + tmp_path / "explicit", + EVAL_FRAMEWORK="tool-use", + EVAL_SUITE="kimi_tool_call_schema", + EVAL_TASKS_DIR="/tmp/ignored.yaml", + ) + + assert from_task["eval_suite"] == "custom_reasoning" + assert explicit["eval_framework"] == "tool-use" + assert explicit["eval_suite"] == "kimi_tool_call_schema" + + _MODAL_CREDS_SCRIPT = r''' source "$BENCHMARK_LIB" @@ -552,14 +1010,28 @@ def test_agentic_eval_limit_full_runs_whole_split(tmp_path): source "$BENCHMARK_LIB" 2>/dev/null _install_swebench_agent_deps() { :; } _ensure_modal_credentials() { :; } -_run_swebench_agentic_generation() { echo "GEN=agentic"; return 42; } -run_lm_eval() { echo "GEN=single-shot"; return 42; } +_run_swebench_agentic_generation() { + echo "GEN=agentic" + echo "SUITE=$EVAL_SUITE" + return 42 +} +run_lm_eval() { + echo "GEN=single-shot" + echo "SUITE=$EVAL_SUITE" + return 42 +} run_swebench_eval --port 8888 echo "RC=$?" ''' -def _gen_mode(tmp_path, *, is_agentic, gen_mode=None) -> str: +def _gen_mode( + tmp_path: Path, + *, + is_agentic, + gen_mode=None, + eval_suite=None, +) -> str: env = {**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB), "KV_OFFLOADING": "none", @@ -567,8 +1039,11 @@ def _gen_mode(tmp_path, *, is_agentic, gen_mode=None) -> str: "EVAL_RESULT_DIR": str(tmp_path / "out")} env.pop("SWEBENCH_GEN_MODE", None) env.pop("SCENARIO_TYPE", None) + env.pop("EVAL_SUITE", None) if gen_mode is not None: env["SWEBENCH_GEN_MODE"] = gen_mode + if eval_suite is not None: + env["EVAL_SUITE"] = eval_suite res = subprocess.run(["bash", "-c", _GENMODE_SCRIPT], env=env, text=True, capture_output=True, cwd=BENCHMARK_LIB.parents[1]) @@ -577,7 +1052,9 @@ def _gen_mode(tmp_path, *, is_agentic, gen_mode=None) -> str: def test_gen_mode_defaults_to_agentic(tmp_path): - assert "GEN=agentic" in _gen_mode(tmp_path, is_agentic="1") + output = _gen_mode(tmp_path, is_agentic="1") + assert "GEN=agentic" in output + assert "SUITE=swebench_lite" in output def test_gen_mode_agentic_even_without_agentic_scenario(tmp_path): @@ -585,7 +1062,20 @@ def test_gen_mode_agentic_even_without_agentic_scenario(tmp_path): def test_explicit_single_shot_escape_hatch(tmp_path): - assert "GEN=single-shot" in _gen_mode(tmp_path, is_agentic="1", gen_mode="single-shot") + output = _gen_mode(tmp_path, is_agentic="1", gen_mode="single-shot") + assert "GEN=single-shot" in output + assert "SUITE=swebench_lite" in output + + +def test_swebench_generation_modes_preserve_explicit_suite(tmp_path): + for gen_mode in ("agentic", "single-shot"): + output = _gen_mode( + tmp_path / gen_mode, + is_agentic="1", + gen_mode=gen_mode, + eval_suite="explicit_swebench", + ) + assert "SUITE=explicit_swebench" in output def test_agent_sandbox_cpu_knob(tmp_path): diff --git a/utils/evals/thresholds.yaml b/utils/evals/thresholds.yaml index 6ff9731c45..bb3a4f58b6 100644 --- a/utils/evals/thresholds.yaml +++ b/utils/evals/thresholds.yaml @@ -1,6 +1,7 @@ # Model thresholds override task defaults. default: gsm8k: 0.90 + kimi_tool_call_schema: 1.0 gpqa_diamond_cot_n_shot: 0.30 swebench_lite: 0.50 models: diff --git a/utils/test_collect_eval_results.py b/utils/test_collect_eval_results.py index 019bbdf123..200be70e64 100644 --- a/utils/test_collect_eval_results.py +++ b/utils/test_collect_eval_results.py @@ -21,6 +21,21 @@ def test_build_row_preserves_sequence_lengths() -> None: assert row["isl"] == 1024 assert row["osl"] == 1024 + assert "eval_framework" not in row + assert "eval_suite" not in row + + +def test_build_row_preserves_explicit_eval_metadata() -> None: + row = build_row( + { + "eval_framework": "tool-use", + "eval_suite": "kimi_tool_call_schema", + }, + {"task": "kimi_tool_call_schema"}, + ) + + assert row["eval_framework"] == "tool-use" + assert row["eval_suite"] == "kimi_tool_call_schema" def _write_lm_eval_result(path: Path, score: float) -> None: @@ -67,6 +82,8 @@ def test_collect_eval_rows_expands_batched_concurrencies( "completed_eval_concs": [4, 16], "failed_eval_concs": [], "conc": 4, + "eval_framework": "lm-eval", + "eval_suite": "gsm8k", })) _write_lm_eval_result( artifact_dir / "results_test_conc4.json", @@ -81,6 +98,8 @@ def test_collect_eval_rows_expands_batched_concurrencies( assert [row["conc"] for row in rows] == [4, 16] assert [row["score"] for row in rows] == [0.90, 0.91] + assert {row["eval_framework"] for row in rows} == {"lm-eval"} + assert {row["eval_suite"] for row in rows} == {"gsm8k"} def test_collect_eval_rows_ignores_failed_batch_points( diff --git a/utils/test_validate_reusable_sweep_artifacts.py b/utils/test_validate_reusable_sweep_artifacts.py index 69eb633cdc..285bb6fcb3 100644 --- a/utils/test_validate_reusable_sweep_artifacts.py +++ b/utils/test_validate_reusable_sweep_artifacts.py @@ -32,8 +32,9 @@ def single_eval_result( runner: str = "h100-dgxc-slurm", isl: int = 8192, osl: int = 1024, + eval_suite: str | None = None, ) -> dict: - return { + row = { "is_multinode": False, "hw": runner.upper(), "model_prefix": "gptoss", @@ -51,6 +52,9 @@ def single_eval_result( "conc": conc, "task": "gsm8k", } + if eval_suite is not None: + row["eval_suite"] = eval_suite + return row def single_eval_meta( @@ -58,8 +62,9 @@ def single_eval_meta( runner: str = "h100-dgxc-slurm", isl: int = 8192, osl: int = 1024, + eval_suite: str | None = None, ) -> dict: - row = single_eval_result(conc, runner, isl, osl) + row = single_eval_result(conc, runner, isl, osl, eval_suite) row["infmax_model_prefix"] = row.pop("model_prefix") return row @@ -72,11 +77,20 @@ def write_raw_eval_artifact( physical_runner: str = "h100-dgxc-slurm_00", isl: int = 8192, osl: int = 1024, + eval_suite: str | None = None, ) -> None: artifact_dir = root / f"eval_result_conc{conc}_{physical_runner}" artifact_dir.mkdir() (artifact_dir / "meta_env.json").write_text( - json.dumps(single_eval_meta(conc, logical_runner, isl, osl)) + json.dumps( + single_eval_meta( + conc, + logical_runner, + isl, + osl, + eval_suite, + ) + ) ) @@ -330,7 +344,7 @@ def test_eval_validation_requires_raw_result_dirs_not_eval_debug_dirs( assert any("unexpected" in error for error in errors) -def test_eval_validation_accepts_matching_raw_and_aggregate( +def test_eval_validation_accepts_matching_legacy_artifacts_without_suite( tmp_path: Path, ) -> None: write_eval_aggregate( @@ -347,6 +361,31 @@ def test_eval_validation_accepts_matching_raw_and_aggregate( assert validate_eval_artifacts(tmp_path) == [] +def test_eval_validation_separates_explicit_suite_identities( + tmp_path: Path, +) -> None: + gsm8k = single_eval_result(32, eval_suite="gsm8k") + tool_use = single_eval_result( + 32, + eval_suite="kimi_tool_call_schema", + ) + write_eval_aggregate(tmp_path, [gsm8k, tool_use]) + write_raw_eval_artifact( + tmp_path, + 32, + eval_suite="gsm8k", + ) + write_raw_eval_artifact( + tmp_path, + 32, + physical_runner="h100-dgxc-slurm_01", + eval_suite="kimi_tool_call_schema", + ) + + assert eval_key(gsm8k) != eval_key(tool_use) + assert validate_eval_artifacts(tmp_path) == [] + + def test_eval_validation_distinguishes_sequence_lengths(tmp_path: Path) -> None: write_eval_aggregate( tmp_path, @@ -634,18 +673,23 @@ def _dd_write_aggregate(root: Path, rows: list[dict]) -> Path: def _dd_write_legacy_raw( - root: Path, name: str, conc: int, timestamp: str | None + root: Path, + name: str, + conc: int, + timestamp: str | None, + result_prefix: str = "results_", ) -> None: artifact_dir = root / name artifact_dir.mkdir() (artifact_dir / "meta_env.json").write_text(json.dumps(_dd_meta(conc))) if timestamp is not None: - (artifact_dir / f"results_{timestamp}.json").write_text("{}") + (artifact_dir / f"{result_prefix}{timestamp}.json").write_text("{}") def test_dedupe_keeps_latest_legacy_rerun(tmp_path: Path) -> None: # Three reruns of one eval plus a result-less attempt, mirroring a flaky # config retried until it passed. + # The latest rerun uses the tool-use adapter's timestamped result prefix. old, mid, new, empty = ( "eval_minimaxm3_conc4096_b300-nv_15", "eval_minimaxm3_conc4096_b300-nv_16", @@ -654,13 +698,23 @@ def test_dedupe_keeps_latest_legacy_rerun(tmp_path: Path) -> None: ) _dd_write_legacy_raw(tmp_path, old, 4096, "2026-06-26T13-00-22.596040") _dd_write_legacy_raw(tmp_path, mid, 4096, "2026-06-26T19-00-52.356121") - _dd_write_legacy_raw(tmp_path, new, 4096, "2026-06-27T04-28-31.838775") + _dd_write_legacy_raw( + tmp_path, + new, + 4096, + "2026-06-27T04-28-31.838775", + result_prefix="results_kimi_vendor_", + ) _dd_write_legacy_raw(tmp_path, empty, 4096, None) _dd_write_aggregate( tmp_path, [ _dd_agg_row(4096, f"eval_results/{old}/results_2026-06-26T13-00-22.596040.json", 0.83), - _dd_agg_row(4096, f"eval_results/{new}/results_2026-06-27T04-28-31.838775.json", 0.95), + _dd_agg_row( + 4096, + f"eval_results/{new}/results_kimi_vendor_2026-06-27T04-28-31.838775.json", + 0.95, + ), _dd_agg_row(4096, f"eval_results/{mid}/results_2026-06-26T19-00-52.356121.json", 0.78), ], ) diff --git a/utils/validate_reusable_sweep_artifacts.py b/utils/validate_reusable_sweep_artifacts.py index 0cfd1d2662..549547b203 100644 --- a/utils/validate_reusable_sweep_artifacts.py +++ b/utils/validate_reusable_sweep_artifacts.py @@ -313,6 +313,16 @@ def normalized_runner(value: Any) -> str: return str(value or "").lower() +LEGACY_EVAL_SUITE = "" + + +def eval_suite_identity(row: dict[str, Any]) -> Any: + """Return an explicit suite or the compatibility identity for old artifacts.""" + if "eval_suite" in row: + return row["eval_suite"] + return LEGACY_EVAL_SUITE + + def eval_key(row: dict[str, Any]) -> tuple[Any, ...]: """Build an eval identity from one aggregate row.""" if as_bool(row.get("is_multinode", False)): @@ -322,6 +332,7 @@ def eval_key(row: dict[str, Any]) -> tuple[Any, ...]: row.get("model_prefix", row.get("infmax_model_prefix")), row.get("framework"), row.get("precision"), + eval_suite_identity(row), row.get("spec_decoding", "none"), as_int(row.get("isl", 8192), 8192), as_int(row.get("osl", 1024), 1024), @@ -347,6 +358,7 @@ def eval_key(row: dict[str, Any]) -> tuple[Any, ...]: row.get("model_prefix", row.get("infmax_model_prefix")), row.get("framework"), row.get("precision"), + eval_suite_identity(row), row.get("spec_decoding", "none"), as_int(row.get("isl", 8192), 8192), as_int(row.get("osl", 1024), 1024), From b7d0d7e35ac2c1290d25676e2a15c966199f3b02 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:36:45 -0500 Subject: [PATCH 02/24] refactor: simplify and expose tool-use eval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：精简工具调用评估实现并接入工作流选择路径 --- .github/workflows/benchmark-tmpl.yml | 6 + .github/workflows/e2e-tests.yml | 11 + benchmarks/benchmark_lib.sh | 124 ++------ utils/collect_eval_results.py | 5 +- utils/evals/EVALS.md | 99 ++----- utils/evals/kimi_vendor_eval.py | 41 +-- utils/evals/test_kimi_vendor_eval.py | 241 ++++++---------- utils/evals/test_run_eval_dispatch.py | 264 +++--------------- utils/test_collect_eval_results.py | 11 +- .../test_validate_reusable_sweep_artifacts.py | 23 +- utils/validate_reusable_sweep_artifacts.py | 10 +- 11 files changed, 204 insertions(+), 631 deletions(-) diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 4dc036a06d..e982384336 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -85,6 +85,11 @@ on: type: boolean required: false default: false + eval-framework: + description: "Eval runner (lm-eval, swebench, or tool-use)" + type: string + required: false + default: "lm-eval" random-range-ratio: required: false type: string @@ -173,6 +178,7 @@ env: DISAGG: ${{ inputs.disagg }} RUN_EVAL: ${{ inputs.run-eval }} EVAL_ONLY: ${{ inputs.eval-only }} + EVAL_FRAMEWORK: ${{ inputs.eval-framework }} # Agentic-coding env. Fixed-seq-len jobs leave these empty. SCENARIO_TYPE: ${{ inputs.scenario-type }} SCENARIO_SUBDIR: ${{ inputs.scenario-type == 'agentic-coding' && 'agentic/' || 'fixed_seq_len/' }} diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 9648605fad..83b48333f9 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -45,6 +45,11 @@ on: required: false type: string default: "" + eval-framework: + description: "Agentic eval runner (lm-eval, swebench, or tool-use)" + required: false + type: string + default: "lm-eval" swebench-gen-mode: description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)." required: false @@ -125,6 +130,11 @@ on: required: false type: string default: "" + eval-framework: + description: "Agentic eval runner (lm-eval, swebench, or tool-use)" + required: false + type: string + default: "lm-eval" swebench-gen-mode: description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)." required: false @@ -459,6 +469,7 @@ jobs: eval-only: true eval-limit: ${{ inputs.eval-limit }} swebench-gen-mode: ${{ inputs.swebench-gen-mode }} + eval-framework: ${{ inputs.eval-framework }} scenario-type: agentic-coding ref: ${{ inputs.ref }} diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 5519682c8f..b621c9d923 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -838,87 +838,23 @@ _install_tool_use_eval_deps() { "pytest==8.4.2" } -_kimi_vendor_checkout_is_valid() { - local checkout_dir="$1" - local expected_ref="$2" - local checkout_ref checkout_status tracked_status untracked_files ignored_files - - [ -f "${checkout_dir}/LICENSE" ] \ - && [ -f "${checkout_dir}/pyproject.toml" ] \ - && [ -f "${checkout_dir}/tests/conftest.py" ] \ - && [ -f "${checkout_dir}/tests/__init__.py" ] \ - && [ -f "${checkout_dir}/tests/tool_call_json_schema/conftest.py" ] \ - && [ -f "${checkout_dir}/tests/tool_call_json_schema/validator.py" ] \ - && [ -f "${checkout_dir}/tests/tool_call_json_schema/test_tool_call_json_schema.py" ] \ - && [ -d "${checkout_dir}/testdata/walle_validator_cases/validator_cases" ] \ - || return 1 - checkout_ref="$(git -C "$checkout_dir" rev-parse HEAD 2>/dev/null)" \ - || return 1 - [ "$checkout_ref" = "$expected_ref" ] || return 1 - checkout_status="$( - git -C "$checkout_dir" status --porcelain --untracked-files=all -- \ - LICENSE \ - pyproject.toml \ - tests/conftest.py \ - tests/__init__.py \ - tests/tool_call_json_schema \ - testdata/walle_validator_cases - )" || return 1 - [ -z "$checkout_status" ] || return 1 - tracked_status="$( - git -C "$checkout_dir" status --porcelain --untracked-files=no - )" || return 1 - [ -z "$tracked_status" ] || return 1 - untracked_files="$( - git -C "$checkout_dir" ls-files --others --exclude-standard -- \ - . ':(exclude,top,glob).pytest_cache/**' - )" || return 1 - [ -z "$untracked_files" ] || return 1 - ignored_files="$( - git -C "$checkout_dir" ls-files --others --ignored --exclude-standard -- \ - . ':(exclude,top,glob).pytest_cache/**' - )" || return 1 - [ -z "$ignored_files" ] -} - _prepare_kimi_vendor_verifier() { local repo_url="$1" local verifier_ref="$2" local checkout_dir - if [ -n "${KIMI_VENDOR_VERIFIER_DIR:-}" ]; then - checkout_dir="$KIMI_VENDOR_VERIFIER_DIR" - if ! _kimi_vendor_checkout_is_valid "$checkout_dir" "$verifier_ref"; then - echo "ERROR: KIMI_VENDOR_VERIFIER_DIR must be at ${verifier_ref}" >&2 - echo "ERROR: required verifier sources must be present and unmodified" >&2 - return 2 - fi - KIMI_VENDOR_VERIFIER_CHECKOUT_DIR="$checkout_dir" - return 0 - fi - - checkout_dir="/tmp/kimi-vendor-verifier-${verifier_ref}" - if _kimi_vendor_checkout_is_valid "$checkout_dir" "$verifier_ref"; then - KIMI_VENDOR_VERIFIER_CHECKOUT_DIR="$checkout_dir" - return 0 - fi - command -v git >/dev/null 2>&1 || { echo "ERROR: git is required to fetch Kimi-Vendor-Verifier" >&2 return 1 } - rm -rf "$checkout_dir" - mkdir -p "$(dirname "$checkout_dir")" || return $? + checkout_dir="$(mktemp -d /tmp/kimi-vendor-verifier-XXXXXX)" || return $? if ! ( git init -q "$checkout_dir" \ && git -C "$checkout_dir" remote add origin "$repo_url" \ - && git -C "$checkout_dir" config remote.origin.promisor true \ - && git -C "$checkout_dir" config remote.origin.partialclonefilter blob:none \ && git -C "$checkout_dir" fetch -q --filter=blob:none --depth=1 \ origin "$verifier_ref" \ && git -C "$checkout_dir" update-ref HEAD FETCH_HEAD \ && git -C "$checkout_dir" sparse-checkout set --no-cone \ - /LICENSE \ /pyproject.toml \ /tests/conftest.py \ /tests/__init__.py \ @@ -930,11 +866,6 @@ _prepare_kimi_vendor_verifier() { echo "ERROR: failed to fetch Kimi-Vendor-Verifier at ${verifier_ref}" >&2 return 1 fi - if ! _kimi_vendor_checkout_is_valid "$checkout_dir" "$verifier_ref"; then - rm -rf "$checkout_dir" - echo "ERROR: fetched Kimi-Vendor-Verifier checkout is incomplete" >&2 - return 1 - fi KIMI_VENDOR_VERIFIER_CHECKOUT_DIR="$checkout_dir" } @@ -990,7 +921,6 @@ run_tool_use_eval() { return 2 ;; esac - export EVAL_FRAMEWORK=tool-use export EVAL_SUITE="$eval_suite" local _repo_root @@ -1001,46 +931,43 @@ run_tool_use_eval() { mkdir -p "$results_dir" || return $? export EVAL_RESULT_DIR="$results_dir" - local setup_rc integration_error - if _require_tool_use_python; then - : - else + local setup_rc=0 integration_error="" + _require_tool_use_python || { setup_rc=$? integration_error="tool-use Python version check failed with exit code ${setup_rc}" - echo "ERROR: ${integration_error}" >&2 - _write_tool_use_integration_error \ - "$adapter_path" "$model_name" "$results_dir" "$integration_error" - return "$setup_rc" - fi - if [ "${INFERENCEX_TOOL_USE_EVAL_RUNTIME_READY:-false}" != "true" ]; then + } + if [ "$setup_rc" -eq 0 ] \ + && [ "${INFERENCEX_TOOL_USE_EVAL_RUNTIME_READY:-false}" != "true" ]; then if _install_tool_use_eval_deps; then export INFERENCEX_TOOL_USE_EVAL_RUNTIME_READY=true else setup_rc=$? integration_error="tool-use dependency installation failed with exit code ${setup_rc}" - echo "ERROR: ${integration_error}" >&2 - _write_tool_use_integration_error \ - "$adapter_path" "$model_name" "$results_dir" "$integration_error" - return "$setup_rc" fi fi - if _prepare_kimi_vendor_verifier "$verifier_repo" "$verifier_ref"; then - : - else - setup_rc=$? - integration_error="tool-use verifier checkout failed with exit code ${setup_rc}" + if [ "$setup_rc" -eq 0 ]; then + _prepare_kimi_vendor_verifier "$verifier_repo" "$verifier_ref" || { + setup_rc=$? + integration_error="tool-use verifier checkout failed with exit code ${setup_rc}" + } + fi + if [ "$setup_rc" -ne 0 ]; then echo "ERROR: ${integration_error}" >&2 _write_tool_use_integration_error \ "$adapter_path" "$model_name" "$results_dir" "$integration_error" return "$setup_rc" fi + local eval_rc=0 python3 "$adapter_path" \ --verifier-dir "$KIMI_VENDOR_VERIFIER_CHECKOUT_DIR" \ --base-url "http://127.0.0.1:${port}/v1" \ --api-key EMPTY \ --model "$model_name" \ - --output-dir "$results_dir" + --output-dir "$results_dir" \ + || eval_rc=$? + rm -rf "$KIMI_VENDOR_VERIFIER_CHECKOUT_DIR" || true + return "$eval_rc" } _eval_patches_dir() { @@ -1161,14 +1088,7 @@ run_lm_eval() { tasks_dir="$_repo_root/$tasks_dir" fi - local effective_suite="${EVAL_SUITE:-}" - local task_basename - if [ -z "$effective_suite" ]; then - task_basename="${tasks_dir##*/}" - effective_suite="${task_basename%.yaml}" - effective_suite="${effective_suite%.yml}" - fi - export EVAL_SUITE="$effective_suite" + export EVAL_TASKS_DIR="$tasks_dir" if [ "${INFERENCEX_LM_EVAL_RUNTIME_READY:-false}" != "true" ]; then _install_lm_eval_deps @@ -1377,7 +1297,6 @@ append_lm_eval_summary() { fi fi fi - local eval_framework="${EVAL_FRAMEWORK:-lm-eval}" local eval_suite="${EVAL_SUITE:-}" if [ -z "$eval_suite" ] && [ -n "${EVAL_TASKS_DIR:-}" ]; then eval_suite="$(basename "${EVAL_TASKS_DIR}")" @@ -1391,7 +1310,6 @@ append_lm_eval_summary() { "framework": "${fw:-unknown}", "precision": "${prec:-unknown}", "spec_decoding": "${SPEC_DECODING:-}", - "eval_framework": "${eval_framework}", "eval_suite": "${eval_suite}", "tp": ${TP:-1}, "pp": ${PP_SIZE:-1}, @@ -1809,10 +1727,6 @@ run_eval() { fi local framework="${EVAL_FRAMEWORK:-${cli_framework:-$scenario_default}}" - if [ "$framework" = "lm_eval" ]; then - framework="lm-eval" - fi - export EVAL_FRAMEWORK="$framework" # Tool-use uses the verifier's fixed request budget and does not consume # EVAL_MAX_MODEL_LEN, so avoid loading model configuration for that path. diff --git a/utils/collect_eval_results.py b/utils/collect_eval_results.py index 7bc49c5d8c..fd7c0b1d05 100644 --- a/utils/collect_eval_results.py +++ b/utils/collect_eval_results.py @@ -284,9 +284,8 @@ def build_row(meta: Dict[str, Any], m: Dict[str, Any]) -> Dict[str, Any]: 'source': m.get('source'), } - for metadata_field in ('eval_framework', 'eval_suite'): - if metadata_field in meta: - row[metadata_field] = meta[metadata_field] + if 'eval_suite' in meta: + row['eval_suite'] = meta['eval_suite'] # Add universal score field (primary metric for unified comparison) if m.get('strict') is not None: diff --git a/utils/evals/EVALS.md b/utils/evals/EVALS.md index 07aa48b4de..64de3ced31 100644 --- a/utils/evals/EVALS.md +++ b/utils/evals/EVALS.md @@ -40,84 +40,43 @@ malformed metadata, duplicates, or raw/aggregate mismatches are not. See ## How? `run_eval` in `benchmarks/benchmark_lib.sh` dispatches to the selected eval -framework against the server's OpenAI-compatible endpoint. The default is -[lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) -(`lm-eval`) with GSM8K. Existing fixed-sequence and agentic paths preserve that -default, and explicit agentic runs can still select SWE-bench. +runner. Existing jobs continue to use lm-eval with GSM8K by default. -The Phase 1 tool-use suite is opt-in. The Kimi-K3 B300 vLLM agentic launcher, -like every existing launcher, continues to select lm-eval/GSM8K by default. To -run the suite after its server is ready, use the existing entrypoint: +The Phase 1 tool-use smoke is opt-in and single-node only. Select it with the +`eval-framework: tool-use` input on `e2e-tests.yml`, or invoke it after a server +is ready: ```bash -EVAL_FRAMEWORK=tool-use EVAL_SUITE=kimi_tool_call_schema \ - run_eval --port "$PORT" +EVAL_FRAMEWORK=tool-use run_eval --port "$PORT" ``` -`run_tool_use_eval` supplies `kimi_tool_call_schema` when `EVAL_SUITE` is unset -for a manual `run_eval --framework tool-use` call and rejects every other suite. -The compatibility result continues through the existing collector, suite-aware -artifact identity, and strict `1.0` threshold. -Phase 1 is single-node only and rejects `IS_MULTINODE=true` or `1`; the -multi-node workflow does not yet preserve the stock native report. - ### Stock Kimi tool-call schema smoke -This suite runs the unmodified +The smoke runs the unmodified [MoonshotAI/Kimi-Vendor-Verifier](https://github.com/MoonshotAI/Kimi-Vendor-Verifier) -at commit `b9ed3a6665bdff2c943246f7d2903cd003d6ddd6`. Its bundled Walle -schema corpus is sourced from MoonshotAI/walle commit -`cc1c6b7dab5496d5184677ecf4c3b95fc1bd1606` (`v0.1.10`). The upstream -prompt, schema loading and selection, request construction, non-stream and -stream assembly, argument validation, and report generation are all stock. -InferenceX owns only the subprocess invocation and compatibility projection. - -Python 3.12 or newer is required; the runner fails with a version error before -installing or checking out anything on older Python. At runtime it installs only -`httpx[http2]==0.28.1`, `openai==2.14.0`, `jsonschema==4.25.1`, and -`pytest==8.4.2`. It then makes a network checkout from GitHub using a sparse, -detached checkout of the pinned verifier commit containing only: - -- `LICENSE` and `pyproject.toml`; -- `tests/__init__.py`, `tests/conftest.py`, and - `tests/tool_call_json_schema/`; -- `testdata/walle_validator_cases/`. - -An explicitly supplied `KIMI_VENDOR_VERIFIER_DIR` is reused only when it is at -that exact commit, required sources are unmodified, and no extra checkout files -can override the verifier (root `.pytest_cache/` is ignored). The verifier -project and its unrelated benchmark dependencies are not installed. - -The thin `utils/evals/kimi_vendor_eval.py` wrapper runs upstream +at commit `b9ed3a6665bdff2c943246f7d2903cd003d6ddd6`. Each run creates a fresh +sparse checkout containing the upstream pytest configuration, tool-call schema +tests, and bundled Walle cases. InferenceX does not install the verifier package +or reimplement its request, streaming, retry, or validation logic. + +Python 3.12 or newer is required. The runner installs the minimal pinned runtime +(`httpx[http2]`, `openai`, `jsonschema`, and `pytest`), then runs upstream `tests/tool_call_json_schema/test_tool_call_json_schema.py` with: -- base URL `http://127.0.0.1:${PORT}/v1`, API key `EMPTY`, and model - `${MODEL_NAME:-$MODEL}`; -- `--case-dir testdata/walle_validator_cases/validator_cases`, - `--think-mode none --selection object --max-cases 1 --max-tokens 2048`; -- `--tool-json-report /kimi_vendor_report.json`. - -That stock selection chooses Walle case `TestAdditionalProperties:1` and -upstream parametrizes it in both `non-stream` and `stream` modes, for two -results. Requests use upstream's `openai.Client(timeout=120)` unchanged, so -OpenAI SDK 2.14.0's stock retry policy remains in effect, including its default -two retries for eligible connection, timeout, 408, 409, 429, and 5xx failures. -InferenceX does not add request retries or make the two modes concurrent. -`EVAL_CONCURRENT_REQUESTS` remains matrix metadata; multi-value batched -concurrency remains supported only by `lm-eval`. - -The unchanged native `kimi_vendor_report.json` is uploaded alongside the -collector-compatible `results_kimi_vendor_.json`. The -compatibility score is `passed / 2` for task `kimi_tool_call_schema`, primary metric -`exact_match,strict-match`, and effective sample count two. Success requires -pytest to exit zero and exactly two upstream mode results to pass. A setup or -collection failure still produces a zero-score compatibility result with -integration error metadata; the native report can be absent when upstream -cannot collect. - -Phase 1 intentionally covers one stock object-schema case only. It does not -measure broader schema coverage, tool selection among multiple tools, parallel -tool calls, multi-turn tool execution, or general agent quality. +- the local OpenAI-compatible endpoint, `EMPTY` API key, and served model name; +- `--think-mode none --selection object --max-cases 1 --max-tokens 2048`; +- the bundled Walle case directory and `--tool-json-report`. + +The selection is `TestAdditionalProperties:1`, parametrized upstream in +non-streaming and streaming modes. The unchanged native report is uploaded as +`kimi_vendor_report.json`. `utils/evals/kimi_vendor_eval.py` only projects its +two outcomes into the existing eval result shape. Both must pass, so the +`kimi_tool_call_schema` threshold is `1.0`. Setup and collection failures emit a +zero-score result with error metadata. + +This smoke validates one object-schema tool call. It does not cover tool choice, +parallel calls, multi-turn execution, or general agent quality. Multi-value +batched concurrency and multi-node execution are unsupported. ### Benchmark script flow @@ -152,7 +111,7 @@ Key eval functions in `benchmarks/benchmark_lib.sh`: | `append_lm_eval_summary` | Writes `meta_env.json` and moves eval artifacts to workspace | | `_install_lm_eval_deps` | Installs lm-eval dependencies | | `_install_tool_use_eval_deps` | Installs the minimal pinned stock-verifier runtime | -| `_prepare_kimi_vendor_verifier` | Prepares or validates the pinned sparse checkout | +| `_prepare_kimi_vendor_verifier` | Fetches a fresh pinned sparse checkout | | `_patch_lm_eval` | Patches lm-eval for reasoning tokens and TRT compatibility | | `compute_eval_context_length` | Computes eval context length (requested benchmark context, capped at model native max) | | `get_native_max_context_length` | Extracts model's native max context length from HF config | @@ -220,7 +179,6 @@ cat ./evals/agg_eval_all.json | jq '[.[] | select(.hw == "B200")]' | `em_flexible` | Flexible extraction (looser number matching) | | `n_eff` | Number of samples evaluated | | `task` | Eval task name (e.g., `gsm8k`) | -| `eval_framework` | Eval runner identity (for example, `lm-eval` or `tool-use`) | | `eval_suite` | Explicit suite identity used for collection and artifact reuse | ### Environment variables @@ -233,7 +191,6 @@ cat ./evals/agg_eval_all.json | jq '[.[] | select(.hw == "B200")]' | `EVAL_SUITE` | basename of `EVAL_TASKS_DIR`, else `gsm8k` | Eval suite metadata; explicit values take precedence | | `EVAL_TASKS_DIR` | `utils/evals/gsm8k.yaml` | Path to lm-eval task YAML | | `EVAL_RESULT_DIR` | `/tmp/eval_out-*` | Output directory for eval results | -| `KIMI_VENDOR_VERIFIER_DIR` | generated pinned checkout | Optional pre-existing verifier checkout; exact ref and required paths are validated | | `EVAL_MAX_MODEL_LEN` | `16384` | Max context for eval (set by `compute_eval_context_length`) | | `EVAL_CONCURRENT_REQUESTS` | `64` | Concurrent requests during eval; a space-separated list enables sequential batched evals against one live engine | | `EVAL_LIMIT` | empty | Limit eval to first N instances (smoke tests); empty = full set | diff --git a/utils/evals/kimi_vendor_eval.py b/utils/evals/kimi_vendor_eval.py index c01343fe15..a71845cc41 100755 --- a/utils/evals/kimi_vendor_eval.py +++ b/utils/evals/kimi_vendor_eval.py @@ -14,7 +14,7 @@ TASK_NAME = "kimi_tool_call_schema" NATIVE_REPORT_FILENAME = "kimi_vendor_report.json" -COMPATIBILITY_GLOB = "results_kimi_vendor_*.json" +COMPATIBILITY_GLOB = "results_*.json" EXPECTED_MODES = {"non-stream", "stream"} @@ -23,7 +23,7 @@ def prepare_compatibility_path(output_dir: Path) -> Path: for stale_path in output_dir.glob(COMPATIBILITY_GLOB): stale_path.unlink() timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S.%f") - return output_dir / f"results_kimi_vendor_{timestamp}.json" + return output_dir / f"results_{timestamp}.json" def build_pytest_command( @@ -96,16 +96,15 @@ def _project_report(model: str, report: Any) -> tuple[dict[str, Any], bool]: if total != len(results) or passed != result_passes: raise ValueError("report summary does not match result records") + if ( + total != 2 + or len(results) != 2 + or set(modes) != EXPECTED_MODES + or len(modes) != len(set(modes)) + ): + raise ValueError("report does not contain the expected stream modes") score = passed / 2.0 - compatibility = _compatibility_result(model, score) - complete_pass = ( - total == 2 - and passed == 2 - and len(results) == 2 - and set(modes) == EXPECTED_MODES - and len(modes) == len(set(modes)) - ) - return compatibility, complete_pass + return _compatibility_result(model, score), passed == 2 def _compatibility_result( @@ -114,7 +113,6 @@ def _compatibility_result( result: dict[str, Any] = { "lm_eval_version": "kimi-vendor-verifier", "model_name": model, - "model_args": f"pretrained={model}", "results": { TASK_NAME: { "exact_match,strict-match": score, @@ -123,25 +121,10 @@ def _compatibility_result( }, "configs": { TASK_NAME: { - "task": TASK_NAME, - "output_type": "generate_until", - "num_fewshot": 0, - "repeats": 1, - "metric_list": [ - { - "metric": "exact_match", - "aggregation": "mean", - "higher_is_better": True, - } - ], - "filter_list": [ - {"name": "strict-match", "filter": [{"function": "identity"}]} - ], + "metric_list": [{"metric": "exact_match"}], + "filter_list": [{"name": "strict-match"}], } }, - "versions": {TASK_NAME: 1}, - "n-shot": {TASK_NAME: 0}, - "higher_is_better": {TASK_NAME: {"exact_match": True}}, "n-samples": {TASK_NAME: {"original": 2, "effective": 2}}, } if integration_error is not None: diff --git a/utils/evals/test_kimi_vendor_eval.py b/utils/evals/test_kimi_vendor_eval.py index bfeb49819d..ff223d3a35 100644 --- a/utils/evals/test_kimi_vendor_eval.py +++ b/utils/evals/test_kimi_vendor_eval.py @@ -4,6 +4,7 @@ import sys from pathlib import Path from types import SimpleNamespace +from typing import Any import pytest @@ -12,21 +13,11 @@ import kimi_vendor_eval as kve -def _native_report(*, stream_status: str = "passed") -> dict: +def _report(stream_status: str = "passed") -> dict[str, Any]: statuses = ["passed", stream_status] - by_status: dict[str, int] = {} - for status in statuses: - by_status[status] = by_status.get(status, 0) + 1 + by_status = {status: statuses.count(status) for status in set(statuses)} return { - "summary": { - "total": 2, - "by_status": by_status, - "by_selection_reason": {"object_schema": 2}, - "by_mode": { - "non-stream": {"passed": 1}, - "stream": {stream_status: 1}, - }, - }, + "summary": {"total": 2, "by_status": by_status}, "results": [ {"mode": "non-stream", "status": "passed"}, {"mode": "stream", "status": stream_status}, @@ -34,37 +25,31 @@ def _native_report(*, stream_status: str = "passed") -> dict: } -def _compatibility_file(output_dir: Path) -> Path: - matches = list(output_dir.glob(kve.COMPATIBILITY_GLOB)) - assert len(matches) == 1 +def _result(output_dir: Path) -> dict[str, Any]: + paths = list(output_dir.glob(kve.COMPATIBILITY_GLOB)) + assert len(paths) == 1 assert re.fullmatch( - r"results_kimi_vendor_\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}\.\d{6}\.json", - matches[0].name, + r"results_\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}\.\d{6}\.json", + paths[0].name, ) - return matches[0] - - -def _projected(output_dir: Path) -> dict: - return json.loads(_compatibility_file(output_dir).read_text(encoding="utf-8")) + return json.loads(paths[0].read_text()) def _score(output_dir: Path) -> float: - return _projected(output_dir)["results"][kve.TASK_NAME][ + return _result(output_dir)["results"][kve.TASK_NAME][ "exact_match,strict-match" ] -def test_builds_exact_upstream_pytest_command(tmp_path: Path) -> None: +def test_builds_fixed_upstream_pytest_command(tmp_path: Path) -> None: report = tmp_path / kve.NATIVE_REPORT_FILENAME - command = kve.build_pytest_command( + assert kve.build_pytest_command( base_url="http://127.0.0.1:8000/v1", api_key="EMPTY", model="test-model", report_path=report, - ) - - assert command == [ + ) == [ sys.executable, "-m", "pytest", @@ -90,198 +75,128 @@ def test_builds_exact_upstream_pytest_command(tmp_path: Path) -> None: ] -def test_full_pass_projects_score_and_preserves_native_report( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch +@pytest.mark.parametrize( + ("stream_status", "return_code", "expected_pass", "expected_score"), + (("passed", 0, True, 1.0), ("failed", 1, False, 0.5)), +) +def test_projects_upstream_outcomes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + stream_status: str, + return_code: int, + expected_pass: bool, + expected_score: float, ) -> None: - verifier_dir = tmp_path / "verifier" - verifier_dir.mkdir() output_dir = tmp_path / "output" - output_dir.mkdir() - (output_dir / "results_kimi_vendor_2000-01-01T00-00-00.000000.json").write_text( - "stale", encoding="utf-8" - ) - native_bytes = (json.dumps(_native_report(), indent=2) + "\n").encode() - invocation: dict[str, object] = {} + native_bytes = json.dumps(_report(stream_status)).encode() + invocation: dict[str, Any] = {} def fake_run(command: list[str], *, cwd: Path, check: bool) -> SimpleNamespace: invocation.update(command=command, cwd=cwd, check=check) - output_dir.mkdir(parents=True, exist_ok=True) - (output_dir / kve.NATIVE_REPORT_FILENAME).write_bytes(native_bytes) - return SimpleNamespace(returncode=0) + Path(command[command.index("--tool-json-report") + 1]).write_bytes( + native_bytes + ) + return SimpleNamespace(returncode=return_code) monkeypatch.setattr(kve.subprocess, "run", fake_run) - passed = kve.run_evaluation( - verifier_dir=verifier_dir, - base_url="http://localhost:8000/v1", - api_key="EMPTY", - model="model-a", - output_dir=output_dir, + assert ( + kve.run_evaluation( + verifier_dir=tmp_path, + base_url="http://localhost/v1", + api_key="EMPTY", + model="model-a", + output_dir=output_dir, + ) + is expected_pass ) - - assert passed - assert invocation["cwd"] == verifier_dir + assert invocation["cwd"] == tmp_path assert invocation["check"] is False - assert invocation["command"] == kve.build_pytest_command( - base_url="http://localhost:8000/v1", - api_key="EMPTY", - model="model-a", - report_path=(output_dir / kve.NATIVE_REPORT_FILENAME).resolve(), - ) + assert _score(output_dir) == expected_score assert (output_dir / kve.NATIVE_REPORT_FILENAME).read_bytes() == native_bytes - projected = _projected(output_dir) - assert _score(output_dir) == 1.0 - assert projected["n-samples"][kve.TASK_NAME] == {"original": 2, "effective": 2} - assert "integration_error" not in projected - - -def test_one_mode_failure_projects_partial_score_and_fails( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - output_dir = tmp_path / "output" - - def fake_run(command: list[str], *, cwd: Path, check: bool) -> SimpleNamespace: - report_path = Path(command[command.index("--tool-json-report") + 1]) - report_path.write_text(json.dumps(_native_report(stream_status="failed"))) - return SimpleNamespace(returncode=1) - - monkeypatch.setattr(kve.subprocess, "run", fake_run) - - passed = kve.run_evaluation( - verifier_dir=tmp_path, - base_url="http://localhost/v1", - api_key="EMPTY", - model="model-a", - output_dir=output_dir, - ) - - assert not passed - assert _score(output_dir) == 0.5 -@pytest.mark.parametrize("native_contents", [None, "{not-json"]) -def test_missing_or_malformed_report_writes_zero_score_with_error( +@pytest.mark.parametrize( + ("failure", "error_type"), + ( + (None, "FileNotFoundError"), + ("{bad-json", "JSONDecodeError"), + (OSError("boom"), "OSError"), + ), +) +def test_collection_failures_write_zero_score( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, - native_contents: str | None, + failure: str | OSError | None, + error_type: str, ) -> None: - output_dir = tmp_path / "output" - def fake_run(command: list[str], *, cwd: Path, check: bool) -> SimpleNamespace: - if native_contents is not None: - report_path = Path(command[command.index("--tool-json-report") + 1]) - report_path.write_text(native_contents, encoding="utf-8") + if isinstance(failure, OSError): + raise failure + if failure is not None: + Path(command[command.index("--tool-json-report") + 1]).write_text( + failure + ) return SimpleNamespace(returncode=1) monkeypatch.setattr(kve.subprocess, "run", fake_run) + output_dir = tmp_path / "output" - passed = kve.run_evaluation( + assert not kve.run_evaluation( verifier_dir=tmp_path, base_url="http://localhost/v1", api_key="EMPTY", model="model-a", output_dir=output_dir, ) - - projected = _projected(output_dir) - assert not passed + projected = _result(output_dir) assert _score(output_dir) == 0.0 - assert projected["integration_error"]["type"] in { - "FileNotFoundError", - "JSONDecodeError", - } - assert projected["integration_error"]["message"] + assert projected["integration_error"]["type"] == error_type -def test_collection_failure_cannot_project_a_stale_passing_report( +def test_failure_cannot_reuse_stale_outputs( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: output_dir = tmp_path / "output" output_dir.mkdir() native_report = output_dir / kve.NATIVE_REPORT_FILENAME - native_report.write_text(json.dumps(_native_report()), encoding="utf-8") + native_report.write_text(json.dumps(_report())) + (output_dir / "results_2000-01-01T00-00-00.000000.json").write_text("{}") - def collection_failure( - command: list[str], *, cwd: Path, check: bool - ) -> SimpleNamespace: + def fail_collection(*args: Any, **kwargs: Any) -> SimpleNamespace: assert not native_report.exists() return SimpleNamespace(returncode=2) - monkeypatch.setattr(kve.subprocess, "run", collection_failure) - - passed = kve.run_evaluation( - verifier_dir=tmp_path, - base_url="http://localhost/v1", - api_key="EMPTY", - model="model-a", - output_dir=output_dir, - ) - - projected = _projected(output_dir) - assert not passed - assert _score(output_dir) == 0.0 - assert projected["integration_error"]["type"] == "FileNotFoundError" - - -def test_subprocess_launch_failure_writes_zero_score_with_error( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - def fail_to_launch(*args: object, **kwargs: object) -> subprocess.CompletedProcess: - raise OSError("pytest could not launch") - - monkeypatch.setattr(kve.subprocess, "run", fail_to_launch) - output_dir = tmp_path / "output" + monkeypatch.setattr(kve.subprocess, "run", fail_collection) - passed = kve.run_evaluation( + assert not kve.run_evaluation( verifier_dir=tmp_path, base_url="http://localhost/v1", api_key="EMPTY", model="model-a", output_dir=output_dir, ) - - projected = _projected(output_dir) - assert not passed assert _score(output_dir) == 0.0 - assert projected["integration_error"] == { - "type": "OSError", - "message": "pytest could not launch", - } + assert not native_report.exists() - -def test_cli_integration_error_writes_failure_without_running_pytest( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - def unexpected_run(*args: object, **kwargs: object) -> None: - pytest.fail("integration-error mode must not launch pytest") - - monkeypatch.setattr(kve.subprocess, "run", unexpected_run) +def test_cli_setup_failure_clears_stale_outputs(tmp_path: Path) -> None: output_dir = tmp_path / "output" output_dir.mkdir() - (output_dir / "results_kimi_vendor_2000-01-01T00-00-00.000000.json").write_text( - "stale", encoding="utf-8" - ) - (output_dir / kve.NATIVE_REPORT_FILENAME).write_text( - json.dumps(_native_report()), encoding="utf-8" - ) + (output_dir / kve.NATIVE_REPORT_FILENAME).write_text(json.dumps(_report())) + (output_dir / "results_2000-01-01T00-00-00.000000.json").write_text("{}") - return_code = kve.main( + assert kve.main( [ "--model", "model-a", "--output-dir", str(output_dir), "--integration-error", - "pinned verifier checkout failed", + "checkout failed", ] - ) - - projected = _projected(output_dir) - assert return_code == 1 + ) == 1 + projected = _result(output_dir) assert not (output_dir / kve.NATIVE_REPORT_FILENAME).exists() assert _score(output_dir) == 0.0 - assert projected["integration_error"] == { - "type": "RuntimeError", - "message": "pinned verifier checkout failed", - } \ No newline at end of file + assert projected["integration_error"]["message"] == "checkout failed" \ No newline at end of file diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index 067f2165e0..1f6478934a 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -7,7 +7,6 @@ import subprocess from pathlib import Path -import pytest BENCHMARK_LIB = Path(__file__).resolve().parents[2] / "benchmarks" / "benchmark_lib.sh" @@ -16,9 +15,9 @@ run_lm_eval() { echo "DISPATCH=lm-eval"; } run_swebench_eval() { echo "DISPATCH=swebench"; } run_tool_use_eval() { echo "DISPATCH=tool-use"; } -append_lm_eval_summary() { echo "STAGED=summary FRAMEWORK=$EVAL_FRAMEWORK"; } +append_lm_eval_summary() { echo "STAGED=summary"; } export EVAL_MAX_MODEL_LEN=16384 -export EVAL_CONCURRENT_REQUESTS="${REQUESTED_CONC:-}" +export EVAL_CONCURRENT_REQUESTS="" run_eval ${CLI_FW:+--framework "$CLI_FW"} --port 8888 ''' @@ -29,7 +28,6 @@ def _dispatch( eval_only: str = "false", cli_fw=None, env_fw=None, - requested_conc=None, ) -> str: env = { **os.environ, @@ -40,14 +38,11 @@ def _dispatch( } env.pop("EVAL_FRAMEWORK", None) env.pop("CLI_FW", None) - env.pop("REQUESTED_CONC", None) env.pop("KV_OFFLOAD_BACKEND", None) if cli_fw is not None: env["CLI_FW"] = cli_fw if env_fw is not None: env["EVAL_FRAMEWORK"] = env_fw - if requested_conc is not None: - env["REQUESTED_CONC"] = str(requested_conc) res = subprocess.run( ["bash", "-c", _SCRIPT], env=env, text=True, capture_output=True, check=True ) @@ -66,7 +61,6 @@ def test_agentic_eval_only_stages_summary(): output = _dispatch(is_agentic="1", eval_only="true") assert "DISPATCH=lm-eval" in output assert "STAGED=summary" in output - assert "FRAMEWORK=lm-eval" in output def test_fixed_seqlen_eval_only_leaves_staging_to_recipe(): @@ -86,24 +80,14 @@ def test_env_can_force_swebench_on_fixed_seqlen(): assert "DISPATCH=swebench" in _dispatch(is_agentic="0", env_fw="swebench") -def test_cli_swebench_framework_is_canonical_in_metadata() -> None: - output = _dispatch( - is_agentic="1", - eval_only="true", - cli_fw="swebench", - ) - assert "DISPATCH=swebench" in output - assert "FRAMEWORK=swebench" in output def test_env_can_force_tool_use_on_agentic_eval() -> None: - output = _dispatch( + assert "DISPATCH=tool-use" in _dispatch( is_agentic="1", eval_only="true", env_fw="tool-use", ) - assert "DISPATCH=tool-use" in output - assert "FRAMEWORK=tool-use" in output def test_tool_use_skips_unused_model_context_loading() -> None: @@ -131,26 +115,12 @@ def test_tool_use_skips_unused_model_context_loading() -> None: assert "UNEXPECTED_CONTEXT_LOAD" not in result.stdout -def test_tool_use_accepts_single_matrix_concurrency_identity() -> None: - assert "DISPATCH=tool-use" in _dispatch( - is_agentic="1", - env_fw="tool-use", - requested_conc=64, - ) def test_recipe_lm_eval_arg_still_lm_eval_on_fixed_seqlen(): assert "DISPATCH=lm-eval" in _dispatch(is_agentic="0", cli_fw="lm-eval") -def test_lm_eval_alias_is_canonicalized_in_metadata() -> None: - output = _dispatch( - is_agentic="1", - eval_only="true", - cli_fw="lm_eval", - ) - assert "DISPATCH=lm-eval" in output - assert "FRAMEWORK=lm-eval" in output def _run_invalid_call(call: str) -> subprocess.CompletedProcess: @@ -201,36 +171,14 @@ def test_tool_use_rejects_multinode() -> None: assert "supports single-node evals only" in result.stderr -@pytest.mark.parametrize( - ("failure_stage", "failure_rc", "message"), - ( - ("python", 11, "tool-use Python version check failed"), - ("dependencies", 12, "tool-use dependency installation failed"), - ("checkout", 13, "tool-use verifier checkout failed"), - ), -) def test_tool_use_setup_failure_writes_compatibility_result( tmp_path: Path, - failure_stage: str, - failure_rc: int, - message: str, ) -> None: results_dir = tmp_path / "results" script = r''' source "$BENCHMARK_LIB" -_require_tool_use_python() { - [ "$FAILURE_STAGE" = python ] && return "$FAILURE_RC" - return 0 -} -_install_tool_use_eval_deps() { - [ "$FAILURE_STAGE" = dependencies ] && return "$FAILURE_RC" - return 0 -} -_prepare_kimi_vendor_verifier() { - [ "$FAILURE_STAGE" = checkout ] && return "$FAILURE_RC" - KIMI_VENDOR_VERIFIER_CHECKOUT_DIR="$FAKE_VERIFIER_DIR" - return 0 -} +_require_tool_use_python() { :; } +_install_tool_use_eval_deps() { return 12; } run_tool_use_eval --results-dir "$RESULTS_DIR" printf 'SETUP_RC=%s\n' "$?" ''' @@ -238,20 +186,14 @@ def test_tool_use_setup_failure_writes_compatibility_result( **os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB), "RESULTS_DIR": str(results_dir), - "FAKE_VERIFIER_DIR": str(tmp_path / "verifier"), - "FAILURE_STAGE": failure_stage, - "FAILURE_RC": str(failure_rc), "MODEL": "test-model", "IS_MULTINODE": "false", "KV_OFFLOADING": "none", } for key in ( - "EVAL_FRAMEWORK", "EVAL_SUITE", "EVAL_RESULT_DIR", "INFERENCEX_TOOL_USE_EVAL_RUNTIME_READY", - "KIMI_VENDOR_VERIFIER_DIR", - "KIMI_VENDOR_VERIFIER_CHECKOUT_DIR", "MODEL_NAME", ): env.pop(key, None) @@ -263,11 +205,11 @@ def test_tool_use_setup_failure_writes_compatibility_result( capture_output=True, check=True, ) - expected_message = f"{message} with exit code {failure_rc}" - - assert f"SETUP_RC={failure_rc}" in result.stdout - assert expected_message in result.stderr + message = "tool-use dependency installation failed with exit code 12" score_files = list(results_dir.glob("results*.json")) + + assert "SETUP_RC=12" in result.stdout + assert message in result.stderr assert len(score_files) == 1 score_result = json.loads(score_files[0].read_text()) assert ( @@ -276,128 +218,25 @@ def test_tool_use_setup_failure_writes_compatibility_result( ] == 0.0 ) - assert score_result["integration_error"]["message"] == expected_message + assert score_result["integration_error"]["message"] == message assert not (results_dir / "kimi_vendor_report.json").exists() -def test_kimi_vendor_checkout_rejects_source_changes(tmp_path: Path) -> None: - checkout = tmp_path / "verifier" - required_files = ( - "LICENSE", - "pyproject.toml", - "tests/conftest.py", - "tests/__init__.py", - "tests/tool_call_json_schema/conftest.py", - "tests/tool_call_json_schema/validator.py", - "tests/tool_call_json_schema/test_tool_call_json_schema.py", - "testdata/walle_validator_cases/validator_cases/case.jsonl", - ) - for relative_path in required_files: - path = checkout / relative_path - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(f"{relative_path}\n") - - subprocess.run(["git", "init", "-q", str(checkout)], check=True) - subprocess.run(["git", "-C", str(checkout), "add", "."], check=True) - subprocess.run( - [ - "git", - "-C", - str(checkout), - "-c", - "user.name=InferenceX Tests", - "-c", - "user.email=tests@inferencex.invalid", - "commit", - "-qm", - "fixture", - ], - check=True, - ) - verifier_ref = subprocess.run( - ["git", "-C", str(checkout), "rev-parse", "HEAD"], - check=True, - capture_output=True, - text=True, - ).stdout.strip() - - def checkout_is_valid() -> bool: - result = subprocess.run( - [ - "bash", - "-c", - 'source "$BENCHMARK_LIB"; ' - '_kimi_vendor_checkout_is_valid "$CHECKOUT" "$VERIFIER_REF"', - ], - env={ - **os.environ, - "BENCHMARK_LIB": str(BENCHMARK_LIB), - "CHECKOUT": str(checkout), - "VERIFIER_REF": verifier_ref, - "KV_OFFLOADING": "none", - }, - ) - return result.returncode == 0 - - assert checkout_is_valid() - pytest_cache = checkout / ".pytest_cache" / "v" / "cache" / "nodeids" - pytest_cache.parent.mkdir(parents=True) - pytest_cache.write_text("[]\n") - assert checkout_is_valid() - - with (checkout / ".git/info/exclude").open("a") as exclude_file: - exclude_file.write("\n/conftest.py\n") - root_override = checkout / "conftest.py" - root_override.write_text("# ignored root override\n") - assert not checkout_is_valid() - root_override.unlink() - - root_override.write_text("# staged root override\n") - subprocess.run( - ["git", "-C", str(checkout), "add", "-f", "conftest.py"], - check=True, - ) - assert not checkout_is_valid() - subprocess.run( - ["git", "-C", str(checkout), "reset", "-q", "HEAD", "--", "conftest.py"], - check=True, - ) - root_override.unlink() - - override = checkout / "tests/tool_call_json_schema/local_override.py" - override.write_text("# untracked override\n") - assert not checkout_is_valid() - override.unlink() - - validator = checkout / "tests/tool_call_json_schema/validator.py" - validator.write_text("# modified verifier\n") - assert not checkout_is_valid() def test_tool_use_runner_uses_fixed_upstream_contract(tmp_path: Path) -> None: - shim_dir = tmp_path / "bin" - shim_dir.mkdir() - python_shim = shim_dir / "python3" - python_shim.write_text( - "#!/usr/bin/env bash\n" - "printf 'PYTHON_ARG=<%s>\\n' \"$@\"\n" - ) - python_shim.chmod( - python_shim.stat().st_mode | stat.S_IXUSR - ) results_dir = tmp_path / "results" verifier_dir = tmp_path / "verifier" script = r''' source "$BENCHMARK_LIB" -_require_tool_use_python() { echo "PYTHON_VERSION=OK"; } +_require_tool_use_python() { :; } _install_tool_use_eval_deps() { echo "INSTALL=UPSTREAM_MINIMAL"; } _prepare_kimi_vendor_verifier() { - printf 'CHECKOUT_REPO=%s\n' "$1" - printf 'CHECKOUT_REF=%s\n' "$2" + printf 'CHECKOUT=%s@%s\n' "$1" "$2" KIMI_VENDOR_VERIFIER_CHECKOUT_DIR="$VERIFIER_DIR" } +python3() { printf 'PYTHON_ARG=<%s>\n' "$@"; } run_tool_use_eval --port 9999 --results-dir "$RESULTS_DIR" -printf 'EVAL_FRAMEWORK=%s\n' "$EVAL_FRAMEWORK" printf 'EVAL_SUITE=%s\n' "$EVAL_SUITE" printf 'EVAL_RESULT_DIR=%s\n' "$EVAL_RESULT_DIR" printf 'RUNTIME_READY=%s\n' "$INFERENCEX_TOOL_USE_EVAL_RUNTIME_READY" @@ -409,20 +248,18 @@ def test_tool_use_runner_uses_fixed_upstream_contract(tmp_path: Path) -> None: "VERIFIER_DIR": str(verifier_dir), "MODEL": "test-model", "OPENAI_API_KEY": "must-not-be-forwarded", - "PATH": f"{shim_dir}:{os.environ['PATH']}", "KV_OFFLOADING": "none", "IS_MULTINODE": "false", } for key in ( - "EVAL_FRAMEWORK", "EVAL_SUITE", "EVAL_RESULT_DIR", "INFERENCEX_TOOL_USE_EVAL_RUNTIME_READY", - "KIMI_VENDOR_VERIFIER_DIR", "KIMI_VENDOR_VERIFIER_CHECKOUT_DIR", "MODEL_NAME", ): env.pop(key, None) + result = subprocess.run( ["bash", "-c", script], env=env, @@ -431,35 +268,23 @@ def test_tool_use_runner_uses_fixed_upstream_contract(tmp_path: Path) -> None: check=True, ) output = result.stdout - expected_adapter = ( - BENCHMARK_LIB.parents[1] / "utils/evals/kimi_vendor_eval.py" - ) + adapter = BENCHMARK_LIB.parents[1] / "utils/evals/kimi_vendor_eval.py" - assert "PYTHON_VERSION=OK" in output assert output.count("INSTALL=UPSTREAM_MINIMAL") == 1 assert ( - "CHECKOUT_REPO=https://github.com/MoonshotAI/Kimi-Vendor-Verifier.git" - in output - ) - assert ( - "CHECKOUT_REF=b9ed3a6665bdff2c943246f7d2903cd003d6ddd6" - in output - ) - assert f"PYTHON_ARG=<{expected_adapter}>" in output - for flag in ( - "--verifier-dir", - "--base-url", - "--api-key", - "--model", - "--output-dir", + "CHECKOUT=https://github.com/MoonshotAI/Kimi-Vendor-Verifier.git" + "@b9ed3a6665bdff2c943246f7d2903cd003d6ddd6" + ) in output + for value in ( + adapter, + verifier_dir, + "http://127.0.0.1:9999/v1", + "EMPTY", + "test-model", + results_dir, ): - assert f"PYTHON_ARG=<{flag}>" in output - assert f"PYTHON_ARG=<{verifier_dir}>" in output - assert "PYTHON_ARG=" in output - assert "PYTHON_ARG=" in output - assert "PYTHON_ARG=" in output - assert f"PYTHON_ARG=<{results_dir}>" in output - assert "EVAL_FRAMEWORK=tool-use" in output + assert f"PYTHON_ARG=<{value}>" in output + assert "must-not-be-forwarded" not in output assert "EVAL_SUITE=kimi_tool_call_schema" in output assert f"EVAL_RESULT_DIR={results_dir}" in output assert "RUNTIME_READY=true" in output @@ -566,7 +391,7 @@ def _summary_metadata(tmp_path: Path, **overrides: str) -> dict: "CONC": "7", "KV_OFFLOADING": "none", } - for key in ("EVAL_FRAMEWORK", "EVAL_SUITE", "EVAL_TASKS_DIR"): + for key in ("EVAL_SUITE", "EVAL_TASKS_DIR"): env.pop(key, None) env.update(overrides) subprocess.run(["bash", "-c", script], env=env, check=True) @@ -576,44 +401,37 @@ def _summary_metadata(tmp_path: Path, **overrides: str) -> dict: def test_summary_metadata_preserves_lm_eval_gsm8k_defaults(tmp_path: Path) -> None: meta = _summary_metadata(tmp_path) - assert meta["eval_framework"] == "lm-eval" assert meta["eval_suite"] == "gsm8k" assert meta["conc"] == 7 -def test_run_lm_eval_exports_cli_task_suite_to_metadata(tmp_path: Path) -> None: - work_dir = tmp_path / "work" - results_dir = tmp_path / "results" - work_dir.mkdir() - results_dir.mkdir() +def test_run_lm_eval_exports_cli_task_path(tmp_path: Path) -> None: script = r''' -set -e source "$BENCHMARK_LIB" -cd "$WORK_DIR" python3() { :; } export EVAL_MAX_MODEL_LEN=16384 export INFERENCEX_LM_EVAL_RUNTIME_READY=true run_lm_eval --task custom.yaml --results-dir "$RESULTS_DIR" -append_lm_eval_summary >/dev/null +printf 'EVAL_TASKS_DIR=%s\n' "$EVAL_TASKS_DIR" ''' env = { **os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB), - "WORK_DIR": str(work_dir), - "RESULTS_DIR": str(results_dir), - "MODEL": "test-model", + "RESULTS_DIR": str(tmp_path / "results"), "MODEL_NAME": "test-model", "OPENAI_API_KEY": "EMPTY", "KV_OFFLOADING": "none", } - for key in ("EVAL_FRAMEWORK", "EVAL_SUITE", "EVAL_TASKS_DIR"): - env.pop(key, None) - - subprocess.run(["bash", "-c", script], env=env, check=True) - meta = json.loads((work_dir / "meta_env.json").read_text()) + env.pop("EVAL_TASKS_DIR", None) + result = subprocess.run( + ["bash", "-c", script], + env=env, + text=True, + capture_output=True, + check=True, + ) - assert meta["eval_framework"] == "lm-eval" - assert meta["eval_suite"] == "custom" + assert "EVAL_TASKS_DIR=custom.yaml" in result.stdout def test_summary_metadata_prefers_explicit_suite_then_task_basename( @@ -625,13 +443,11 @@ def test_summary_metadata_prefers_explicit_suite_then_task_basename( ) explicit = _summary_metadata( tmp_path / "explicit", - EVAL_FRAMEWORK="tool-use", EVAL_SUITE="kimi_tool_call_schema", EVAL_TASKS_DIR="/tmp/ignored.yaml", ) assert from_task["eval_suite"] == "custom_reasoning" - assert explicit["eval_framework"] == "tool-use" assert explicit["eval_suite"] == "kimi_tool_call_schema" diff --git a/utils/test_collect_eval_results.py b/utils/test_collect_eval_results.py index 200be70e64..ba9ebef44c 100644 --- a/utils/test_collect_eval_results.py +++ b/utils/test_collect_eval_results.py @@ -21,20 +21,15 @@ def test_build_row_preserves_sequence_lengths() -> None: assert row["isl"] == 1024 assert row["osl"] == 1024 - assert "eval_framework" not in row assert "eval_suite" not in row -def test_build_row_preserves_explicit_eval_metadata() -> None: +def test_build_row_preserves_explicit_eval_suite() -> None: row = build_row( - { - "eval_framework": "tool-use", - "eval_suite": "kimi_tool_call_schema", - }, + {"eval_suite": "kimi_tool_call_schema"}, {"task": "kimi_tool_call_schema"}, ) - assert row["eval_framework"] == "tool-use" assert row["eval_suite"] == "kimi_tool_call_schema" @@ -82,7 +77,6 @@ def test_collect_eval_rows_expands_batched_concurrencies( "completed_eval_concs": [4, 16], "failed_eval_concs": [], "conc": 4, - "eval_framework": "lm-eval", "eval_suite": "gsm8k", })) _write_lm_eval_result( @@ -98,7 +92,6 @@ def test_collect_eval_rows_expands_batched_concurrencies( assert [row["conc"] for row in rows] == [4, 16] assert [row["score"] for row in rows] == [0.90, 0.91] - assert {row["eval_framework"] for row in rows} == {"lm-eval"} assert {row["eval_suite"] for row in rows} == {"gsm8k"} diff --git a/utils/test_validate_reusable_sweep_artifacts.py b/utils/test_validate_reusable_sweep_artifacts.py index 285bb6fcb3..cfa0a1df1a 100644 --- a/utils/test_validate_reusable_sweep_artifacts.py +++ b/utils/test_validate_reusable_sweep_artifacts.py @@ -673,23 +673,18 @@ def _dd_write_aggregate(root: Path, rows: list[dict]) -> Path: def _dd_write_legacy_raw( - root: Path, - name: str, - conc: int, - timestamp: str | None, - result_prefix: str = "results_", + root: Path, name: str, conc: int, timestamp: str | None ) -> None: artifact_dir = root / name artifact_dir.mkdir() (artifact_dir / "meta_env.json").write_text(json.dumps(_dd_meta(conc))) if timestamp is not None: - (artifact_dir / f"{result_prefix}{timestamp}.json").write_text("{}") + (artifact_dir / f"results_{timestamp}.json").write_text("{}") def test_dedupe_keeps_latest_legacy_rerun(tmp_path: Path) -> None: # Three reruns of one eval plus a result-less attempt, mirroring a flaky # config retried until it passed. - # The latest rerun uses the tool-use adapter's timestamped result prefix. old, mid, new, empty = ( "eval_minimaxm3_conc4096_b300-nv_15", "eval_minimaxm3_conc4096_b300-nv_16", @@ -698,23 +693,13 @@ def test_dedupe_keeps_latest_legacy_rerun(tmp_path: Path) -> None: ) _dd_write_legacy_raw(tmp_path, old, 4096, "2026-06-26T13-00-22.596040") _dd_write_legacy_raw(tmp_path, mid, 4096, "2026-06-26T19-00-52.356121") - _dd_write_legacy_raw( - tmp_path, - new, - 4096, - "2026-06-27T04-28-31.838775", - result_prefix="results_kimi_vendor_", - ) + _dd_write_legacy_raw(tmp_path, new, 4096, "2026-06-27T04-28-31.838775") _dd_write_legacy_raw(tmp_path, empty, 4096, None) _dd_write_aggregate( tmp_path, [ _dd_agg_row(4096, f"eval_results/{old}/results_2026-06-26T13-00-22.596040.json", 0.83), - _dd_agg_row( - 4096, - f"eval_results/{new}/results_kimi_vendor_2026-06-27T04-28-31.838775.json", - 0.95, - ), + _dd_agg_row(4096, f"eval_results/{new}/results_2026-06-27T04-28-31.838775.json", 0.95), _dd_agg_row(4096, f"eval_results/{mid}/results_2026-06-26T19-00-52.356121.json", 0.78), ], ) diff --git a/utils/validate_reusable_sweep_artifacts.py b/utils/validate_reusable_sweep_artifacts.py index 549547b203..8942d88fbf 100644 --- a/utils/validate_reusable_sweep_artifacts.py +++ b/utils/validate_reusable_sweep_artifacts.py @@ -316,12 +316,6 @@ def normalized_runner(value: Any) -> str: LEGACY_EVAL_SUITE = "" -def eval_suite_identity(row: dict[str, Any]) -> Any: - """Return an explicit suite or the compatibility identity for old artifacts.""" - if "eval_suite" in row: - return row["eval_suite"] - return LEGACY_EVAL_SUITE - def eval_key(row: dict[str, Any]) -> tuple[Any, ...]: """Build an eval identity from one aggregate row.""" @@ -332,7 +326,7 @@ def eval_key(row: dict[str, Any]) -> tuple[Any, ...]: row.get("model_prefix", row.get("infmax_model_prefix")), row.get("framework"), row.get("precision"), - eval_suite_identity(row), + row.get("eval_suite", LEGACY_EVAL_SUITE), row.get("spec_decoding", "none"), as_int(row.get("isl", 8192), 8192), as_int(row.get("osl", 1024), 1024), @@ -358,7 +352,7 @@ def eval_key(row: dict[str, Any]) -> tuple[Any, ...]: row.get("model_prefix", row.get("infmax_model_prefix")), row.get("framework"), row.get("precision"), - eval_suite_identity(row), + row.get("eval_suite", LEGACY_EVAL_SUITE), row.get("spec_decoding", "none"), as_int(row.get("isl", 8192), 8192), as_int(row.get("osl", 1024), 1024), From da29994e2faa6a720969d05cf9540679f08bb144 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:34:56 -0500 Subject: [PATCH 03/24] refactor: isolate and clarify verifier integration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：隔离 Kimi 验证器依赖并明确供应商适配边界，同时加入执行超时和通用结果格式标识。 --- .github/workflows/benchmark-tmpl.yml | 12 ++- .github/workflows/e2e-tests.yml | 15 +++- benchmarks/benchmark_lib.sh | 123 ++++++++++++++++--------- utils/collect_eval_results.py | 9 +- utils/evals/EVALS.md | 40 ++++++--- utils/evals/kimi_vendor_eval.py | 31 ++++++- utils/evals/test_kimi_vendor_eval.py | 81 ++++++++++------- utils/evals/test_run_eval_dispatch.py | 124 ++++++++++++++++++-------- utils/test_collect_eval_results.py | 23 ++++- 9 files changed, 324 insertions(+), 134 deletions(-) diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index e982384336..afafe2585f 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -86,10 +86,15 @@ on: required: false default: false eval-framework: - description: "Eval runner (lm-eval, swebench, or tool-use)" + description: "Eval runner (lm-eval, swebench, or kimi-vendor)" type: string required: false default: "lm-eval" + eval-suite: + description: "Suite interpreted by the selected eval runner" + type: string + required: false + default: "" random-range-ratio: required: false type: string @@ -179,6 +184,7 @@ env: RUN_EVAL: ${{ inputs.run-eval }} EVAL_ONLY: ${{ inputs.eval-only }} EVAL_FRAMEWORK: ${{ inputs.eval-framework }} + EVAL_SUITE: ${{ inputs.eval-suite }} # Agentic-coding env. Fixed-seq-len jobs leave these empty. SCENARIO_TYPE: ${{ inputs.scenario-type }} SCENARIO_SUBDIR: ${{ inputs.scenario-type == 'agentic-coding' && 'agentic/' || 'fixed_seq_len/' }} @@ -398,7 +404,7 @@ jobs: path: | meta_env.json results*.json - kimi_vendor_report.json + *_vendor_report.json sample*.jsonl agent_preds.json predictions.jsonl @@ -416,7 +422,7 @@ jobs: rm -f meta_env.json || true # Remove any eval results JSONs that were moved into workspace rm -f results*.json || true - rm -f kimi_vendor_report.json || true + rm -f -- ./*_vendor_report.json || true rm -f sample*.jsonl || true rm -f agent_preds.json predictions.jsonl swebench_report_*.json *.traj* || true diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 83b48333f9..6b9c67410b 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -46,10 +46,15 @@ on: type: string default: "" eval-framework: - description: "Agentic eval runner (lm-eval, swebench, or tool-use)" + description: "Agentic eval runner (lm-eval, swebench, or kimi-vendor)" required: false type: string default: "lm-eval" + eval-suite: + description: "Agentic eval suite interpreted by the selected runner" + required: false + type: string + default: "" swebench-gen-mode: description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)." required: false @@ -131,10 +136,15 @@ on: type: string default: "" eval-framework: - description: "Agentic eval runner (lm-eval, swebench, or tool-use)" + description: "Agentic eval runner (lm-eval, swebench, or kimi-vendor)" required: false type: string default: "lm-eval" + eval-suite: + description: "Agentic eval suite interpreted by the selected runner" + required: false + type: string + default: "" swebench-gen-mode: description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)." required: false @@ -470,6 +480,7 @@ jobs: eval-limit: ${{ inputs.eval-limit }} swebench-gen-mode: ${{ inputs.swebench-gen-mode }} eval-framework: ${{ inputs.eval-framework }} + eval-suite: ${{ inputs.eval-suite }} scenario-type: agentic-coding ref: ${{ inputs.ref }} diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index b621c9d923..1c1c3d8f5f 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -819,25 +819,37 @@ _install_lm_eval_deps() { fi } -_require_tool_use_python() { +_require_kimi_vendor_python() { if python3 -c 'import sys; raise SystemExit(sys.version_info < (3, 12))'; then return 0 fi local python_version python_version="$(python3 -c 'import platform; print(platform.python_version())' 2>/dev/null || printf 'unavailable')" - echo "ERROR: tool-use requires Python >=3.12 (python3 is ${python_version})" >&2 + echo "ERROR: Kimi Vendor Verifier requires Python >=3.12 (python3 is ${python_version})" >&2 return 2 } -_install_tool_use_eval_deps() { - python3 -m pip install -q --no-cache-dir --break-system-packages \ +_install_kimi_vendor_eval_deps() { + local target_dir="$1" + python3 -m pip install -q --no-cache-dir --target "$target_dir" \ "httpx[http2]==0.28.1" \ "openai==2.14.0" \ "jsonschema==4.25.1" \ "pytest==8.4.2" } +_prepare_kimi_vendor_runtime() { + local runtime_dir install_rc=0 + runtime_dir="$(mktemp -d /tmp/kimi-vendor-runtime-XXXXXX)" || return $? + _install_kimi_vendor_eval_deps "$runtime_dir" >&2 || install_rc=$? + if [ "$install_rc" -ne 0 ]; then + rm -rf "$runtime_dir" + return "$install_rc" + fi + printf '%s\n' "$runtime_dir" +} + _prepare_kimi_vendor_verifier() { local repo_url="$1" local verifier_ref="$2" @@ -866,10 +878,17 @@ _prepare_kimi_vendor_verifier() { echo "ERROR: failed to fetch Kimi-Vendor-Verifier at ${verifier_ref}" >&2 return 1 fi - KIMI_VENDOR_VERIFIER_CHECKOUT_DIR="$checkout_dir" + printf '%s\n' "$checkout_dir" } -_write_tool_use_integration_error() { +_cleanup_kimi_vendor_eval() { + local path + for path in "$@"; do + [ -z "$path" ] || rm -rf "$path" || true + done +} + +_write_kimi_vendor_integration_error() { local adapter_path="$1" local model_name="$2" local results_dir="$3" @@ -878,11 +897,10 @@ _write_tool_use_integration_error() { python3 "$adapter_path" \ --model "$model_name" \ --output-dir "$results_dir" \ - --integration-error "$message" \ - || true + --integration-error "$message" } -run_tool_use_eval() { +_run_kimi_tool_call_schema_eval() { local port="${PORT:-8888}" local results_dir="${EVAL_RESULT_DIR:-$(mktemp -d /tmp/eval_out-XXXXXX)}" local verifier_repo="https://github.com/MoonshotAI/Kimi-Vendor-Verifier.git" @@ -908,68 +926,85 @@ run_tool_use_eval() { esac done - local eval_suite="${EVAL_SUITE:-kimi_tool_call_schema}" - if [ "$eval_suite" != "kimi_tool_call_schema" ]; then - echo "ERROR: tool-use supports only EVAL_SUITE=kimi_tool_call_schema" >&2 - export EVAL_RESULT_DIR="" - return 2 - fi case "${IS_MULTINODE:-false}" in true|1) - echo "ERROR: tool-use Phase 1 supports single-node evals only" >&2 + echo "ERROR: Kimi tool-call schema eval supports single-node only" >&2 export EVAL_RESULT_DIR="" return 2 ;; esac - export EVAL_SUITE="$eval_suite" - local _repo_root - _repo_root="$(cd "$INFERENCEX_BENCHMARK_LIB_DIR/.." && pwd)" + local repo_root + repo_root="$(cd "$INFERENCEX_BENCHMARK_LIB_DIR/.." && pwd)" local model_name="${MODEL_NAME:-${MODEL:-}}" - local adapter_path="${_repo_root}/utils/evals/kimi_vendor_eval.py" + local adapter_path="${repo_root}/utils/evals/kimi_vendor_eval.py" + local runtime_dir="" + local checkout_dir="" mkdir -p "$results_dir" || return $? export EVAL_RESULT_DIR="$results_dir" local setup_rc=0 integration_error="" - _require_tool_use_python || { + _require_kimi_vendor_python || { setup_rc=$? - integration_error="tool-use Python version check failed with exit code ${setup_rc}" + integration_error="Kimi Vendor Verifier Python version check failed with exit code ${setup_rc}" } - if [ "$setup_rc" -eq 0 ] \ - && [ "${INFERENCEX_TOOL_USE_EVAL_RUNTIME_READY:-false}" != "true" ]; then - if _install_tool_use_eval_deps; then - export INFERENCEX_TOOL_USE_EVAL_RUNTIME_READY=true - else + if [ "$setup_rc" -eq 0 ]; then + runtime_dir=$(_prepare_kimi_vendor_runtime) || { setup_rc=$? - integration_error="tool-use dependency installation failed with exit code ${setup_rc}" - fi + integration_error="Kimi Vendor Verifier dependency installation failed with exit code ${setup_rc}" + } fi if [ "$setup_rc" -eq 0 ]; then - _prepare_kimi_vendor_verifier "$verifier_repo" "$verifier_ref" || { + checkout_dir=$( + _prepare_kimi_vendor_verifier "$verifier_repo" "$verifier_ref" + ) || { setup_rc=$? - integration_error="tool-use verifier checkout failed with exit code ${setup_rc}" + integration_error="Kimi Vendor Verifier checkout failed with exit code ${setup_rc}" } fi if [ "$setup_rc" -ne 0 ]; then + _cleanup_kimi_vendor_eval "$runtime_dir" "$checkout_dir" echo "ERROR: ${integration_error}" >&2 - _write_tool_use_integration_error \ - "$adapter_path" "$model_name" "$results_dir" "$integration_error" + local artifact_rc=0 + _write_kimi_vendor_integration_error \ + "$adapter_path" "$model_name" "$results_dir" "$integration_error" \ + || artifact_rc=$? + if [ "$artifact_rc" -ne 0 ]; then + echo "ERROR: failed to write Kimi verifier failure artifact (exit code ${artifact_rc})" >&2 + fi return "$setup_rc" fi local eval_rc=0 - python3 "$adapter_path" \ - --verifier-dir "$KIMI_VENDOR_VERIFIER_CHECKOUT_DIR" \ - --base-url "http://127.0.0.1:${port}/v1" \ - --api-key EMPTY \ - --model "$model_name" \ - --output-dir "$results_dir" \ - || eval_rc=$? - rm -rf "$KIMI_VENDOR_VERIFIER_CHECKOUT_DIR" || true + PYTHONPATH="${runtime_dir}${PYTHONPATH:+:${PYTHONPATH}}" \ + python3 "$adapter_path" \ + --verifier-dir "$checkout_dir" \ + --base-url "http://127.0.0.1:${port}/v1" \ + --api-key EMPTY \ + --model "$model_name" \ + --output-dir "$results_dir" \ + || eval_rc=$? + _cleanup_kimi_vendor_eval "$runtime_dir" "$checkout_dir" return "$eval_rc" } +run_kimi_vendor_eval() { + local eval_suite="${EVAL_SUITE:-kimi_tool_call_schema}" + export EVAL_SUITE="$eval_suite" + + case "$eval_suite" in + kimi_tool_call_schema) + _run_kimi_tool_call_schema_eval "$@" + ;; + *) + echo "ERROR: unsupported Kimi Vendor Verifier suite '${eval_suite}'" >&2 + export EVAL_RESULT_DIR="" + return 2 + ;; + esac +} + _eval_patches_dir() { cd "$(dirname "${BASH_SOURCE[0]}")/../utils/evals/patches" && pwd } @@ -1728,9 +1763,9 @@ run_eval() { local framework="${EVAL_FRAMEWORK:-${cli_framework:-$scenario_default}}" - # Tool-use uses the verifier's fixed request budget and does not consume + # Kimi Vendor Verifier uses a fixed request budget and does not consume # EVAL_MAX_MODEL_LEN, so avoid loading model configuration for that path. - if [ "$framework" != "tool-use" ] && [ -z "${EVAL_MAX_MODEL_LEN:-}" ]; then + if [ "$framework" != "kimi-vendor" ] && [ -z "${EVAL_MAX_MODEL_LEN:-}" ]; then compute_eval_context_length "$MODEL" "${MAX_MODEL_LEN:-0}" > /dev/null fi @@ -1801,7 +1836,7 @@ run_eval() { case "$framework" in lm-eval|lm_eval) run_lm_eval "${forwarded[@]}" || eval_rc=$? ;; swebench) run_swebench_eval "${forwarded[@]}" || eval_rc=$? ;; - tool-use) run_tool_use_eval "${forwarded[@]}" || eval_rc=$? ;; + kimi-vendor) run_kimi_vendor_eval "${forwarded[@]}" || eval_rc=$? ;; *) echo "Unknown framework '${framework}'"; eval_rc=1 ;; esac diff --git a/utils/collect_eval_results.py b/utils/collect_eval_results.py index fd7c0b1d05..1070a9305e 100644 --- a/utils/collect_eval_results.py +++ b/utils/collect_eval_results.py @@ -32,6 +32,7 @@ SPEC_DECODING = "Spec Decode" CONC_SUFFIX_RE = re.compile(r"_conc(\d+)(?:_\d+)?\.json$") +EVAL_RESULT_FORMAT = "inferencex-eval-v1" def load_json(path: Path) -> Optional[Dict[str, Any]]: @@ -71,10 +72,10 @@ def result_concurrency(path: Path) -> Optional[int]: def detect_lm_eval_jsons(d: Path, batched: bool = False) -> List[Path]: - """Return lm-eval result JSONs from one artifact directory. + """Return collector-compatible eval result JSONs from one artifact directory. - Legacy artifacts contribute their latest result file. Batched artifacts - contribute the latest result file for each `_concN` suffix. + Legacy lm-eval artifacts contribute their latest result file. Batched + artifacts contribute the latest result file for each `_concN` suffix. """ immediate_jsons = set(d.glob('results*.json')) immediate_jsons.update( @@ -86,7 +87,7 @@ def detect_lm_eval_jsons(d: Path, batched: bool = False) -> List[Path]: data = load_json(p) if not isinstance(data, dict): continue - if 'lm_eval_version' in data: + if data.get('result_format') == EVAL_RESULT_FORMAT or 'lm_eval_version' in data: lm_paths.append(p) if not lm_paths: diff --git a/utils/evals/EVALS.md b/utils/evals/EVALS.md index b85059e415..dcad93c066 100644 --- a/utils/evals/EVALS.md +++ b/utils/evals/EVALS.md @@ -44,14 +44,21 @@ runner. Existing jobs continue to use lm-eval with GSM8K by default. The default eval framework is [lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) (`lm-eval`). Agentic eval-only matrix jobs inherit this default and therefore run the same GSM8K task as 8k1k. Explicit agentic runs can still select SWE-bench. -The Phase 1 tool-use smoke is opt-in and single-node only. Select it with the -`eval-framework: tool-use` input on `e2e-tests.yml`, or invoke it after a server -is ready: +The Phase 1 Kimi smoke is opt-in and single-node only. Select +`eval-framework: kimi-vendor` and `eval-suite: kimi_tool_call_schema` on +`e2e-tests.yml`, or invoke it after a server is ready: ```bash -EVAL_FRAMEWORK=tool-use run_eval --port "$PORT" +EVAL_FRAMEWORK=kimi-vendor EVAL_SUITE=kimi_tool_call_schema \ + run_eval --port "$PORT" ``` +The framework selects a provider-specific subprocess adapter, while the suite +selects a case set understood by that adapter. Each adapter owns its endpoint +format, dependencies, native report, metrics, and pass policy. Future MiniMax +or BFCL support should add explicit `run_eval` cases rather than a shared +request or report abstraction. + ### Stock Kimi tool-call schema smoke The smoke runs the unmodified @@ -62,7 +69,8 @@ tests, and bundled Walle cases. InferenceX does not install the verifier package or reimplement its request, streaming, retry, or validation logic. Python 3.12 or newer is required. The runner installs the minimal pinned runtime -(`httpx[http2]`, `openai`, `jsonschema`, and `pytest`), then runs upstream +(`httpx[http2]`, `openai`, `jsonschema`, and `pytest`) into a temporary isolated +package directory, then runs upstream `tests/tool_call_json_schema/test_tool_call_json_schema.py` with: - the local OpenAI-compatible endpoint, `EMPTY` API key, and served model name; @@ -73,8 +81,9 @@ The selection is `TestAdditionalProperties:1`, parametrized upstream in non-streaming and streaming modes. The unchanged native report is uploaded as `kimi_vendor_report.json`. `utils/evals/kimi_vendor_eval.py` only projects its two outcomes into the existing eval result shape. Both must pass, so the -`kimi_tool_call_schema` threshold is `1.0`. Setup and collection failures emit a -zero-score result with error metadata. +`kimi_tool_call_schema` threshold is `1.0`. Setup, timeout, and collection +failures emit a zero-score result with error metadata. The adapter bounds the +upstream pytest process to 900 seconds. This smoke validates one object-schema tool call. It does not cover tool choice, parallel calls, multi-turn execution, or general agent quality. Multi-value @@ -109,10 +118,10 @@ Key eval functions in `benchmarks/benchmark_lib.sh`: |----------|-------------| | `run_eval` | Unified entrypoint - dispatches to framework-specific runner | | `run_lm_eval` | Runs lm-eval harness against the OpenAI-compatible endpoint | -| `run_tool_use_eval` | Runs the pinned stock verifier in non-stream and stream modes | +| `run_kimi_vendor_eval` | Selects and runs a pinned Kimi Vendor Verifier suite | | `append_lm_eval_summary` | Writes `meta_env.json` and moves eval artifacts to workspace | | `_install_lm_eval_deps` | Installs lm-eval dependencies | -| `_install_tool_use_eval_deps` | Installs the minimal pinned stock-verifier runtime | +| `_prepare_kimi_vendor_runtime` | Installs the minimal pinned runtime in an isolated temp path | | `_prepare_kimi_vendor_verifier` | Fetches a fresh pinned sparse checkout | | `_patch_lm_eval` | Patches lm-eval for reasoning tokens and TRT compatibility | | `compute_eval_context_length` | Computes eval context length (requested benchmark context, capped at model native max) | @@ -189,8 +198,8 @@ cat ./evals/agg_eval_all.json | jq '[.[] | select(.hw == "B200")]' |----------|---------|-------------| | `RUN_EVAL` | `false` | Enable eval after throughput benchmark | | `EVAL_ONLY` | `false` | Skip throughput, only run evals (set by workflow) | -| `EVAL_FRAMEWORK` | `lm-eval` | Eval framework to use | -| `EVAL_SUITE` | basename of `EVAL_TASKS_DIR`, else `gsm8k` | Eval suite metadata; explicit values take precedence | +| `EVAL_FRAMEWORK` | `lm-eval` | Eval runner (`lm-eval`, `swebench`, or `kimi-vendor`) | +| `EVAL_SUITE` | basename of `EVAL_TASKS_DIR`, else `gsm8k` | Runner-specific suite selector and artifact identity; the workflow `eval-suite` input sets it explicitly | | `EVAL_TASKS_DIR` | `utils/evals/gsm8k.yaml` | Path to lm-eval task YAML | | `EVAL_RESULT_DIR` | `/tmp/eval_out-*` | Output directory for eval results | | `EVAL_MAX_MODEL_LEN` | `16384` | Max context for eval (set by `compute_eval_context_length`) | @@ -206,6 +215,15 @@ cat ./evals/agg_eval_all.json | jq '[.[] | select(.hw == "B200")]' 2. Set `EVAL_TASKS_DIR=utils/evals/.yaml` when running benchmarks. 3. Update `utils/collect_eval_results.py` if new metrics need extraction. +### Adding a provider verifier + +1. Add a provider-specific adapter under `utils/evals/`. +2. Add an explicit framework case in `run_eval`; keep suite-specific policy in + that adapter's shell runner. +3. Install dependencies in a provider-specific isolated runtime. +4. Emit `result_format: inferencex-eval-v1`, preserve the native report as + `*_vendor_report.json`, set `EVAL_SUITE`, and add a threshold. + ### Runtime patches (`utils/evals/patches/`) The benchmark helpers invoke these standalone scripts against pinned dependencies. diff --git a/utils/evals/kimi_vendor_eval.py b/utils/evals/kimi_vendor_eval.py index a71845cc41..01d024771c 100755 --- a/utils/evals/kimi_vendor_eval.py +++ b/utils/evals/kimi_vendor_eval.py @@ -14,8 +14,11 @@ TASK_NAME = "kimi_tool_call_schema" NATIVE_REPORT_FILENAME = "kimi_vendor_report.json" -COMPATIBILITY_GLOB = "results_*.json" +COMPATIBILITY_GLOB = "results_kimi_vendor_*.json" EXPECTED_MODES = {"non-stream", "stream"} +DEFAULT_TIMEOUT_SECONDS = 900 +RESULT_FORMAT = "inferencex-eval-v1" +ADAPTER_NAME = "kimi-vendor-verifier" def prepare_compatibility_path(output_dir: Path) -> Path: @@ -23,7 +26,7 @@ def prepare_compatibility_path(output_dir: Path) -> Path: for stale_path in output_dir.glob(COMPATIBILITY_GLOB): stale_path.unlink() timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S.%f") - return output_dir / f"results_{timestamp}.json" + return output_dir / f"results_kimi_vendor_{timestamp}.json" def build_pytest_command( @@ -111,7 +114,8 @@ def _compatibility_result( model: str, score: float, integration_error: BaseException | None = None ) -> dict[str, Any]: result: dict[str, Any] = { - "lm_eval_version": "kimi-vendor-verifier", + "result_format": RESULT_FORMAT, + "eval_adapter": ADAPTER_NAME, "model_name": model, "results": { TASK_NAME: { @@ -146,6 +150,7 @@ def run_evaluation( api_key: str, model: str, output_dir: Path, + timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS, ) -> bool: """Run upstream pytest and always attempt to publish a compatibility result.""" output_dir.mkdir(parents=True, exist_ok=True) @@ -167,11 +172,18 @@ def run_evaluation( ), cwd=verifier_dir, check=False, + timeout=timeout_seconds, ) subprocess_rc = completed.returncode report = json.loads(native_report.read_text(encoding="utf-8")) compatibility, complete_pass = _project_report(model, report) - except (OSError, ValueError, json.JSONDecodeError) as exc: + if subprocess_rc != 0 and complete_pass: + integration_error = RuntimeError( + f"upstream verifier exited with code {subprocess_rc}" + ) + compatibility = _compatibility_result(model, 0.0, integration_error) + complete_pass = False + except (OSError, ValueError, subprocess.TimeoutExpired) as exc: integration_error = exc compatibility = _compatibility_result(model, 0.0, exc) finally: @@ -185,6 +197,13 @@ def run_evaluation( return subprocess_rc == 0 and complete_pass and integration_error is None +def _positive_int(value: str) -> int: + parsed = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("must be a positive integer") + return parsed + + def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser( description="Run the pinned stock Kimi Vendor Verifier tool-schema smoke test." @@ -194,6 +213,9 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser.add_argument("--api-key", default="EMPTY") parser.add_argument("--model", required=True) parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument( + "--timeout-seconds", type=_positive_int, default=DEFAULT_TIMEOUT_SECONDS + ) parser.add_argument("--integration-error") args = parser.parse_args(argv) if args.integration_error is None: @@ -230,6 +252,7 @@ def main(argv: Sequence[str] | None = None) -> int: api_key=args.api_key, model=args.model, output_dir=args.output_dir, + timeout_seconds=args.timeout_seconds, ) return 0 if passed else 1 diff --git a/utils/evals/test_kimi_vendor_eval.py b/utils/evals/test_kimi_vendor_eval.py index ff223d3a35..db08ce17d9 100644 --- a/utils/evals/test_kimi_vendor_eval.py +++ b/utils/evals/test_kimi_vendor_eval.py @@ -29,16 +29,14 @@ def _result(output_dir: Path) -> dict[str, Any]: paths = list(output_dir.glob(kve.COMPATIBILITY_GLOB)) assert len(paths) == 1 assert re.fullmatch( - r"results_\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}\.\d{6}\.json", + r"results_kimi_vendor_\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}\.\d{6}\.json", paths[0].name, ) return json.loads(paths[0].read_text()) def _score(output_dir: Path) -> float: - return _result(output_dir)["results"][kve.TASK_NAME][ - "exact_match,strict-match" - ] + return _result(output_dir)["results"][kve.TASK_NAME]["exact_match,strict-match"] def test_builds_fixed_upstream_pytest_command(tmp_path: Path) -> None: @@ -77,7 +75,11 @@ def test_builds_fixed_upstream_pytest_command(tmp_path: Path) -> None: @pytest.mark.parametrize( ("stream_status", "return_code", "expected_pass", "expected_score"), - (("passed", 0, True, 1.0), ("failed", 1, False, 0.5)), + ( + ("passed", 0, True, 1.0), + ("passed", 1, False, 0.0), + ("failed", 1, False, 0.5), + ), ) def test_projects_upstream_outcomes( tmp_path: Path, @@ -91,11 +93,11 @@ def test_projects_upstream_outcomes( native_bytes = json.dumps(_report(stream_status)).encode() invocation: dict[str, Any] = {} - def fake_run(command: list[str], *, cwd: Path, check: bool) -> SimpleNamespace: - invocation.update(command=command, cwd=cwd, check=check) - Path(command[command.index("--tool-json-report") + 1]).write_bytes( - native_bytes - ) + def fake_run( + command: list[str], *, cwd: Path, check: bool, timeout: int + ) -> SimpleNamespace: + invocation.update(command=command, cwd=cwd, check=check, timeout=timeout) + Path(command[command.index("--tool-json-report") + 1]).write_bytes(native_bytes) return SimpleNamespace(returncode=return_code) monkeypatch.setattr(kve.subprocess, "run", fake_run) @@ -112,7 +114,12 @@ def fake_run(command: list[str], *, cwd: Path, check: bool) -> SimpleNamespace: ) assert invocation["cwd"] == tmp_path assert invocation["check"] is False + assert invocation["timeout"] == kve.DEFAULT_TIMEOUT_SECONDS assert _score(output_dir) == expected_score + projected = _result(output_dir) + assert projected["result_format"] == kve.RESULT_FORMAT + assert projected["eval_adapter"] == kve.ADAPTER_NAME + assert "lm_eval_version" not in projected assert (output_dir / kve.NATIVE_REPORT_FILENAME).read_bytes() == native_bytes @@ -122,21 +129,25 @@ def fake_run(command: list[str], *, cwd: Path, check: bool) -> SimpleNamespace: (None, "FileNotFoundError"), ("{bad-json", "JSONDecodeError"), (OSError("boom"), "OSError"), + ( + subprocess.TimeoutExpired("pytest", kve.DEFAULT_TIMEOUT_SECONDS), + "TimeoutExpired", + ), ), ) def test_collection_failures_write_zero_score( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, - failure: str | OSError | None, + failure: str | BaseException | None, error_type: str, ) -> None: - def fake_run(command: list[str], *, cwd: Path, check: bool) -> SimpleNamespace: - if isinstance(failure, OSError): + def fake_run( + command: list[str], *, cwd: Path, check: bool, timeout: int + ) -> SimpleNamespace: + if isinstance(failure, BaseException): raise failure if failure is not None: - Path(command[command.index("--tool-json-report") + 1]).write_text( - failure - ) + Path(command[command.index("--tool-json-report") + 1]).write_text(failure) return SimpleNamespace(returncode=1) monkeypatch.setattr(kve.subprocess, "run", fake_run) @@ -161,7 +172,11 @@ def test_failure_cannot_reuse_stale_outputs( output_dir.mkdir() native_report = output_dir / kve.NATIVE_REPORT_FILENAME native_report.write_text(json.dumps(_report())) - (output_dir / "results_2000-01-01T00-00-00.000000.json").write_text("{}") + (output_dir / "results_kimi_vendor_2000-01-01T00-00-00.000000.json").write_text( + "{}" + ) + foreign_result = output_dir / "results_other_eval.json" + foreign_result.write_text("{}") def fail_collection(*args: Any, **kwargs: Any) -> SimpleNamespace: assert not native_report.exists() @@ -178,25 +193,31 @@ def fail_collection(*args: Any, **kwargs: Any) -> SimpleNamespace: ) assert _score(output_dir) == 0.0 assert not native_report.exists() + assert foreign_result.exists() def test_cli_setup_failure_clears_stale_outputs(tmp_path: Path) -> None: output_dir = tmp_path / "output" output_dir.mkdir() (output_dir / kve.NATIVE_REPORT_FILENAME).write_text(json.dumps(_report())) - (output_dir / "results_2000-01-01T00-00-00.000000.json").write_text("{}") - - assert kve.main( - [ - "--model", - "model-a", - "--output-dir", - str(output_dir), - "--integration-error", - "checkout failed", - ] - ) == 1 + (output_dir / "results_kimi_vendor_2000-01-01T00-00-00.000000.json").write_text( + "{}" + ) + + assert ( + kve.main( + [ + "--model", + "model-a", + "--output-dir", + str(output_dir), + "--integration-error", + "checkout failed", + ] + ) + == 1 + ) projected = _result(output_dir) assert not (output_dir / kve.NATIVE_REPORT_FILENAME).exists() assert _score(output_dir) == 0.0 - assert projected["integration_error"]["message"] == "checkout failed" \ No newline at end of file + assert projected["integration_error"]["message"] == "checkout failed" diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index 1f6478934a..62d0ebaad2 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -14,7 +14,7 @@ source "$BENCHMARK_LIB" run_lm_eval() { echo "DISPATCH=lm-eval"; } run_swebench_eval() { echo "DISPATCH=swebench"; } -run_tool_use_eval() { echo "DISPATCH=tool-use"; } +run_kimi_vendor_eval() { echo "DISPATCH=kimi-vendor"; } append_lm_eval_summary() { echo "STAGED=summary"; } export EVAL_MAX_MODEL_LEN=16384 export EVAL_CONCURRENT_REQUESTS="" @@ -82,21 +82,21 @@ def test_env_can_force_swebench_on_fixed_seqlen(): -def test_env_can_force_tool_use_on_agentic_eval() -> None: - assert "DISPATCH=tool-use" in _dispatch( +def test_env_can_force_kimi_vendor_on_agentic_eval() -> None: + assert "DISPATCH=kimi-vendor" in _dispatch( is_agentic="1", eval_only="true", - env_fw="tool-use", + env_fw="kimi-vendor", ) -def test_tool_use_skips_unused_model_context_loading() -> None: +def test_kimi_vendor_skips_unused_model_context_loading() -> None: script = r''' source "$BENCHMARK_LIB" unset EVAL_MAX_MODEL_LEN compute_eval_context_length() { echo "UNEXPECTED_CONTEXT_LOAD"; return 99; } -run_tool_use_eval() { echo "DISPATCH=tool-use"; } -export EVAL_FRAMEWORK=tool-use +run_kimi_vendor_eval() { echo "DISPATCH=kimi-vendor"; } +export EVAL_FRAMEWORK=kimi-vendor export EVAL_CONCURRENT_REQUESTS="" export EVAL_ONLY=false export IS_AGENTIC=0 @@ -111,7 +111,7 @@ def test_tool_use_skips_unused_model_context_loading() -> None: ) assert result.returncode == 0, result.stderr - assert "DISPATCH=tool-use" in result.stdout + assert "DISPATCH=kimi-vendor" in result.stdout assert "UNEXPECTED_CONTEXT_LOAD" not in result.stdout @@ -143,43 +143,43 @@ def test_run_eval_rejects_missing_framework_value(): assert "--framework requires a value" in result.stderr -def test_tool_use_rejects_batched_concurrency() -> None: +def test_kimi_vendor_rejects_batched_concurrency() -> None: result = _run_invalid_call( "EVAL_MAX_MODEL_LEN=16384 " "EVAL_CONCURRENT_REQUESTS='1 4' " - "run_eval --framework tool-use" + "run_eval --framework kimi-vendor" ) assert result.returncode == 1 assert "batched eval concurrency is only supported for lm-eval" in result.stderr -def test_tool_use_rejects_unsupported_suite() -> None: +def test_kimi_vendor_rejects_unsupported_suite() -> None: result = _run_invalid_call( - "EVAL_SUITE=gsm8k run_tool_use_eval" + "EVAL_SUITE=gsm8k run_kimi_vendor_eval" ) assert result.returncode == 2 - assert "supports only EVAL_SUITE=kimi_tool_call_schema" in result.stderr + assert "unsupported Kimi Vendor Verifier suite 'gsm8k'" in result.stderr -def test_tool_use_rejects_multinode() -> None: +def test_kimi_vendor_rejects_multinode() -> None: for value in ("true", "1"): result = _run_invalid_call( f"EVAL_SUITE=kimi_tool_call_schema IS_MULTINODE={value} " - "run_tool_use_eval" + "run_kimi_vendor_eval" ) assert result.returncode == 2 - assert "supports single-node evals only" in result.stderr + assert "supports single-node only" in result.stderr -def test_tool_use_setup_failure_writes_compatibility_result( +def test_kimi_vendor_setup_failure_writes_compatibility_result( tmp_path: Path, ) -> None: results_dir = tmp_path / "results" script = r''' source "$BENCHMARK_LIB" -_require_tool_use_python() { :; } -_install_tool_use_eval_deps() { return 12; } -run_tool_use_eval --results-dir "$RESULTS_DIR" +_require_kimi_vendor_python() { :; } +_prepare_kimi_vendor_runtime() { return 12; } +run_kimi_vendor_eval --results-dir "$RESULTS_DIR" printf 'SETUP_RC=%s\n' "$?" ''' env = { @@ -193,7 +193,6 @@ def test_tool_use_setup_failure_writes_compatibility_result( for key in ( "EVAL_SUITE", "EVAL_RESULT_DIR", - "INFERENCEX_TOOL_USE_EVAL_RUNTIME_READY", "MODEL_NAME", ): env.pop(key, None) @@ -205,7 +204,7 @@ def test_tool_use_setup_failure_writes_compatibility_result( capture_output=True, check=True, ) - message = "tool-use dependency installation failed with exit code 12" + message = "Kimi Vendor Verifier dependency installation failed with exit code 12" score_files = list(results_dir.glob("results*.json")) assert "SETUP_RC=12" in result.stdout @@ -222,24 +221,79 @@ def test_tool_use_setup_failure_writes_compatibility_result( assert not (results_dir / "kimi_vendor_report.json").exists() +def test_kimi_vendor_dependency_install_is_isolated(tmp_path: Path) -> None: + runtime_dir = tmp_path / "runtime" + script = r''' +source "$BENCHMARK_LIB" +python3() { printf 'PYTHON_ARG=<%s>\n' "$@"; } +_install_kimi_vendor_eval_deps "$RUNTIME_DIR" +''' + result = subprocess.run( + ["bash", "-c", script], + env={ + **os.environ, + "BENCHMARK_LIB": str(BENCHMARK_LIB), + "RUNTIME_DIR": str(runtime_dir), + }, + text=True, + capture_output=True, + check=True, + ) + + assert "PYTHON_ARG=<--target>" in result.stdout + assert f"PYTHON_ARG=<{runtime_dir}>" in result.stdout + assert "--break-system-packages" not in result.stdout -def test_tool_use_runner_uses_fixed_upstream_contract(tmp_path: Path) -> None: +def test_kimi_vendor_surfaces_failure_artifact_error(tmp_path: Path) -> None: + script = r''' +source "$BENCHMARK_LIB" +_require_kimi_vendor_python() { return 12; } +_write_kimi_vendor_integration_error() { return 23; } +run_kimi_vendor_eval --results-dir "$RESULTS_DIR" +printf 'EVAL_RC=%s\n' "$?" +''' + result = subprocess.run( + ["bash", "-c", script], + env={ + **os.environ, + "BENCHMARK_LIB": str(BENCHMARK_LIB), + "RESULTS_DIR": str(tmp_path / "results"), + "MODEL": "test-model", + "IS_MULTINODE": "false", + }, + text=True, + capture_output=True, + check=True, + ) + + assert "EVAL_RC=12" in result.stdout + assert "failed to write Kimi verifier failure artifact" in result.stderr + + + + +def test_kimi_vendor_runner_uses_fixed_upstream_contract(tmp_path: Path) -> None: results_dir = tmp_path / "results" verifier_dir = tmp_path / "verifier" + runtime_dir = tmp_path / "runtime" + runtime_dir.mkdir() + verifier_dir.mkdir() script = r''' source "$BENCHMARK_LIB" -_require_tool_use_python() { :; } -_install_tool_use_eval_deps() { echo "INSTALL=UPSTREAM_MINIMAL"; } +_require_kimi_vendor_python() { :; } +_prepare_kimi_vendor_runtime() { printf '%s\n' "$RUNTIME_DIR"; } _prepare_kimi_vendor_verifier() { - printf 'CHECKOUT=%s@%s\n' "$1" "$2" - KIMI_VENDOR_VERIFIER_CHECKOUT_DIR="$VERIFIER_DIR" + printf 'CHECKOUT=%s@%s\n' "$1" "$2" >&2 + printf '%s\n' "$VERIFIER_DIR" } -python3() { printf 'PYTHON_ARG=<%s>\n' "$@"; } -run_tool_use_eval --port 9999 --results-dir "$RESULTS_DIR" +python3() { + printf 'PYTHONPATH=<%s>\n' "$PYTHONPATH" + printf 'PYTHON_ARG=<%s>\n' "$@" +} +run_kimi_vendor_eval --port 9999 --results-dir "$RESULTS_DIR" printf 'EVAL_SUITE=%s\n' "$EVAL_SUITE" printf 'EVAL_RESULT_DIR=%s\n' "$EVAL_RESULT_DIR" -printf 'RUNTIME_READY=%s\n' "$INFERENCEX_TOOL_USE_EVAL_RUNTIME_READY" ''' env = { **os.environ, @@ -247,6 +301,7 @@ def test_tool_use_runner_uses_fixed_upstream_contract(tmp_path: Path) -> None: "RESULTS_DIR": str(results_dir), "VERIFIER_DIR": str(verifier_dir), "MODEL": "test-model", + "RUNTIME_DIR": str(runtime_dir), "OPENAI_API_KEY": "must-not-be-forwarded", "KV_OFFLOADING": "none", "IS_MULTINODE": "false", @@ -254,8 +309,6 @@ def test_tool_use_runner_uses_fixed_upstream_contract(tmp_path: Path) -> None: for key in ( "EVAL_SUITE", "EVAL_RESULT_DIR", - "INFERENCEX_TOOL_USE_EVAL_RUNTIME_READY", - "KIMI_VENDOR_VERIFIER_CHECKOUT_DIR", "MODEL_NAME", ): env.pop(key, None) @@ -267,10 +320,10 @@ def test_tool_use_runner_uses_fixed_upstream_contract(tmp_path: Path) -> None: capture_output=True, check=True, ) - output = result.stdout + output = result.stdout + result.stderr adapter = BENCHMARK_LIB.parents[1] / "utils/evals/kimi_vendor_eval.py" - assert output.count("INSTALL=UPSTREAM_MINIMAL") == 1 + assert f"PYTHONPATH=<{tmp_path / 'runtime'}" in output assert ( "CHECKOUT=https://github.com/MoonshotAI/Kimi-Vendor-Verifier.git" "@b9ed3a6665bdff2c943246f7d2903cd003d6ddd6" @@ -287,7 +340,8 @@ def test_tool_use_runner_uses_fixed_upstream_contract(tmp_path: Path) -> None: assert "must-not-be-forwarded" not in output assert "EVAL_SUITE=kimi_tool_call_schema" in output assert f"EVAL_RESULT_DIR={results_dir}" in output - assert "RUNTIME_READY=true" in output + assert not (tmp_path / "runtime").exists() + assert not verifier_dir.exists() def test_run_lm_eval_rejects_missing_option_value(): diff --git a/utils/test_collect_eval_results.py b/utils/test_collect_eval_results.py index ba9ebef44c..3842aeae4c 100644 --- a/utils/test_collect_eval_results.py +++ b/utils/test_collect_eval_results.py @@ -3,7 +3,7 @@ import json from pathlib import Path -from collect_eval_results import build_row, collect_eval_rows +from collect_eval_results import EVAL_RESULT_FORMAT, build_row, collect_eval_rows def test_build_row_preserves_sequence_lengths() -> None: @@ -119,3 +119,24 @@ def test_collect_eval_rows_ignores_failed_batch_points( rows = collect_eval_rows(tmp_path) assert [row["conc"] for row in rows] == [4] + + + +def test_collect_eval_rows_accepts_neutral_result_format(tmp_path: Path) -> None: + artifact_dir = tmp_path / "eval_provider" + artifact_dir.mkdir() + (artifact_dir / "meta_env.json").write_text( + json.dumps({"eval_suite": "provider_smoke"}) + ) + result_path = artifact_dir / "results_provider.json" + _write_lm_eval_result(result_path, 1.0) + result = json.loads(result_path.read_text()) + result.pop("lm_eval_version") + result["result_format"] = EVAL_RESULT_FORMAT + result_path.write_text(json.dumps(result)) + + rows = collect_eval_rows(tmp_path) + + assert len(rows) == 1 + assert rows[0]["score"] == 1.0 + assert rows[0]["eval_suite"] == "provider_smoke" \ No newline at end of file From b0fd8cc53c97b415197795825cb8a22f99e343b3 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:50:12 -0500 Subject: [PATCH 04/24] fix: distinguish successful failure artifact writes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：区分失败结果产物写入成功与写入失败，同时保留原始安装失败退出码。 --- utils/evals/kimi_vendor_eval.py | 2 +- utils/evals/test_kimi_vendor_eval.py | 4 ++-- utils/evals/test_run_eval_dispatch.py | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/utils/evals/kimi_vendor_eval.py b/utils/evals/kimi_vendor_eval.py index 01d024771c..3f183c5423 100755 --- a/utils/evals/kimi_vendor_eval.py +++ b/utils/evals/kimi_vendor_eval.py @@ -245,7 +245,7 @@ def main(argv: Sequence[str] | None = None) -> int: args.model, 0.0, RuntimeError(args.integration_error) ), ) - return 1 + return 0 passed = run_evaluation( verifier_dir=args.verifier_dir, base_url=args.base_url, diff --git a/utils/evals/test_kimi_vendor_eval.py b/utils/evals/test_kimi_vendor_eval.py index db08ce17d9..fa126e3134 100644 --- a/utils/evals/test_kimi_vendor_eval.py +++ b/utils/evals/test_kimi_vendor_eval.py @@ -196,7 +196,7 @@ def fail_collection(*args: Any, **kwargs: Any) -> SimpleNamespace: assert foreign_result.exists() -def test_cli_setup_failure_clears_stale_outputs(tmp_path: Path) -> None: +def test_cli_setup_failure_writes_zero_score_artifact(tmp_path: Path) -> None: output_dir = tmp_path / "output" output_dir.mkdir() (output_dir / kve.NATIVE_REPORT_FILENAME).write_text(json.dumps(_report())) @@ -215,7 +215,7 @@ def test_cli_setup_failure_clears_stale_outputs(tmp_path: Path) -> None: "checkout failed", ] ) - == 1 + == 0 ) projected = _result(output_dir) assert not (output_dir / kve.NATIVE_REPORT_FILENAME).exists() diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index 62d0ebaad2..e670b2fc2d 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -209,6 +209,7 @@ def test_kimi_vendor_setup_failure_writes_compatibility_result( assert "SETUP_RC=12" in result.stdout assert message in result.stderr + assert "failed to write Kimi verifier failure artifact" not in result.stderr assert len(score_files) == 1 score_result = json.loads(score_files[0].read_text()) assert ( From 1684f55273309fb68816d84f17c3e29fc3c771d3 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:02:42 -0500 Subject: [PATCH 05/24] fix: preserve agentic eval decoding mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：将 agentic 评估矩阵的投机解码模式传递给启动器，避免 MTP 配置静默回退到 STP。 --- .github/workflows/e2e-tests.yml | 2 +- utils/evals/EVALS.md | 2 ++ utils/evals/test_run_eval_dispatch.py | 14 +++++++++++++- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 6b9c67410b..b96cf4eab8 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -473,7 +473,7 @@ jobs: isl: '0' osl: '0' max-model-len: '0' - spec-decoding: 'none' + spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ 'false' }} run-eval: true eval-only: true diff --git a/utils/evals/EVALS.md b/utils/evals/EVALS.md index dcad93c066..b42316ecac 100644 --- a/utils/evals/EVALS.md +++ b/utils/evals/EVALS.md @@ -58,6 +58,8 @@ selects a case set understood by that adapter. Each adapter owns its endpoint format, dependencies, native report, metrics, and pass policy. Future MiniMax or BFCL support should add explicit `run_eval` cases rather than a shared request or report abstraction. +Agentic eval jobs forward the matrix `spec-decoding` value, so MTP entries +launch their existing `*_mtp.sh` server instead of silently falling back to STP. ### Stock Kimi tool-call schema smoke diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index e670b2fc2d..cad7cb09ad 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -7,8 +7,11 @@ import subprocess from pathlib import Path +import yaml -BENCHMARK_LIB = Path(__file__).resolve().parents[2] / "benchmarks" / "benchmark_lib.sh" +REPO_ROOT = Path(__file__).resolve().parents[2] +BENCHMARK_LIB = REPO_ROOT / "benchmarks" / "benchmark_lib.sh" +E2E_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "e2e-tests.yml" _SCRIPT = r''' source "$BENCHMARK_LIB" @@ -1001,3 +1004,12 @@ def test_eval_limit_full_and_zero_accepted(tmp_path): assert "GEN_RC=0" in res.stdout, f"EVAL_LIMIT={sentinel!r}: {res.stdout}{res.stderr}" argv = (shim / "argv.log").read_text() assert "--slice" not in argv + + +def test_agentic_eval_workflow_forwards_runner_contract() -> None: + workflow = yaml.safe_load(E2E_WORKFLOW.read_text()) + forwarded = workflow["jobs"]["test-sweep-agentic-evals"]["with"] + + assert forwarded["spec-decoding"] == "${{ matrix.config.spec-decoding }}" + assert forwarded["eval-framework"] == "${{ inputs.eval-framework }}" + assert forwarded["eval-suite"] == "${{ inputs.eval-suite }}" From 58858593f73219274224ef73537da445c249e36a Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:31:22 -0500 Subject: [PATCH 06/24] fix: complete eval-only workflow runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：补全纯评估工作流的结果收集依赖，避免所有评估任务成功后工作流仍显示失败。 --- .github/workflows/e2e-tests.yml | 4 ++-- utils/evals/EVALS.md | 2 +- utils/evals/test_run_eval_dispatch.py | 13 +++++++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index b96cf4eab8..91b3d57e53 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -618,8 +618,8 @@ jobs: ref: ${{ inputs.ref }} collect-results: - needs: [test-sweep-multi-node, test-sweep-single-node, test-sweep-agentic, test-sweep-multi-node-agentic] - if: ${{ always() && (needs.test-sweep-multi-node.result != 'skipped' || needs.test-sweep-single-node.result != 'skipped' || needs.test-sweep-agentic.result != 'skipped' || needs.test-sweep-multi-node-agentic.result != 'skipped') }} + needs: [test-sweep-multi-node, test-sweep-single-node, test-sweep-agentic, test-sweep-multi-node-agentic, test-sweep-evals, test-sweep-multi-node-evals, test-sweep-agentic-evals] + if: ${{ always() && (needs.test-sweep-multi-node.result != 'skipped' || needs.test-sweep-single-node.result != 'skipped' || needs.test-sweep-agentic.result != 'skipped' || needs.test-sweep-multi-node-agentic.result != 'skipped' || needs.test-sweep-evals.result != 'skipped' || needs.test-sweep-multi-node-evals.result != 'skipped' || needs.test-sweep-agentic-evals.result != 'skipped') }} uses: ./.github/workflows/collect-results.yml secrets: inherit with: diff --git a/utils/evals/EVALS.md b/utils/evals/EVALS.md index b42316ecac..0af37d1f4e 100644 --- a/utils/evals/EVALS.md +++ b/utils/evals/EVALS.md @@ -156,7 +156,7 @@ For multi-node `all-evals`, `EVAL_CONC` is a space-separated list. When it conta - `e2e-tests.yml`: `test-sweep-evals` (single-node) and `test-sweep-multi-node-evals` (multi-node) - `run-sweep.yml`: `sweep-evals` (single-node) and `sweep-multi-node-evals` (multi-node) - Both use their respective benchmark templates with `eval-only: true`, `run-eval: true` -- `collect-evals` depends on both eval jobs, while `collect-results` only runs when benchmark jobs ran +- `collect-evals` depends on the eval jobs. `run-sweep.yml` collects throughput results only when benchmark jobs ran; `e2e-tests.yml` also completes the throughput collector dependency after an eval-only dispatch so the workflow can finish successfully. - `process_changelog.py` splits eval results into `evals` (single-node) and `multinode_evals` ### Result collection diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index cad7cb09ad..0129a69e83 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -1013,3 +1013,16 @@ def test_agentic_eval_workflow_forwards_runner_contract() -> None: assert forwarded["spec-decoding"] == "${{ matrix.config.spec-decoding }}" assert forwarded["eval-framework"] == "${{ inputs.eval-framework }}" assert forwarded["eval-suite"] == "${{ inputs.eval-suite }}" + + +def test_eval_only_workflow_completes_throughput_collection_dependency() -> None: + workflow = yaml.safe_load(E2E_WORKFLOW.read_text()) + collect_results = workflow["jobs"]["collect-results"] + + for eval_job in ( + "test-sweep-evals", + "test-sweep-multi-node-evals", + "test-sweep-agentic-evals", + ): + assert eval_job in collect_results["needs"] + assert f"needs.{eval_job}.result != 'skipped'" in collect_results["if"] From 83d7f575664f1331c62ccd5dd423c43ffb093798 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:43:20 -0500 Subject: [PATCH 07/24] revert: keep eval-only collection scoped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The collector dependency change did not affect GitHub’s workflow conclusion and added unrelated orchestration scope. 中文：恢复纯评估结果收集的原有范围。该依赖调整未改变 GitHub 工作流结论，且扩大了无关改动范围。 --- .github/workflows/e2e-tests.yml | 4 ++-- utils/evals/EVALS.md | 2 +- utils/evals/test_run_eval_dispatch.py | 13 ------------- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 91b3d57e53..b96cf4eab8 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -618,8 +618,8 @@ jobs: ref: ${{ inputs.ref }} collect-results: - needs: [test-sweep-multi-node, test-sweep-single-node, test-sweep-agentic, test-sweep-multi-node-agentic, test-sweep-evals, test-sweep-multi-node-evals, test-sweep-agentic-evals] - if: ${{ always() && (needs.test-sweep-multi-node.result != 'skipped' || needs.test-sweep-single-node.result != 'skipped' || needs.test-sweep-agentic.result != 'skipped' || needs.test-sweep-multi-node-agentic.result != 'skipped' || needs.test-sweep-evals.result != 'skipped' || needs.test-sweep-multi-node-evals.result != 'skipped' || needs.test-sweep-agentic-evals.result != 'skipped') }} + needs: [test-sweep-multi-node, test-sweep-single-node, test-sweep-agentic, test-sweep-multi-node-agentic] + if: ${{ always() && (needs.test-sweep-multi-node.result != 'skipped' || needs.test-sweep-single-node.result != 'skipped' || needs.test-sweep-agentic.result != 'skipped' || needs.test-sweep-multi-node-agentic.result != 'skipped') }} uses: ./.github/workflows/collect-results.yml secrets: inherit with: diff --git a/utils/evals/EVALS.md b/utils/evals/EVALS.md index 0af37d1f4e..b42316ecac 100644 --- a/utils/evals/EVALS.md +++ b/utils/evals/EVALS.md @@ -156,7 +156,7 @@ For multi-node `all-evals`, `EVAL_CONC` is a space-separated list. When it conta - `e2e-tests.yml`: `test-sweep-evals` (single-node) and `test-sweep-multi-node-evals` (multi-node) - `run-sweep.yml`: `sweep-evals` (single-node) and `sweep-multi-node-evals` (multi-node) - Both use their respective benchmark templates with `eval-only: true`, `run-eval: true` -- `collect-evals` depends on the eval jobs. `run-sweep.yml` collects throughput results only when benchmark jobs ran; `e2e-tests.yml` also completes the throughput collector dependency after an eval-only dispatch so the workflow can finish successfully. +- `collect-evals` depends on both eval jobs, while `collect-results` only runs when benchmark jobs ran - `process_changelog.py` splits eval results into `evals` (single-node) and `multinode_evals` ### Result collection diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index 0129a69e83..cad7cb09ad 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -1013,16 +1013,3 @@ def test_agentic_eval_workflow_forwards_runner_contract() -> None: assert forwarded["spec-decoding"] == "${{ matrix.config.spec-decoding }}" assert forwarded["eval-framework"] == "${{ inputs.eval-framework }}" assert forwarded["eval-suite"] == "${{ inputs.eval-suite }}" - - -def test_eval_only_workflow_completes_throughput_collection_dependency() -> None: - workflow = yaml.safe_load(E2E_WORKFLOW.read_text()) - collect_results = workflow["jobs"]["collect-results"] - - for eval_job in ( - "test-sweep-evals", - "test-sweep-multi-node-evals", - "test-sweep-agentic-evals", - ): - assert eval_job in collect_results["needs"] - assert f"needs.{eval_job}.result != 'skipped'" in collect_results["if"] From 07dc9d95bac5ccfd72a73e578589e0c3ac26bfcb Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:13:44 -0500 Subject: [PATCH 08/24] fix: correct verifier failure metadata and links MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：修正验证器失败样本数、共享格式契约、路径复用及双语文档链接。 --- benchmarks/benchmark_lib.sh | 12 +-- docs/eval-agentx-procedures.md | 20 ++--- docs/eval-agentx-procedures_zh.md | 20 ++--- utils/evals/kimi_vendor_eval.py | 25 ++++-- utils/evals/test_kimi_vendor_eval.py | 7 ++ utils/test_collect_eval_results.py | 110 +++++++++++++++------------ 6 files changed, 114 insertions(+), 80 deletions(-) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 1c1c3d8f5f..4fc7ce6839 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -11,6 +11,9 @@ mkdir -p "$PYTHONPYCACHEPREFIX" 2>/dev/null || true INFERENCEX_BENCHMARK_LIB_DIR="$( cd "$(dirname "${BASH_SOURCE[0]}")" && pwd )" +INFERENCEX_REPO_ROOT="$( + cd "$INFERENCEX_BENCHMARK_LIB_DIR/.." && pwd +)" # Inference server port shared by every benchmark recipe. Launchers that need # a non-default value (e.g. launch_mi355x-amds.sh derives PORT from RUNNER_NAME @@ -934,10 +937,8 @@ _run_kimi_tool_call_schema_eval() { ;; esac - local repo_root - repo_root="$(cd "$INFERENCEX_BENCHMARK_LIB_DIR/.." && pwd)" local model_name="${MODEL_NAME:-${MODEL:-}}" - local adapter_path="${repo_root}/utils/evals/kimi_vendor_eval.py" + local adapter_path="${INFERENCEX_REPO_ROOT}/utils/evals/kimi_vendor_eval.py" local runtime_dir="" local checkout_dir="" @@ -1006,7 +1007,7 @@ run_kimi_vendor_eval() { } _eval_patches_dir() { - cd "$(dirname "${BASH_SOURCE[0]}")/../utils/evals/patches" && pwd + printf '%s\n' "${INFERENCEX_REPO_ROOT}/utils/evals/patches" } _patch_lm_eval() { @@ -1115,8 +1116,7 @@ run_lm_eval() { done # Serving images may use a different WORKDIR. - local _repo_root - _repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + local _repo_root="$INFERENCEX_REPO_ROOT" if [[ "$tasks_dir" == *.yaml && "$tasks_dir" != /* \ && ! -f "$tasks_dir" && -f "$_repo_root/$tasks_dir" ]]; then echo "run_lm_eval: anchoring relative task '$tasks_dir' to repo root -> $_repo_root/$tasks_dir" diff --git a/docs/eval-agentx-procedures.md b/docs/eval-agentx-procedures.md index a3afb3076a..e5c349fd9c 100644 --- a/docs/eval-agentx-procedures.md +++ b/docs/eval-agentx-procedures.md @@ -85,7 +85,7 @@ append_lm_eval_summary python3 utils/evals/validate_scores.py --model-prefix "$MODEL_PREFIX" ``` -`run_lm_eval` passes concurrency through `num_concurrent` in `--model_args`. It is deliberately an environment variable, not a `run_eval` CLI option. The exact invocation is in [`run_lm_eval()`](../benchmarks/benchmark_lib.sh#L890-L970). +`run_lm_eval` passes concurrency through `num_concurrent` in `--model_args`. It is deliberately an environment variable, not a `run_eval` CLI option. The exact invocation is in [`run_lm_eval()`](../benchmarks/benchmark_lib.sh#L1080-L1162). ## 3. `EVAL_ONLY` is a launcher contract @@ -97,9 +97,9 @@ Set `EVAL_ONLY=true` **before server launch**. It is not merely a switch inside 4. Throughput returns immediately or is skipped. 5. `run_eval` and artifact staging run. -Relevant implementation: [context setup](../benchmarks/benchmark_lib.sh#L853-L888), [eval dispatch and failure policy](../benchmarks/benchmark_lib.sh#L1537-L1654), and [workflow inputs](../.github/workflows/benchmark-tmpl.yml#L162-L185). +Relevant implementation: [context setup](../benchmarks/benchmark_lib.sh#L1049-L1078), [eval dispatch and failure policy](../benchmarks/benchmark_lib.sh#L1737-L1856), and [workflow inputs](../.github/workflows/benchmark-tmpl.yml#L79-L97). -Do not toggle `EVAL_ONLY` after a throughput-sized server is already running and assume the context changed. Restart through the recipe. In eval-only mode an eval failure is returned after available artifacts are staged. In a workflow, upload happens with `always()` before score validation so failed evidence survives ([single-node upload and gate](../.github/workflows/benchmark-tmpl.yml#L387-L404), [multi-node upload and gate](../.github/workflows/benchmark-multinode-tmpl.yml#L450-L468)). +Do not toggle `EVAL_ONLY` after a throughput-sized server is already running and assume the context changed. Restart through the recipe. In eval-only mode an eval failure is returned after available artifacts are staged. In a workflow, upload happens with `always()` before score validation so failed evidence survives ([single-node upload and gate](../.github/workflows/benchmark-tmpl.yml#L399-L417), [multi-node upload and gate](../.github/workflows/benchmark-multinode-tmpl.yml#L450-L468)). ## 4. Batched eval concurrency @@ -121,9 +121,9 @@ The batch runner creates a fresh temporary output directory per point, stages fi - `completed_eval_concs`: eval and staging both succeeded. - `failed_eval_concs`: either eval or staging failed. -A failed point is deferred so artifacts from every attempted point can upload. The post-upload validator then fails the job. Batched mode accepts positive integers and supports only `lm-eval`. See [`run_eval` batching](../benchmarks/benchmark_lib.sh#L1537-L1631), [artifact suffixing](../benchmarks/benchmark_lib.sh#L972-L1030), and [manifest validation](../utils/evals/validate_scores.py#L72-L171). +A failed point is deferred so artifacts from every attempted point can upload. The post-upload validator then fails the job. Batched mode accepts positive integers and supports only `lm-eval`. See [`run_eval` batching](../benchmarks/benchmark_lib.sh#L1737-L1832), [artifact suffixing](../benchmarks/benchmark_lib.sh#L1163-L1222), and [manifest validation](../utils/evals/validate_scores.py#L72-L171). -For multi-node `all-evals`, the workflow constructs `EVAL_CONC` by joining the topology's concurrency list ([dispatch](../.github/workflows/e2e-tests.yml#L375-L378)). Never compare a point if its `_conc` result or completed-manifest entry is missing. +For multi-node `all-evals`, the workflow constructs `EVAL_CONC` by joining the topology's concurrency list ([dispatch](../.github/workflows/e2e-tests.yml#L394-L398)). Never compare a point if its `_conc` result or completed-manifest entry is missing. ## 5. Validate scores, not file existence @@ -173,7 +173,7 @@ Retain `meta_env.json`, `results*.json`, and `sample*.jsonl`. Agentic SWE-bench ## 7. Run AgentX: fast feedback versus canonical evidence -AgentX is AIPerf `inferencex-agentx-mvp` trace replay, not a fixed-token synthetic benchmark. The checked-in default uses ten additional warmup requests per trajectory lane and the recipe's configured profile duration. `agentx-fast` forces one warmup request per lane and a 1,200-second profile. It affects single- and multi-node AgentX throughput only. Fixed-sequence throughput and evals remain canonical. Fast runs are not eligible for artifact reuse ([workflow policy](../.github/workflows/README.md#agentx-fast-mode), [fast replay settings](../benchmarks/benchmark_lib.sh#L1824-L1848)). +AgentX is AIPerf `inferencex-agentx-mvp` trace replay, not a fixed-token synthetic benchmark. The checked-in default uses ten additional warmup requests per trajectory lane and the recipe's configured profile duration. `agentx-fast` forces one warmup request per lane and a 1,200-second profile. It affects single- and multi-node AgentX throughput only. Fixed-sequence throughput and evals remain canonical. Fast runs are not eligible for artifact reuse ([workflow policy](../.github/workflows/README.md#agentx-fast-mode), [fast replay settings](../benchmarks/benchmark_lib.sh#L2026-L2050)). Targeted canonical run (configured duration and warmup, with fast and duration overrides omitted): @@ -205,11 +205,11 @@ gh workflow run e2e-tests.yml --repo SemiAnalysisAI/InferenceX --ref "$REF" \ For a publishable SWE-bench score, omit `eval-limit`. Do not use `single-shot`, which is only a debugging escape hatch. SWE-bench generation/scoring controls and its `0.50` full-split threshold are documented next to the implementation in [`utils/evals/EVALS.md`](../utils/evals/EVALS.md#swe-bench-lite---framework-swebench). -Treat fast results as bring-up evidence, never as a replacement for the canonical candidate. A duration below 900 seconds or `AIPERF_UNSAFE_OVERRIDE=true` adds AIPerf's `--unsafe-override` and flags the submission invalid. Use it only for smoke diagnosis ([source](../benchmarks/benchmark_lib.sh#L1982-L1989)). After a fast run is healthy, run the exact candidate canonically before claiming benchmark success. +Treat fast results as bring-up evidence, never as a replacement for the canonical candidate. A duration below 900 seconds or `AIPERF_UNSAFE_OVERRIDE=true` adds AIPerf's `--unsafe-override` and flags the submission invalid. Use it only for smoke diagnosis ([source](../benchmarks/benchmark_lib.sh#L2188-L2190)). After a fast run is healthy, run the exact candidate canonically before claiming benchmark success. ## 8. Preserve trace and run provenance -AgentX defaults to recorded assistant-response replay. Live server outputs are measured but discarded when constructing later turns. Set `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1` only for an explicitly different live-assistant experiment. The selected trace corpus is model-family dependent unless `WEKA_LOADER_OVERRIDE` pins it. The resolver logs both loader and Hugging Face dataset ([trace resolution](../benchmarks/benchmark_lib.sh#L1743-L1822), [replay semantics](../benchmarks/benchmark_lib.sh#L1824-L1850)). +AgentX defaults to recorded assistant-response replay. Live server outputs are measured but discarded when constructing later turns. Set `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1` only for an explicitly different live-assistant experiment. The selected trace corpus is model-family dependent unless `WEKA_LOADER_OVERRIDE` pins it. The resolver logs both loader and Hugging Face dataset ([trace resolution](../benchmarks/benchmark_lib.sh#L1945-L2024), [replay semantics](../benchmarks/benchmark_lib.sh#L2026-L2192)). Capture orchestration provenance immediately: @@ -241,7 +241,7 @@ For each concurrency retain: - server/frontend logs and every metrics endpoint represented. - run URL/ID, attempt, head SHA, recipe/config identity, image, topology, fast flag, and any override. -The runner writes the command before replay and validates raw results after aggregation ([execution path](../benchmarks/benchmark_lib.sh#L2040-L2079)). Aggregation preserves dataset provenance and hardware/model/topology fields ([aggregate construction](../utils/agentic/aggregation/process_agentic_result.py#L194-L272)). Raw workflow uploads intentionally omit very large `inputs.json` and `profile_export_raw.jsonl`. If those are required for an investigation, preserve them from the live allocation before cleanup ([single-node artifact contract](../.github/workflows/benchmark-tmpl.yml#L337-L346), [multi-node contract](../.github/workflows/benchmark-multinode-tmpl.yml#L439-L448)). +The runner writes the command before replay and validates raw results after aggregation ([execution path](../benchmarks/benchmark_lib.sh#L2242-L2282)). Aggregation preserves dataset provenance and hardware/model/topology fields ([aggregate construction](../utils/agentic/aggregation/process_agentic_result.py#L194-L272)). Raw workflow uploads intentionally omit very large `inputs.json` and `profile_export_raw.jsonl`. If those are required for an investigation, preserve them from the live allocation before cleanup ([single-node artifact contract](../.github/workflows/benchmark-tmpl.yml#L349-L358), [multi-node contract](../.github/workflows/benchmark-multinode-tmpl.yml#L439-L448)). ## 9. Debug long AgentX runs from live evidence @@ -290,7 +290,7 @@ curl -fsS '' | \ rg -i 'request|queue|cache|token|prefill|decode|error|fail' ``` -Track trends over repeated samples: running/waiting requests, KV usage, prefix hits, input/output token rates, completed/cancelled/errored requests, frontend routing balance, and disaggregated KV transfer. AIPerf records endpoint identity for every server series ([metrics wiring](../benchmarks/benchmark_lib.sh#L1963-L1980)). +Track trends over repeated samples: running/waiting requests, KV usage, prefix hits, input/output token rates, completed/cancelled/errored requests, frontend routing balance, and disaggregated KV transfer. AIPerf records endpoint identity for every server series ([metrics wiring](../benchmarks/benchmark_lib.sh#L2158-L2181)). Use phase markers, not total Slurm age: diff --git a/docs/eval-agentx-procedures_zh.md b/docs/eval-agentx-procedures_zh.md index 669ed290c3..904cc3d1b7 100644 --- a/docs/eval-agentx-procedures_zh.md +++ b/docs/eval-agentx-procedures_zh.md @@ -85,7 +85,7 @@ append_lm_eval_summary python3 utils/evals/validate_scores.py --model-prefix "$MODEL_PREFIX" ``` -`run_lm_eval` 通过 `--model_args` 中的 `num_concurrent` 传递并发；它刻意采用环境变量，而不是 `run_eval` CLI 选项。准确调用见 [`run_lm_eval()`](../benchmarks/benchmark_lib.sh#L890-L970)。 +`run_lm_eval` 通过 `--model_args` 中的 `num_concurrent` 传递并发；它刻意采用环境变量，而不是 `run_eval` CLI 选项。准确调用见 [`run_lm_eval()`](../benchmarks/benchmark_lib.sh#L1080-L1162)。 ## 3. `EVAL_ONLY` 是 launcher 约定 @@ -97,9 +97,9 @@ python3 utils/evals/validate_scores.py --model-prefix "$MODEL_PREFIX" 4. 吞吐量路径立即返回或被跳过。 5. 运行 `run_eval` 和 artifact staging。 -相关实现：[context 设置](../benchmarks/benchmark_lib.sh#L853-L888)、[eval 分派与失败策略](../benchmarks/benchmark_lib.sh#L1537-L1654) 和[工作流输入](../.github/workflows/benchmark-tmpl.yml#L162-L185)。 +相关实现：[context 设置](../benchmarks/benchmark_lib.sh#L1049-L1078)、[eval 分派与失败策略](../benchmarks/benchmark_lib.sh#L1737-L1856) 和[工作流输入](../.github/workflows/benchmark-tmpl.yml#L79-L97)。 -不要在吞吐量规格的服务已经运行后才切换 `EVAL_ONLY`，并假定 context 会随之变化。应通过 recipe 重启。Eval-only 模式会在暂存已有 artifact 后返回 eval 失败；在工作流中，上传步骤使用 `always()`，并位于分数校验前，因此失败证据仍会保留（[单节点上传与 gate](../.github/workflows/benchmark-tmpl.yml#L387-L404)、[多节点上传与 gate](../.github/workflows/benchmark-multinode-tmpl.yml#L450-L468)）。 +不要在吞吐量规格的服务已经运行后才切换 `EVAL_ONLY`，并假定 context 会随之变化。应通过 recipe 重启。Eval-only 模式会在暂存已有 artifact 后返回 eval 失败；在工作流中，上传步骤使用 `always()`，并位于分数校验前，因此失败证据仍会保留（[单节点上传与 gate](../.github/workflows/benchmark-tmpl.yml#L399-L417)、[多节点上传与 gate](../.github/workflows/benchmark-multinode-tmpl.yml#L450-L468)）。 ## 4. 批量 eval 并发 @@ -121,9 +121,9 @@ python3 utils/evals/validate_scores.py --expected-concs '16 32 64' - `completed_eval_concs`：eval 与 staging 均成功的点； - `failed_eval_concs`：eval 或 staging 失败的点。 -失败点会延迟报错，使所有已尝试点的 artifact 都能上传；随后 post-upload validator 会使作业失败。批量模式只接受正整数，且仅支持 `lm-eval`。参见 [`run_eval` batching](../benchmarks/benchmark_lib.sh#L1537-L1631)、[artifact 后缀处理](../benchmarks/benchmark_lib.sh#L972-L1030) 和[manifest 校验](../utils/evals/validate_scores.py#L72-L171)。 +失败点会延迟报错，使所有已尝试点的 artifact 都能上传；随后 post-upload validator 会使作业失败。批量模式只接受正整数，且仅支持 `lm-eval`。参见 [`run_eval` batching](../benchmarks/benchmark_lib.sh#L1737-L1832)、[artifact 后缀处理](../benchmarks/benchmark_lib.sh#L1163-L1222) 和[manifest 校验](../utils/evals/validate_scores.py#L72-L171)。 -对于多节点 `all-evals`，工作流通过连接拓扑的并发列表构造 `EVAL_CONC`（[分派](../.github/workflows/e2e-tests.yml#L375-L378)）。如果缺少某点的 `_conc` 结果或 completed manifest 条目，绝不能比较该点。 +对于多节点 `all-evals`，工作流通过连接拓扑的并发列表构造 `EVAL_CONC`（[分派](../.github/workflows/e2e-tests.yml#L394-L398)）。如果缺少某点的 `_conc` 结果或 completed manifest 条目，绝不能比较该点。 ## 5. 校验分数，而不只是检查文件存在 @@ -173,7 +173,7 @@ gh run download "$RUN_ID" --repo SemiAnalysisAI/InferenceX \ ## 7. 运行 AgentX：快速反馈与 canonical 证据 -AgentX 是 AIPerf `inferencex-agentx-mvp` trace replay，不是固定 token 的合成 benchmark。仓库默认设置对每条 trajectory lane 额外执行十个 warmup 请求，并使用 recipe 配置的 profile 时长。`agentx-fast` 强制每条 lane 只运行一个 warmup 请求，并将 profile 设为 1,200 秒。它只影响单节点和多节点 AgentX 吞吐量；定长序列吞吐量与 eval 保持 canonical。Fast 运行不符合 artifact reuse 条件（[工作流策略](../.github/workflows/README.md#agentx-fast-mode)、[Fast replay 设置](../benchmarks/benchmark_lib.sh#L1824-L1848)）。 +AgentX 是 AIPerf `inferencex-agentx-mvp` trace replay，不是固定 token 的合成 benchmark。仓库默认设置对每条 trajectory lane 额外执行十个 warmup 请求，并使用 recipe 配置的 profile 时长。`agentx-fast` 强制每条 lane 只运行一个 warmup 请求，并将 profile 设为 1,200 秒。它只影响单节点和多节点 AgentX 吞吐量；定长序列吞吐量与 eval 保持 canonical。Fast 运行不符合 artifact reuse 条件（[工作流策略](../.github/workflows/README.md#agentx-fast-mode)、[fast replay 设置](../benchmarks/benchmark_lib.sh#L2026-L2050)）。 目标 canonical 运行（使用配置的 duration 和 warmup；不要加 fast 或 duration override）： @@ -205,11 +205,11 @@ gh workflow run e2e-tests.yml --repo SemiAnalysisAI/InferenceX --ref "$REF" \ 要得到可发布的 SWE-bench 分数，省略 `eval-limit`；不要使用 `single-shot`，它只是诊断逃生选项。SWE-bench generation/scoring 控制项以及完整 split 的 `0.50` 阈值在实现旁的 [`utils/evals/EVALS.md`](../utils/evals/EVALS.md#swe-bench-lite---framework-swebench) 中说明。 -Fast 结果只能作为 bring-up 证据，绝不能替代 canonical candidate。小于 900 秒的 duration 或 `AIPERF_UNSAFE_OVERRIDE=true` 会添加 AIPerf 的 `--unsafe-override` 并将 submission 标记为无效；只能用于 smoke 诊断（[源码](../benchmarks/benchmark_lib.sh#L1982-L1989)）。Fast 运行健康后，必须对完全相同的 candidate 进行 canonical 运行，才能宣称 benchmark 成功。 +Fast 结果只能作为 bring-up 证据，绝不能替代 canonical candidate。小于 900 秒的 duration 或 `AIPERF_UNSAFE_OVERRIDE=true` 会添加 AIPerf 的 `--unsafe-override` 并将 submission 标记为无效；只能用于 smoke 诊断（[源码](../benchmarks/benchmark_lib.sh#L2188-L2190)）。Fast 运行健康后，必须对完全相同的 candidate 进行 canonical 运行，才能宣称 benchmark 成功。 ## 8. 保留 trace 与运行 provenance -AgentX 默认 replay 已记录的 assistant response。实时服务输出会被测量，但构造后续 turn 时会丢弃。只有在明确要进行不同的 live-assistant 实验时，才设置 `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1`。除非用 `WEKA_LOADER_OVERRIDE` 固定，否则所选 trace corpus 依赖模型 family；resolver 会同时记录 loader 与 Hugging Face dataset（[trace 解析](../benchmarks/benchmark_lib.sh#L1743-L1822)、[replay 语义](../benchmarks/benchmark_lib.sh#L1824-L1850)）。 +AgentX 默认 replay 已记录的 assistant response。实时服务输出会被测量，但构造后续 turn 时会丢弃。只有在明确要进行不同的 live-assistant 实验时，才设置 `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1`。除非用 `WEKA_LOADER_OVERRIDE` 固定，否则所选 trace corpus 依赖模型 family；resolver 会同时记录 loader 与 Hugging Face dataset（[trace 解析](../benchmarks/benchmark_lib.sh#L1945-L2024)、[replay 语义](../benchmarks/benchmark_lib.sh#L2026-L2192)）。 立即记录 orchestration provenance： @@ -241,7 +241,7 @@ gh run download "$RUN_ID" --repo SemiAnalysisAI/InferenceX \ - server/frontend 日志以及所代表的每个 metrics endpoint； - run URL/ID、attempt、head SHA、recipe/config 标识、image、topology、fast 标志和所有 override。 -Runner 会在 replay 前写入命令，并在聚合后校验原始结果（[执行路径](../benchmarks/benchmark_lib.sh#L2040-L2079)）。聚合会保留 dataset provenance 以及硬件/模型/拓扑字段（[aggregate 构造](../utils/agentic/aggregation/process_agentic_result.py#L194-L272)）。工作流的 raw upload 会有意排除体积很大的 `inputs.json` 和 `profile_export_raw.jsonl`；如果调查需要这些文件，应在清理前从实时 allocation 保存（[单节点 artifact 约定](../.github/workflows/benchmark-tmpl.yml#L337-L346)、[多节点约定](../.github/workflows/benchmark-multinode-tmpl.yml#L439-L448)）。 +Runner 会在 replay 前写入命令，并在聚合后校验原始结果（[执行路径](../benchmarks/benchmark_lib.sh#L2242-L2282)）。聚合会保留 dataset provenance 以及硬件/模型/拓扑字段（[aggregate 构造](../utils/agentic/aggregation/process_agentic_result.py#L194-L272)）。工作流的 raw upload 会有意排除体积很大的 `inputs.json` 和 `profile_export_raw.jsonl`；如果调查需要这些文件，应在清理前从实时 allocation 保存（[单节点 artifact 约定](../.github/workflows/benchmark-tmpl.yml#L349-L358)、[多节点约定](../.github/workflows/benchmark-multinode-tmpl.yml#L439-L448)）。 ## 9. 用实时证据调试长时间 AgentX 运行 @@ -290,7 +290,7 @@ curl -fsS '' | \ rg -i 'request|queue|cache|token|prefill|decode|error|fail' ``` -通过重复 sample 跟踪趋势：running/waiting request、KV usage、prefix hit、input/output token rate、completed/cancelled/errored request、frontend routing balance，以及 disaggregated KV transfer。AIPerf 会为每条 server series 记录 endpoint identity（[metrics 接线](../benchmarks/benchmark_lib.sh#L1963-L1980)）。 +通过重复 sample 跟踪趋势：running/waiting request、KV usage、prefix hit、input/output token rate、completed/cancelled/errored request、frontend routing balance，以及 disaggregated KV transfer。AIPerf 会为每条 server series 记录 endpoint identity（[metrics 接线](../benchmarks/benchmark_lib.sh#L2158-L2181)）。 应使用 phase marker，而不是 Slurm 总运行时间： diff --git a/utils/evals/kimi_vendor_eval.py b/utils/evals/kimi_vendor_eval.py index 3f183c5423..f29c621c6a 100755 --- a/utils/evals/kimi_vendor_eval.py +++ b/utils/evals/kimi_vendor_eval.py @@ -107,11 +107,15 @@ def _project_report(model: str, report: Any) -> tuple[dict[str, Any], bool]: ): raise ValueError("report does not contain the expected stream modes") score = passed / 2.0 - return _compatibility_result(model, score), passed == 2 + return _compatibility_result(model, score, n_samples=2), passed == 2 def _compatibility_result( - model: str, score: float, integration_error: BaseException | None = None + model: str, + score: float, + *, + n_samples: int, + integration_error: BaseException | None = None, ) -> dict[str, Any]: result: dict[str, Any] = { "result_format": RESULT_FORMAT, @@ -129,7 +133,7 @@ def _compatibility_result( "filter_list": [{"name": "strict-match"}], } }, - "n-samples": {TASK_NAME: {"original": 2, "effective": 2}}, + "n-samples": {TASK_NAME: {"original": n_samples, "effective": n_samples}}, } if integration_error is not None: result["integration_error"] = { @@ -158,7 +162,7 @@ def run_evaluation( compatibility_path = prepare_compatibility_path(output_dir) subprocess_rc: int | None = None integration_error: BaseException | None = None - compatibility = _compatibility_result(model, 0.0) + compatibility = _compatibility_result(model, 0.0, n_samples=0) complete_pass = False try: @@ -181,11 +185,15 @@ def run_evaluation( integration_error = RuntimeError( f"upstream verifier exited with code {subprocess_rc}" ) - compatibility = _compatibility_result(model, 0.0, integration_error) + compatibility = _compatibility_result( + model, 0.0, n_samples=2, integration_error=integration_error + ) complete_pass = False except (OSError, ValueError, subprocess.TimeoutExpired) as exc: integration_error = exc - compatibility = _compatibility_result(model, 0.0, exc) + compatibility = _compatibility_result( + model, 0.0, n_samples=0, integration_error=exc + ) finally: try: _write_compatibility(compatibility_path, compatibility) @@ -242,7 +250,10 @@ def main(argv: Sequence[str] | None = None) -> int: _write_compatibility( prepare_compatibility_path(args.output_dir), _compatibility_result( - args.model, 0.0, RuntimeError(args.integration_error) + args.model, + 0.0, + n_samples=0, + integration_error=RuntimeError(args.integration_error), ), ) return 0 diff --git a/utils/evals/test_kimi_vendor_eval.py b/utils/evals/test_kimi_vendor_eval.py index fa126e3134..fe8394efc8 100644 --- a/utils/evals/test_kimi_vendor_eval.py +++ b/utils/evals/test_kimi_vendor_eval.py @@ -39,6 +39,10 @@ def _score(output_dir: Path) -> float: return _result(output_dir)["results"][kve.TASK_NAME]["exact_match,strict-match"] +def _n_eff(output_dir: Path) -> int: + return _result(output_dir)["n-samples"][kve.TASK_NAME]["effective"] + + def test_builds_fixed_upstream_pytest_command(tmp_path: Path) -> None: report = tmp_path / kve.NATIVE_REPORT_FILENAME @@ -116,6 +120,7 @@ def fake_run( assert invocation["check"] is False assert invocation["timeout"] == kve.DEFAULT_TIMEOUT_SECONDS assert _score(output_dir) == expected_score + assert _n_eff(output_dir) == 2 projected = _result(output_dir) assert projected["result_format"] == kve.RESULT_FORMAT assert projected["eval_adapter"] == kve.ADAPTER_NAME @@ -163,6 +168,7 @@ def fake_run( projected = _result(output_dir) assert _score(output_dir) == 0.0 assert projected["integration_error"]["type"] == error_type + assert _n_eff(output_dir) == 0 def test_failure_cannot_reuse_stale_outputs( @@ -221,3 +227,4 @@ def test_cli_setup_failure_writes_zero_score_artifact(tmp_path: Path) -> None: assert not (output_dir / kve.NATIVE_REPORT_FILENAME).exists() assert _score(output_dir) == 0.0 assert projected["integration_error"]["message"] == "checkout failed" + assert _n_eff(output_dir) == 0 diff --git a/utils/test_collect_eval_results.py b/utils/test_collect_eval_results.py index 3842aeae4c..2c3b54a02a 100644 --- a/utils/test_collect_eval_results.py +++ b/utils/test_collect_eval_results.py @@ -4,6 +4,11 @@ from pathlib import Path from collect_eval_results import EVAL_RESULT_FORMAT, build_row, collect_eval_rows +from evals.kimi_vendor_eval import RESULT_FORMAT as KIMI_VENDOR_RESULT_FORMAT + + +def test_kimi_vendor_result_format_matches_collector_contract() -> None: + assert KIMI_VENDOR_RESULT_FORMAT == EVAL_RESULT_FORMAT def test_build_row_preserves_sequence_lengths() -> None: @@ -34,23 +39,27 @@ def test_build_row_preserves_explicit_eval_suite() -> None: def _write_lm_eval_result(path: Path, score: float) -> None: - path.write_text(json.dumps({ - "lm_eval_version": "0.4.0", - "model_name": "test-model", - "results": { - "gsm8k": { - "exact_match,strict-match": score, - "exact_match_stderr,strict-match": 0.01, - }, - }, - "configs": { - "gsm8k": { - "metric_list": [{"metric": "exact_match"}], - "filter_list": [{"name": "strict-match"}], - }, - }, - "n-samples": {"gsm8k": {"effective": 10}}, - })) + path.write_text( + json.dumps( + { + "lm_eval_version": "0.4.0", + "model_name": "test-model", + "results": { + "gsm8k": { + "exact_match,strict-match": score, + "exact_match_stderr,strict-match": 0.01, + }, + }, + "configs": { + "gsm8k": { + "metric_list": [{"metric": "exact_match"}], + "filter_list": [{"name": "strict-match"}], + }, + }, + "n-samples": {"gsm8k": {"effective": 10}}, + } + ) + ) def test_collect_eval_rows_expands_batched_concurrencies( @@ -58,27 +67,31 @@ def test_collect_eval_rows_expands_batched_concurrencies( ) -> None: artifact_dir = tmp_path / "eval_batch" artifact_dir.mkdir() - (artifact_dir / "meta_env.json").write_text(json.dumps({ - "is_multinode": True, - "infmax_model_prefix": "gptoss", - "hw": "gb200", - "framework": "dynamo-sglang", - "precision": "fp8", - "spec_decoding": "none", - "isl": 8192, - "osl": 1024, - "prefill_tp": 4, - "prefill_ep": 1, - "prefill_num_workers": 1, - "decode_tp": 8, - "decode_ep": 1, - "decode_num_workers": 2, - "eval_concs": [4, 16], - "completed_eval_concs": [4, 16], - "failed_eval_concs": [], - "conc": 4, - "eval_suite": "gsm8k", - })) + (artifact_dir / "meta_env.json").write_text( + json.dumps( + { + "is_multinode": True, + "infmax_model_prefix": "gptoss", + "hw": "gb200", + "framework": "dynamo-sglang", + "precision": "fp8", + "spec_decoding": "none", + "isl": 8192, + "osl": 1024, + "prefill_tp": 4, + "prefill_ep": 1, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_num_workers": 2, + "eval_concs": [4, 16], + "completed_eval_concs": [4, 16], + "failed_eval_concs": [], + "conc": 4, + "eval_suite": "gsm8k", + } + ) + ) _write_lm_eval_result( artifact_dir / "results_test_conc4.json", 0.90, @@ -100,13 +113,17 @@ def test_collect_eval_rows_ignores_failed_batch_points( ) -> None: artifact_dir = tmp_path / "eval_batch" artifact_dir.mkdir() - (artifact_dir / "meta_env.json").write_text(json.dumps({ - "is_multinode": True, - "eval_concs": [4, 16], - "completed_eval_concs": [4], - "failed_eval_concs": [16], - "conc": 4, - })) + (artifact_dir / "meta_env.json").write_text( + json.dumps( + { + "is_multinode": True, + "eval_concs": [4, 16], + "completed_eval_concs": [4], + "failed_eval_concs": [16], + "conc": 4, + } + ) + ) _write_lm_eval_result( artifact_dir / "results_test_conc4.json", 0.90, @@ -121,7 +138,6 @@ def test_collect_eval_rows_ignores_failed_batch_points( assert [row["conc"] for row in rows] == [4] - def test_collect_eval_rows_accepts_neutral_result_format(tmp_path: Path) -> None: artifact_dir = tmp_path / "eval_provider" artifact_dir.mkdir() @@ -139,4 +155,4 @@ def test_collect_eval_rows_accepts_neutral_result_format(tmp_path: Path) -> None assert len(rows) == 1 assert rows[0]["score"] == 1.0 - assert rows[0]["eval_suite"] == "provider_smoke" \ No newline at end of file + assert rows[0]["eval_suite"] == "provider_smoke" From 6ec09bbd883a47688240fb6f77ed99beee27c41c Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:14:50 -0500 Subject: [PATCH 09/24] chore: preserve existing collector test formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：保留结果收集器测试的现有格式，仅添加格式契约测试。 --- utils/test_collect_eval_results.py | 105 +++++++++++++---------------- 1 file changed, 47 insertions(+), 58 deletions(-) diff --git a/utils/test_collect_eval_results.py b/utils/test_collect_eval_results.py index 2c3b54a02a..fa7f9901dc 100644 --- a/utils/test_collect_eval_results.py +++ b/utils/test_collect_eval_results.py @@ -39,27 +39,23 @@ def test_build_row_preserves_explicit_eval_suite() -> None: def _write_lm_eval_result(path: Path, score: float) -> None: - path.write_text( - json.dumps( - { - "lm_eval_version": "0.4.0", - "model_name": "test-model", - "results": { - "gsm8k": { - "exact_match,strict-match": score, - "exact_match_stderr,strict-match": 0.01, - }, - }, - "configs": { - "gsm8k": { - "metric_list": [{"metric": "exact_match"}], - "filter_list": [{"name": "strict-match"}], - }, - }, - "n-samples": {"gsm8k": {"effective": 10}}, - } - ) - ) + path.write_text(json.dumps({ + "lm_eval_version": "0.4.0", + "model_name": "test-model", + "results": { + "gsm8k": { + "exact_match,strict-match": score, + "exact_match_stderr,strict-match": 0.01, + }, + }, + "configs": { + "gsm8k": { + "metric_list": [{"metric": "exact_match"}], + "filter_list": [{"name": "strict-match"}], + }, + }, + "n-samples": {"gsm8k": {"effective": 10}}, + })) def test_collect_eval_rows_expands_batched_concurrencies( @@ -67,31 +63,27 @@ def test_collect_eval_rows_expands_batched_concurrencies( ) -> None: artifact_dir = tmp_path / "eval_batch" artifact_dir.mkdir() - (artifact_dir / "meta_env.json").write_text( - json.dumps( - { - "is_multinode": True, - "infmax_model_prefix": "gptoss", - "hw": "gb200", - "framework": "dynamo-sglang", - "precision": "fp8", - "spec_decoding": "none", - "isl": 8192, - "osl": 1024, - "prefill_tp": 4, - "prefill_ep": 1, - "prefill_num_workers": 1, - "decode_tp": 8, - "decode_ep": 1, - "decode_num_workers": 2, - "eval_concs": [4, 16], - "completed_eval_concs": [4, 16], - "failed_eval_concs": [], - "conc": 4, - "eval_suite": "gsm8k", - } - ) - ) + (artifact_dir / "meta_env.json").write_text(json.dumps({ + "is_multinode": True, + "infmax_model_prefix": "gptoss", + "hw": "gb200", + "framework": "dynamo-sglang", + "precision": "fp8", + "spec_decoding": "none", + "isl": 8192, + "osl": 1024, + "prefill_tp": 4, + "prefill_ep": 1, + "prefill_num_workers": 1, + "decode_tp": 8, + "decode_ep": 1, + "decode_num_workers": 2, + "eval_concs": [4, 16], + "completed_eval_concs": [4, 16], + "failed_eval_concs": [], + "conc": 4, + "eval_suite": "gsm8k", + })) _write_lm_eval_result( artifact_dir / "results_test_conc4.json", 0.90, @@ -113,17 +105,13 @@ def test_collect_eval_rows_ignores_failed_batch_points( ) -> None: artifact_dir = tmp_path / "eval_batch" artifact_dir.mkdir() - (artifact_dir / "meta_env.json").write_text( - json.dumps( - { - "is_multinode": True, - "eval_concs": [4, 16], - "completed_eval_concs": [4], - "failed_eval_concs": [16], - "conc": 4, - } - ) - ) + (artifact_dir / "meta_env.json").write_text(json.dumps({ + "is_multinode": True, + "eval_concs": [4, 16], + "completed_eval_concs": [4], + "failed_eval_concs": [16], + "conc": 4, + })) _write_lm_eval_result( artifact_dir / "results_test_conc4.json", 0.90, @@ -138,6 +126,7 @@ def test_collect_eval_rows_ignores_failed_batch_points( assert [row["conc"] for row in rows] == [4] + def test_collect_eval_rows_accepts_neutral_result_format(tmp_path: Path) -> None: artifact_dir = tmp_path / "eval_provider" artifact_dir.mkdir() @@ -155,4 +144,4 @@ def test_collect_eval_rows_accepts_neutral_result_format(tmp_path: Path) -> None assert len(rows) == 1 assert rows[0]["score"] == 1.0 - assert rows[0]["eval_suite"] == "provider_smoke" + assert rows[0]["eval_suite"] == "provider_smoke" \ No newline at end of file From 2b6e5e8987abe847472e4e932ed7aeeb91260c84 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:21:37 -0500 Subject: [PATCH 10/24] fix: preserve intended verifier sample count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：在启动失败时保留验证器计划样本数，同时将有效样本数记录为零。 --- utils/evals/kimi_vendor_eval.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/utils/evals/kimi_vendor_eval.py b/utils/evals/kimi_vendor_eval.py index f29c621c6a..4fe93fd973 100755 --- a/utils/evals/kimi_vendor_eval.py +++ b/utils/evals/kimi_vendor_eval.py @@ -133,7 +133,12 @@ def _compatibility_result( "filter_list": [{"name": "strict-match"}], } }, - "n-samples": {TASK_NAME: {"original": n_samples, "effective": n_samples}}, + "n-samples": { + TASK_NAME: { + "original": len(EXPECTED_MODES), + "effective": n_samples, + } + }, } if integration_error is not None: result["integration_error"] = { From 6793c9e3f2bdef98040e9d106b948ab2bf564e06 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 11 Aug 2026 00:59:22 -0500 Subject: [PATCH 11/24] fix: preserve configurable eval dispatch behavior MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：保留可配置的 eval 调度行为，并修正失败返回码、Bash 兼容性和 eval 文档。 --- benchmarks/benchmark_lib.sh | 6 +- .../agentic/minimaxm3_fp4_b200_mtp.sh | 2 - .../agentic/minimaxm3_fp4_b300_mtp.sh | 2 - .../agentic/minimaxm3_fp4_mi355x.sh | 7 -- .../agentic/qwen3.5_fp4_b200_sglang_mtp.sh | 3 - .../agentic/qwen3.5_fp4_b300_sglang_mtp.sh | 3 - .../agentic/qwen3.5_fp8_b200_sglang_mtp.sh | 3 - .../agentic/qwen3.5_fp8_b300_sglang_mtp.sh | 3 - docs/eval-agentx-procedures.md | 27 ++++---- docs/eval-agentx-procedures_zh.md | 27 ++++---- utils/evals/EVALS.md | 27 ++++---- utils/evals/test_run_eval_dispatch.py | 68 +++++++++++++++++++ utils/matrix_logic/generate_sweep_configs.py | 20 +++--- 13 files changed, 120 insertions(+), 78 deletions(-) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index a6eb95bd60..6c198ea230 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1240,8 +1240,8 @@ _eval_concs_to_json() { } _env_is_true() { - case "${1,,}" in - 1|true|yes|on) return 0 ;; + case "${1:-}" in + 1|[Tt][Rr][Uu][Ee]|[Yy][Ee][Ss]|[Oo][Nn]) return 0 ;; *) return 1 ;; esac } @@ -1899,7 +1899,7 @@ run_eval() { if [ "$eval_rc" -ne 0 ]; then echo "ERROR: run_eval failed with exit code $eval_rc" >&2 - if [ "${EVAL_ONLY}" = "true" ]; then + if [ "${EVAL_ONLY:-false}" = "true" ]; then echo "Eval-only mode: failing after artifact collection" >&2 return "$eval_rc" fi diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh index be705c0c21..58566be51f 100755 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh @@ -21,8 +21,6 @@ set -x source "$(dirname "$0")/../../benchmark_lib.sh" -export EVAL_FRAMEWORK="lm-eval" - check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh index 3593cac4ce..1b65687c18 100755 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh @@ -7,8 +7,6 @@ set -x source "$(dirname "$0")/../../benchmark_lib.sh" -export EVAL_FRAMEWORK="lm-eval" - check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh index 7faaf69a51..a82924b167 100644 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh @@ -10,13 +10,6 @@ set -x source "$(dirname "$0")/../../benchmark_lib.sh" -# Force the eval framework to lm-eval for this recipe. run_eval derives its -# default as swebench for agentic scenarios (scenario_default=swebench when -# IS_AGENTIC/SCENARIO_TYPE=agentic-coding), but EVAL_FRAMEWORK takes precedence -# over that default (benchmark_lib.sh: framework=${EVAL_FRAMEWORK:-...}), so -# setting it here makes the effective framework always lm-eval, never swebench. -export EVAL_FRAMEWORK="lm-eval" - check_env_vars MODEL TP CONC KV_OFFLOADING KV_OFFLOAD_BACKEND TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION echo "MODEL=$MODEL TP=$TP CONC=$CONC KV_OFFLOADING=$KV_OFFLOADING TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB RESULT_DIR=$RESULT_DIR DURATION=$DURATION EP_SIZE=$EP_SIZE DP_ATTENTION=$DP_ATTENTION" diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh index f0fc1d09da..fc956e14a0 100755 --- a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh @@ -8,9 +8,6 @@ set -x source "$(dirname "$0")/../../benchmark_lib.sh" -# Use the lightweight GSM8K eval instead of the AgentX SWE-bench default. -export EVAL_FRAMEWORK="lm-eval" - check_env_vars \ MODEL TP CONC EP_SIZE KV_OFFLOADING \ TOTAL_CPU_DRAM_GB RESULT_DIR DURATION diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_b300_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_b300_sglang_mtp.sh index a6a0ccf3bd..b43eb06454 100755 --- a/benchmarks/single_node/agentic/qwen3.5_fp4_b300_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp4_b300_sglang_mtp.sh @@ -8,9 +8,6 @@ set -x source "$(dirname "$0")/../../benchmark_lib.sh" -# Use the lightweight GSM8K eval instead of the AgentX SWE-bench default. -export EVAL_FRAMEWORK="lm-eval" - check_env_vars \ MODEL TP CONC EP_SIZE KV_OFFLOADING \ TOTAL_CPU_DRAM_GB RESULT_DIR DURATION diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh index 472b9e4b12..536602459b 100755 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh @@ -8,9 +8,6 @@ set -x source "$(dirname "$0")/../../benchmark_lib.sh" -# Use the lightweight GSM8K eval instead of the AgentX SWE-bench default. -export EVAL_FRAMEWORK="lm-eval" - check_env_vars \ MODEL TP CONC EP_SIZE KV_OFFLOADING \ TOTAL_CPU_DRAM_GB RESULT_DIR DURATION diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang_mtp.sh index 2d0a390250..96583ebe82 100755 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang_mtp.sh @@ -8,9 +8,6 @@ set -x source "$(dirname "$0")/../../benchmark_lib.sh" -# Use the lightweight GSM8K eval instead of the AgentX SWE-bench default. -export EVAL_FRAMEWORK="lm-eval" - check_env_vars \ MODEL TP CONC EP_SIZE KV_OFFLOADING \ TOTAL_CPU_DRAM_GB RESULT_DIR DURATION diff --git a/docs/eval-agentx-procedures.md b/docs/eval-agentx-procedures.md index e5c349fd9c..66d96838cb 100644 --- a/docs/eval-agentx-procedures.md +++ b/docs/eval-agentx-procedures.md @@ -18,11 +18,11 @@ There are two distinct layers: the matrix generator decides **which jobs exist** | Normal sweep | no eval option | Throughput jobs plus the selected 8k/1k eval subset | | Throughput only | `--no-evals` | No eval jobs | | Selected eval subset only | `--evals-only` | Jobs have `RUN_EVAL=true`, `EVAL_ONLY=true` | -| Every eligible eval only | `--all-evals` | Equivalent to `--evals-only --all-evals` and includes all fixed-sequence 8k/1k rows, plus single-node agentic SWE-bench rows | +| Every eligible eval only | `--all-evals` | Equivalent to `--evals-only --all-evals` and includes all fixed-sequence 8k/1k rows plus single-node and multi-node agentic GSM8K rows | | Throughput then eval in one recipe | `RUN_EVAL=true`, `EVAL_ONLY=false` | Server starts, throughput runs, then `run_eval` runs | | Eval against a freshly started server | `RUN_EVAL=true`, `EVAL_ONLY=true` | Launcher expands eval context, skips throughput, and runs the eval | -Default selection is scenario-aware. Single-node fixed-sequence evals use the median and highest eligible concurrency for each 8k/1k model/runner/framework/precision/parallelism group. Multi-node evals use the highest eligible concurrency per topology. Concurrency below 16 is not selected. Agentic evals are opt-in, and multi-node agentic eval is unsupported. See [`mark_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L238-L339) and [`mark_all_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L342-L398). +Default selection is scenario-aware. Single-node fixed-sequence evals use the median and highest eligible concurrency for each 8k/1k model/runner/framework/precision/parallelism group. Multi-node evals use the highest eligible concurrency per topology. Concurrency below 16 is not selected. Agentic evals are opt-in; single-node and multi-node agentic rows select their highest eligible concurrency. See [`mark_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L276-L396) and [`mark_all_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L399-L482). On a PR, combine one primary sweep label (normally `full-sweep-fail-fast`) with eval modifiers. `all-evals` expands coverage without suppressing throughput. `evals-only` suppresses throughput. Together they run all eligible evals only. Runs with `evals-only` are not reusable, while normal full sweeps and `all-evals` full sweeps are reusable. Adding or removing a modifier restarts the active sweep ([label policy](../.github/workflows/README.md#pr-eval-modifiers)). @@ -48,14 +48,14 @@ uv run --no-project --with pydantic --with pyyaml --python 3.12 \ --config-files configs/nvidia-master.yaml | jq . ``` -A correct AgentX eval row contains `"scenario-type": "agentic-coding"`, `"run-eval": true`, and `"eval-only": true`. The workflow splits generated rows into throughput, fixed-sequence eval, and agentic eval jobs in [`.github/workflows/e2e-tests.yml`](../.github/workflows/e2e-tests.yml#L257-L271). +A correct AgentX eval row contains `"scenario-type": "agentic-coding"`, `"run-eval": true`, and `"eval-only": true`. The workflow splits generated rows into throughput, fixed-sequence eval, and agentic eval jobs in [`.github/workflows/e2e-tests.yml`](../.github/workflows/e2e-tests.yml#L278-L293). ## 2. Add a graded eval 1. Add `utils/evals/.yaml` using the lm-evaluation-harness task format. Pin the dataset/split, deterministic generation settings, prompt contract, filters, and primary metric. Use [`gsm8k.yaml`](../utils/evals/gsm8k.yaml) or [`gpqa_diamond.yaml`](../utils/evals/gpqa_diamond.yaml) as an in-tree pattern. 2. Give `task:` a stable name. That exact name is the key used by score thresholds and appears in collected rows. 3. Add the minimum accepted score to [`utils/evals/thresholds.yaml`](../utils/evals/thresholds.yaml). Put a general floor under `default`. Add `models..` only when a justified model-specific floor is required. -4. If the task's primary result is not compatible with the collector's strict/extract/accuracy rules, extend [`extract_lm_metrics()`](../utils/collect_eval_results.py#L114-L181). Do not publish a row whose `score` is null. +4. If the task's primary result is not compatible with the collector's strict/extract/accuracy rules, extend [`extract_lm_metrics()`](../utils/collect_eval_results.py#L115-L197). Do not publish a row whose `score` is null. 5. Run a small explicit slice, inspect samples, then run the full split. `EVAL_LIMIT` is a smoke-test control, not a publishable score setting. Against an already healthy OpenAI-compatible server: @@ -97,9 +97,9 @@ Set `EVAL_ONLY=true` **before server launch**. It is not merely a switch inside 4. Throughput returns immediately or is skipped. 5. `run_eval` and artifact staging run. -Relevant implementation: [context setup](../benchmarks/benchmark_lib.sh#L1049-L1078), [eval dispatch and failure policy](../benchmarks/benchmark_lib.sh#L1737-L1856), and [workflow inputs](../.github/workflows/benchmark-tmpl.yml#L79-L97). +Relevant implementation: [context setup](../benchmarks/benchmark_lib.sh#L1049-L1078), [eval dispatch and failure policy](../benchmarks/benchmark_lib.sh#L1789-L1908), and [workflow inputs](../.github/workflows/benchmark-tmpl.yml#L79-L97). -Do not toggle `EVAL_ONLY` after a throughput-sized server is already running and assume the context changed. Restart through the recipe. In eval-only mode an eval failure is returned after available artifacts are staged. In a workflow, upload happens with `always()` before score validation so failed evidence survives ([single-node upload and gate](../.github/workflows/benchmark-tmpl.yml#L399-L417), [multi-node upload and gate](../.github/workflows/benchmark-multinode-tmpl.yml#L450-L468)). +Do not toggle `EVAL_ONLY` after a throughput-sized server is already running and assume the context changed. Restart through the recipe. In eval-only mode an eval failure is returned after available artifacts are staged. In a workflow, upload happens with `always()` before score validation so failed evidence survives ([single-node upload and gate](../.github/workflows/benchmark-tmpl.yml#L399-L417), [multi-node upload and gate](../.github/workflows/benchmark-multinode-tmpl.yml#L466-L488)). ## 4. Batched eval concurrency @@ -121,9 +121,9 @@ The batch runner creates a fresh temporary output directory per point, stages fi - `completed_eval_concs`: eval and staging both succeeded. - `failed_eval_concs`: either eval or staging failed. -A failed point is deferred so artifacts from every attempted point can upload. The post-upload validator then fails the job. Batched mode accepts positive integers and supports only `lm-eval`. See [`run_eval` batching](../benchmarks/benchmark_lib.sh#L1737-L1832), [artifact suffixing](../benchmarks/benchmark_lib.sh#L1163-L1222), and [manifest validation](../utils/evals/validate_scores.py#L72-L171). +A failed point is deferred so artifacts from every attempted point can upload. The post-upload validator then fails the job. Batched mode accepts positive integers and supports only `lm-eval`. See [`run_eval` batching](../benchmarks/benchmark_lib.sh#L1824-L1885), [artifact suffixing](../benchmarks/benchmark_lib.sh#L1163-L1222), and [manifest validation](../utils/evals/validate_scores.py#L72-L171). -For multi-node `all-evals`, the workflow constructs `EVAL_CONC` by joining the topology's concurrency list ([dispatch](../.github/workflows/e2e-tests.yml#L394-L398)). Never compare a point if its `_conc` result or completed-manifest entry is missing. +For multi-node `all-evals`, the workflow constructs `EVAL_CONC` by joining the topology's concurrency list ([dispatch](../.github/workflows/e2e-tests.yml#L397-L400)). Never compare a point if its `_conc` result or completed-manifest entry is missing. ## 5. Validate scores, not file existence @@ -173,7 +173,7 @@ Retain `meta_env.json`, `results*.json`, and `sample*.jsonl`. Agentic SWE-bench ## 7. Run AgentX: fast feedback versus canonical evidence -AgentX is AIPerf `inferencex-agentx-mvp` trace replay, not a fixed-token synthetic benchmark. The checked-in default uses ten additional warmup requests per trajectory lane and the recipe's configured profile duration. `agentx-fast` forces one warmup request per lane and a 1,200-second profile. It affects single- and multi-node AgentX throughput only. Fixed-sequence throughput and evals remain canonical. Fast runs are not eligible for artifact reuse ([workflow policy](../.github/workflows/README.md#agentx-fast-mode), [fast replay settings](../benchmarks/benchmark_lib.sh#L2026-L2050)). +AgentX is AIPerf `inferencex-agentx-mvp` trace replay, not a fixed-token synthetic benchmark. The checked-in default uses ten additional warmup requests per trajectory lane and the recipe's configured profile duration. `agentx-fast` forces one warmup request per lane and a 1,200-second profile. It affects single- and multi-node AgentX throughput only. Fixed-sequence throughput and evals remain canonical. Fast runs are not eligible for artifact reuse ([workflow policy](../.github/workflows/README.md#agentx-fast-mode), [fast replay settings](../benchmarks/benchmark_lib.sh#L2089-L2113)). Targeted canonical run (configured duration and warmup, with fast and duration overrides omitted): @@ -199,17 +199,18 @@ Targeted AgentX SWE-bench smoke eval (first ten instances, real agentic generati gh workflow run e2e-tests.yml --repo SemiAnalysisAI/InferenceX --ref "$REF" \ -f generate-cli-command='test-config --config-keys qwen3.5-fp8-b200-sglang-agentic --conc 1 --evals-only --config-files configs/nvidia-master.yaml' \ -f test-name='swebench-smoke-qwen35-c1' \ + -f eval-framework=swebench \ -f eval-limit='10' \ -f swebench-gen-mode='agentic' ``` For a publishable SWE-bench score, omit `eval-limit`. Do not use `single-shot`, which is only a debugging escape hatch. SWE-bench generation/scoring controls and its `0.50` full-split threshold are documented next to the implementation in [`utils/evals/EVALS.md`](../utils/evals/EVALS.md#swe-bench-lite---framework-swebench). -Treat fast results as bring-up evidence, never as a replacement for the canonical candidate. A duration below 900 seconds or `AIPERF_UNSAFE_OVERRIDE=true` adds AIPerf's `--unsafe-override` and flags the submission invalid. Use it only for smoke diagnosis ([source](../benchmarks/benchmark_lib.sh#L2188-L2190)). After a fast run is healthy, run the exact candidate canonically before claiming benchmark success. +Treat fast results as bring-up evidence, never as a replacement for the canonical candidate. A duration below 900 seconds or `AIPERF_UNSAFE_OVERRIDE=true` adds AIPerf's `--unsafe-override` and flags the submission invalid. Use it only for smoke diagnosis ([source](../benchmarks/benchmark_lib.sh#L2251-L2253)). After a fast run is healthy, run the exact candidate canonically before claiming benchmark success. ## 8. Preserve trace and run provenance -AgentX defaults to recorded assistant-response replay. Live server outputs are measured but discarded when constructing later turns. Set `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1` only for an explicitly different live-assistant experiment. The selected trace corpus is model-family dependent unless `WEKA_LOADER_OVERRIDE` pins it. The resolver logs both loader and Hugging Face dataset ([trace resolution](../benchmarks/benchmark_lib.sh#L1945-L2024), [replay semantics](../benchmarks/benchmark_lib.sh#L2026-L2192)). +AgentX defaults to recorded assistant-response replay. Live server outputs are measured but discarded when constructing later turns. Set `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1` only for an explicitly different live-assistant experiment. The selected trace corpus is model-family dependent unless `WEKA_LOADER_OVERRIDE` pins it. The resolver logs both loader and Hugging Face dataset ([trace resolution](../benchmarks/benchmark_lib.sh#L2008-L2087), [replay semantics](../benchmarks/benchmark_lib.sh#L2089-L2255)). Capture orchestration provenance immediately: @@ -241,7 +242,7 @@ For each concurrency retain: - server/frontend logs and every metrics endpoint represented. - run URL/ID, attempt, head SHA, recipe/config identity, image, topology, fast flag, and any override. -The runner writes the command before replay and validates raw results after aggregation ([execution path](../benchmarks/benchmark_lib.sh#L2242-L2282)). Aggregation preserves dataset provenance and hardware/model/topology fields ([aggregate construction](../utils/agentic/aggregation/process_agentic_result.py#L194-L272)). Raw workflow uploads intentionally omit very large `inputs.json` and `profile_export_raw.jsonl`. If those are required for an investigation, preserve them from the live allocation before cleanup ([single-node artifact contract](../.github/workflows/benchmark-tmpl.yml#L349-L358), [multi-node contract](../.github/workflows/benchmark-multinode-tmpl.yml#L439-L448)). +The runner writes the command before replay and validates raw results after aggregation ([execution path](../benchmarks/benchmark_lib.sh#L2305-L2345)). Aggregation preserves dataset provenance and hardware/model/topology fields ([aggregate construction](../utils/agentic/aggregation/process_agentic_result.py#L194-L272)). Raw workflow uploads intentionally omit very large `inputs.json` and `profile_export_raw.jsonl`. If those are required for an investigation, preserve them from the live allocation before cleanup ([single-node artifact contract](../.github/workflows/benchmark-tmpl.yml#L349-L358), [multi-node contract](../.github/workflows/benchmark-multinode-tmpl.yml#L455-L464)). ## 9. Debug long AgentX runs from live evidence @@ -290,7 +291,7 @@ curl -fsS '' | \ rg -i 'request|queue|cache|token|prefill|decode|error|fail' ``` -Track trends over repeated samples: running/waiting requests, KV usage, prefix hits, input/output token rates, completed/cancelled/errored requests, frontend routing balance, and disaggregated KV transfer. AIPerf records endpoint identity for every server series ([metrics wiring](../benchmarks/benchmark_lib.sh#L2158-L2181)). +Track trends over repeated samples: running/waiting requests, KV usage, prefix hits, input/output token rates, completed/cancelled/errored requests, frontend routing balance, and disaggregated KV transfer. AIPerf records endpoint identity for every server series ([metrics wiring](../benchmarks/benchmark_lib.sh#L2221-L2245)). Use phase markers, not total Slurm age: diff --git a/docs/eval-agentx-procedures_zh.md b/docs/eval-agentx-procedures_zh.md index 904cc3d1b7..c4c8733e69 100644 --- a/docs/eval-agentx-procedures_zh.md +++ b/docs/eval-agentx-procedures_zh.md @@ -18,11 +18,11 @@ | 常规 sweep | 不加 eval 选项 | 吞吐量作业，加上选定的 8k/1k eval 子集 | | 仅吞吐量 | `--no-evals` | 不生成 eval 作业 | | 仅选定的 eval 子集 | `--evals-only` | 作业带有 `RUN_EVAL=true`、`EVAL_ONLY=true` | -| 仅运行所有符合条件的 eval | `--all-evals` | 等价于 `--evals-only --all-evals`；包含全部定长序列 8k/1k 行，以及单节点 agentic SWE-bench 行 | +| 仅运行所有符合条件的 eval | `--all-evals` | 等价于 `--evals-only --all-evals`；包含全部定长序列 8k/1k 行，以及单节点和多节点 agentic GSM8K 行 | | 在一个 recipe 中先跑吞吐量再跑 eval | `RUN_EVAL=true`、`EVAL_ONLY=false` | 启动服务，运行吞吐量，然后执行 `run_eval` | | 对新启动的服务仅运行 eval | `RUN_EVAL=true`、`EVAL_ONLY=true` | launcher 扩大 eval context，跳过吞吐量并运行 eval | -默认选择会区分场景。单节点定长序列 eval 对每个 8k/1k 的模型/runner/framework/precision/并行配置分组选取符合条件的中位和最高并发；多节点 eval 对每种拓扑选取符合条件的最高并发。低于 16 的并发不会被选中。Agentic eval 需要显式启用；不支持多节点 agentic eval。参见 [`mark_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L238-L339) 与 [`mark_all_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L342-L398)。 +默认选择会区分场景。单节点定长序列 eval 对每个 8k/1k 的模型/runner/framework/precision/并行配置分组选取符合条件的中位和最高并发；多节点 eval 对每种拓扑选取符合条件的最高并发。低于 16 的并发不会被选中。Agentic eval 需要显式启用；单节点和多节点 agentic 行均选择符合条件的最高并发。参见 [`mark_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L276-L396) 与 [`mark_all_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L399-L482)。 在 PR 上，应将一个主要 sweep label（通常为 `full-sweep-fail-fast`）与 eval modifier 组合使用。`all-evals` 在不抑制吞吐量的情况下扩大覆盖范围；`evals-only` 会抑制吞吐量；两者一起使用时只运行所有符合条件的 eval。带有 `evals-only` 的运行不可复用，而常规 full sweep 和 `all-evals` full sweep 可以复用。添加或移除 modifier 会重启当前 sweep（[label 策略](../.github/workflows/README.md#pr-eval-modifiers)）。 @@ -48,14 +48,14 @@ uv run --no-project --with pydantic --with pyyaml --python 3.12 \ --config-files configs/nvidia-master.yaml | jq . ``` -正确的 AgentX eval 行包含 `"scenario-type": "agentic-coding"`、`"run-eval": true` 和 `"eval-only": true`。工作流会在 [`.github/workflows/e2e-tests.yml`](../.github/workflows/e2e-tests.yml#L257-L271) 中将生成的行拆分到吞吐量、定长序列 eval 和 agentic eval 作业。 +正确的 AgentX eval 行包含 `"scenario-type": "agentic-coding"`、`"run-eval": true` 和 `"eval-only": true`。工作流会在 [`.github/workflows/e2e-tests.yml`](../.github/workflows/e2e-tests.yml#L278-L293) 中将生成的行拆分到吞吐量、定长序列 eval 和 agentic eval 作业。 ## 2. 添加评分 eval 1. 按照 lm-evaluation-harness task 格式添加 `utils/evals/.yaml`。固定 dataset/split、确定性生成设置、prompt 约定、filter 和主指标。可参考仓库内的 [`gsm8k.yaml`](../utils/evals/gsm8k.yaml) 或 [`gpqa_diamond.yaml`](../utils/evals/gpqa_diamond.yaml)。 2. 为 `task:` 指定稳定名称。分数阈值以该精确名称为键，收集后的行中也会出现该名称。 3. 在 [`utils/evals/thresholds.yaml`](../utils/evals/thresholds.yaml) 中添加最低可接受分数。通用下限放在 `default`；只有在确有依据需要模型专用下限时，才添加 `models..`。 -4. 如果 task 的主结果与 collector 的 strict/extract/accuracy 规则不兼容，请扩展 [`extract_lm_metrics()`](../utils/collect_eval_results.py#L114-L181)。不要发布 `score` 为 null 的行。 +4. 如果 task 的主结果与 collector 的 strict/extract/accuracy 规则不兼容，请扩展 [`extract_lm_metrics()`](../utils/collect_eval_results.py#L115-L197)。不要发布 `score` 为 null 的行。 5. 先运行一个显式的小切片并检查样本，再运行完整 split。`EVAL_LIMIT` 是 smoke test 控制项，不是可发布分数的运行设置。 对已经健康的 OpenAI-compatible 服务执行： @@ -97,9 +97,9 @@ python3 utils/evals/validate_scores.py --model-prefix "$MODEL_PREFIX" 4. 吞吐量路径立即返回或被跳过。 5. 运行 `run_eval` 和 artifact staging。 -相关实现：[context 设置](../benchmarks/benchmark_lib.sh#L1049-L1078)、[eval 分派与失败策略](../benchmarks/benchmark_lib.sh#L1737-L1856) 和[工作流输入](../.github/workflows/benchmark-tmpl.yml#L79-L97)。 +相关实现：[context 设置](../benchmarks/benchmark_lib.sh#L1049-L1078)、[eval 分派与失败策略](../benchmarks/benchmark_lib.sh#L1789-L1908) 和[工作流输入](../.github/workflows/benchmark-tmpl.yml#L79-L97)。 -不要在吞吐量规格的服务已经运行后才切换 `EVAL_ONLY`，并假定 context 会随之变化。应通过 recipe 重启。Eval-only 模式会在暂存已有 artifact 后返回 eval 失败；在工作流中，上传步骤使用 `always()`，并位于分数校验前，因此失败证据仍会保留（[单节点上传与 gate](../.github/workflows/benchmark-tmpl.yml#L399-L417)、[多节点上传与 gate](../.github/workflows/benchmark-multinode-tmpl.yml#L450-L468)）。 +不要在吞吐量规格的服务已经运行后才切换 `EVAL_ONLY`，并假定 context 会随之变化。应通过 recipe 重启。Eval-only 模式会在暂存已有 artifact 后返回 eval 失败；在工作流中，上传步骤使用 `always()`，并位于分数校验前，因此失败证据仍会保留（[单节点上传与 gate](../.github/workflows/benchmark-tmpl.yml#L399-L417)、[多节点上传与 gate](../.github/workflows/benchmark-multinode-tmpl.yml#L466-L488)）。 ## 4. 批量 eval 并发 @@ -121,9 +121,9 @@ python3 utils/evals/validate_scores.py --expected-concs '16 32 64' - `completed_eval_concs`：eval 与 staging 均成功的点； - `failed_eval_concs`：eval 或 staging 失败的点。 -失败点会延迟报错，使所有已尝试点的 artifact 都能上传；随后 post-upload validator 会使作业失败。批量模式只接受正整数，且仅支持 `lm-eval`。参见 [`run_eval` batching](../benchmarks/benchmark_lib.sh#L1737-L1832)、[artifact 后缀处理](../benchmarks/benchmark_lib.sh#L1163-L1222) 和[manifest 校验](../utils/evals/validate_scores.py#L72-L171)。 +失败点会延迟报错，使所有已尝试点的 artifact 都能上传；随后 post-upload validator 会使作业失败。批量模式只接受正整数，且仅支持 `lm-eval`。参见 [`run_eval` batching](../benchmarks/benchmark_lib.sh#L1824-L1885)、[artifact 后缀处理](../benchmarks/benchmark_lib.sh#L1163-L1222) 和[manifest 校验](../utils/evals/validate_scores.py#L72-L171)。 -对于多节点 `all-evals`，工作流通过连接拓扑的并发列表构造 `EVAL_CONC`（[分派](../.github/workflows/e2e-tests.yml#L394-L398)）。如果缺少某点的 `_conc` 结果或 completed manifest 条目，绝不能比较该点。 +对于多节点 `all-evals`，工作流通过连接拓扑的并发列表构造 `EVAL_CONC`（[分派](../.github/workflows/e2e-tests.yml#L397-L400)）。如果缺少某点的 `_conc` 结果或 completed manifest 条目，绝不能比较该点。 ## 5. 校验分数，而不只是检查文件存在 @@ -173,7 +173,7 @@ gh run download "$RUN_ID" --repo SemiAnalysisAI/InferenceX \ ## 7. 运行 AgentX：快速反馈与 canonical 证据 -AgentX 是 AIPerf `inferencex-agentx-mvp` trace replay，不是固定 token 的合成 benchmark。仓库默认设置对每条 trajectory lane 额外执行十个 warmup 请求，并使用 recipe 配置的 profile 时长。`agentx-fast` 强制每条 lane 只运行一个 warmup 请求，并将 profile 设为 1,200 秒。它只影响单节点和多节点 AgentX 吞吐量；定长序列吞吐量与 eval 保持 canonical。Fast 运行不符合 artifact reuse 条件（[工作流策略](../.github/workflows/README.md#agentx-fast-mode)、[fast replay 设置](../benchmarks/benchmark_lib.sh#L2026-L2050)）。 +AgentX 是 AIPerf `inferencex-agentx-mvp` trace replay，不是固定 token 的合成 benchmark。仓库默认设置对每条 trajectory lane 额外执行十个 warmup 请求，并使用 recipe 配置的 profile 时长。`agentx-fast` 强制每条 lane 只运行一个 warmup 请求，并将 profile 设为 1,200 秒。它只影响单节点和多节点 AgentX 吞吐量；定长序列吞吐量与 eval 保持 canonical。Fast 运行不符合 artifact reuse 条件（[工作流策略](../.github/workflows/README.md#agentx-fast-mode)、[fast replay 设置](../benchmarks/benchmark_lib.sh#L2089-L2113)）。 目标 canonical 运行（使用配置的 duration 和 warmup；不要加 fast 或 duration override）： @@ -199,17 +199,18 @@ gh workflow run e2e-tests.yml --repo SemiAnalysisAI/InferenceX --ref "$REF" \ gh workflow run e2e-tests.yml --repo SemiAnalysisAI/InferenceX --ref "$REF" \ -f generate-cli-command='test-config --config-keys qwen3.5-fp8-b200-sglang-agentic --conc 1 --evals-only --config-files configs/nvidia-master.yaml' \ -f test-name='swebench-smoke-qwen35-c1' \ + -f eval-framework=swebench \ -f eval-limit='10' \ -f swebench-gen-mode='agentic' ``` 要得到可发布的 SWE-bench 分数，省略 `eval-limit`；不要使用 `single-shot`，它只是诊断逃生选项。SWE-bench generation/scoring 控制项以及完整 split 的 `0.50` 阈值在实现旁的 [`utils/evals/EVALS.md`](../utils/evals/EVALS.md#swe-bench-lite---framework-swebench) 中说明。 -Fast 结果只能作为 bring-up 证据，绝不能替代 canonical candidate。小于 900 秒的 duration 或 `AIPERF_UNSAFE_OVERRIDE=true` 会添加 AIPerf 的 `--unsafe-override` 并将 submission 标记为无效；只能用于 smoke 诊断（[源码](../benchmarks/benchmark_lib.sh#L2188-L2190)）。Fast 运行健康后，必须对完全相同的 candidate 进行 canonical 运行，才能宣称 benchmark 成功。 +Fast 结果只能作为 bring-up 证据，绝不能替代 canonical candidate。小于 900 秒的 duration 或 `AIPERF_UNSAFE_OVERRIDE=true` 会添加 AIPerf 的 `--unsafe-override` 并将 submission 标记为无效；只能用于 smoke 诊断（[源码](../benchmarks/benchmark_lib.sh#L2251-L2253)）。Fast 运行健康后，必须对完全相同的 candidate 进行 canonical 运行，才能宣称 benchmark 成功。 ## 8. 保留 trace 与运行 provenance -AgentX 默认 replay 已记录的 assistant response。实时服务输出会被测量，但构造后续 turn 时会丢弃。只有在明确要进行不同的 live-assistant 实验时，才设置 `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1`。除非用 `WEKA_LOADER_OVERRIDE` 固定，否则所选 trace corpus 依赖模型 family；resolver 会同时记录 loader 与 Hugging Face dataset（[trace 解析](../benchmarks/benchmark_lib.sh#L1945-L2024)、[replay 语义](../benchmarks/benchmark_lib.sh#L2026-L2192)）。 +AgentX 默认 replay 已记录的 assistant response。实时服务输出会被测量，但构造后续 turn 时会丢弃。只有在明确要进行不同的 live-assistant 实验时，才设置 `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1`。除非用 `WEKA_LOADER_OVERRIDE` 固定，否则所选 trace corpus 依赖模型 family；resolver 会同时记录 loader 与 Hugging Face dataset（[trace 解析](../benchmarks/benchmark_lib.sh#L2008-L2087)、[replay 语义](../benchmarks/benchmark_lib.sh#L2089-L2255)）。 立即记录 orchestration provenance： @@ -241,7 +242,7 @@ gh run download "$RUN_ID" --repo SemiAnalysisAI/InferenceX \ - server/frontend 日志以及所代表的每个 metrics endpoint； - run URL/ID、attempt、head SHA、recipe/config 标识、image、topology、fast 标志和所有 override。 -Runner 会在 replay 前写入命令，并在聚合后校验原始结果（[执行路径](../benchmarks/benchmark_lib.sh#L2242-L2282)）。聚合会保留 dataset provenance 以及硬件/模型/拓扑字段（[aggregate 构造](../utils/agentic/aggregation/process_agentic_result.py#L194-L272)）。工作流的 raw upload 会有意排除体积很大的 `inputs.json` 和 `profile_export_raw.jsonl`；如果调查需要这些文件，应在清理前从实时 allocation 保存（[单节点 artifact 约定](../.github/workflows/benchmark-tmpl.yml#L349-L358)、[多节点约定](../.github/workflows/benchmark-multinode-tmpl.yml#L439-L448)）。 +Runner 会在 replay 前写入命令，并在聚合后校验原始结果（[执行路径](../benchmarks/benchmark_lib.sh#L2305-L2345)）。聚合会保留 dataset provenance 以及硬件/模型/拓扑字段（[aggregate 构造](../utils/agentic/aggregation/process_agentic_result.py#L194-L272)）。工作流的 raw upload 会有意排除体积很大的 `inputs.json` 和 `profile_export_raw.jsonl`；如果调查需要这些文件，应在清理前从实时 allocation 保存（[单节点 artifact 约定](../.github/workflows/benchmark-tmpl.yml#L349-L358)、[多节点约定](../.github/workflows/benchmark-multinode-tmpl.yml#L455-L464)）。 ## 9. 用实时证据调试长时间 AgentX 运行 @@ -290,7 +291,7 @@ curl -fsS '' | \ rg -i 'request|queue|cache|token|prefill|decode|error|fail' ``` -通过重复 sample 跟踪趋势：running/waiting request、KV usage、prefix hit、input/output token rate、completed/cancelled/errored request、frontend routing balance，以及 disaggregated KV transfer。AIPerf 会为每条 server series 记录 endpoint identity（[metrics 接线](../benchmarks/benchmark_lib.sh#L2158-L2181)）。 +通过重复 sample 跟踪趋势：running/waiting request、KV usage、prefix hit、input/output token rate、completed/cancelled/errored request、frontend routing balance，以及 disaggregated KV transfer。AIPerf 会为每条 server series 记录 endpoint identity（[metrics 接线](../benchmarks/benchmark_lib.sh#L2221-L2245)）。 应使用 phase marker，而不是 Slurm 总运行时间： diff --git a/utils/evals/EVALS.md b/utils/evals/EVALS.md index ce3e6a07b6..f808ce9d88 100644 --- a/utils/evals/EVALS.md +++ b/utils/evals/EVALS.md @@ -11,11 +11,10 @@ from throughput. Selection lives in `mark_eval_entries()` in runner, framework, precision, TP, and decoding configuration. - **Multi-node:** 8k1k only, with one job per parallelism topology at its highest eligible concurrency. Rows differing only by concurrency share a topology. -- **Agentic (SWE-bench), single-node:** highest-conc entry per (model, - runner, framework, precision) group. -- **Agentic (SWE-bench), multi-node:** same policy as multi-node fixed-seq-len - above (highest eligible conc per parallelism topology), since SWE-bench - doesn't support batched concurrencies the way lm-eval does. +- **Agentic (GSM8K), single-node:** highest-conc entry per (model, runner, + framework, precision) group. +- **Agentic (GSM8K), multi-node:** highest eligible concurrency per + parallelism topology. Generator eval modes: @@ -128,14 +127,14 @@ Key eval functions in `benchmarks/benchmark_lib.sh`: | `run_kimi_vendor_eval` | Selects and runs a pinned Kimi Vendor Verifier suite | | `append_lm_eval_summary` | Writes `meta_env.json` and moves eval artifacts to workspace | | `_install_lm_eval_deps` | Installs lm-eval dependencies | -| `_prepare_kimi_vendor_runtime` | Installs the minimal pinned runtime in an isolated temp path | +| `_prepare_kimi_vendor_runtime` | Installs the pinned verifier dependencies in an isolated temp path | | `_prepare_kimi_vendor_verifier` | Fetches a fresh pinned sparse checkout | | `_patch_lm_eval` | Patches lm-eval for reasoning tokens and TRT compatibility | | `compute_eval_context_length` | Computes eval context length (requested benchmark context, capped at model native max) | | `get_native_max_context_length` | Extracts model's native max context length from HF config | ### Single-node -In eval-only mode (`EVAL_ONLY=true`), the benchmark script computes `EVAL_MAX_MODEL_LEN` via `compute_eval_context_length`, starts the server with that context length, skips throughput, and runs lm-eval directly. Each framework wires that context differently (`--context-length` for SGLang, `--max_seq_len` for TRT-LLM). +For default lm-eval jobs in eval-only mode (`EVAL_ONLY=true`), the benchmark script computes `EVAL_MAX_MODEL_LEN` via `compute_eval_context_length`, starts the server with that context length, skips throughput, and runs lm-eval. Each framework wires that context differently (`--context-length` for SGLang, `--max_seq_len` for TRT-LLM). ### Multi-node Multi-node evals support two hardware paths: @@ -143,13 +142,13 @@ Multi-node evals support two hardware paths: **MI355X (AMD)** — `benchmarks/multi_node/amd_utils/server_sglang.sh` - Skips throughput when `EVAL_ONLY=true` - Fixed-seq-len: runs lm-eval via `run_eval --framework lm-eval` against the router on port 30000 -- Agentic-coding (disaggregated, `IS_AGENTIC=1`): runs SWE-bench via `run_eval --port 30000` (no - `--framework` override, same auto-selection as single-node agentic eval-only). Since there's no - single "TP" for a disaggregated topology, and the workflow spells a couple of metadata fields - differently (`PREFILL_DP_ATTN`/`DECODE_DP_ATTN`) than `append_lm_eval_summary` expects - (`PREFILL_DP_ATTENTION`/`DECODE_DP_ATTENTION`), the agentic branch bridges those before calling - `run_eval`; `append_lm_eval_summary` itself runs automatically inside `run_eval()` (same - `EVAL_ONLY=true && IS_AGENTIC` auto-staging as single-node), not as a separate call. +- Agentic-coding (disaggregated, `IS_AGENTIC=1`): follows the same GSM8K/lm-eval path via + `run_eval --framework lm-eval`. Since there's no single "TP" for a disaggregated topology, + and the workflow spells a couple of metadata fields differently + (`PREFILL_DP_ATTN`/`DECODE_DP_ATTN`) than `append_lm_eval_summary` expects + (`PREFILL_DP_ATTENTION`/`DECODE_DP_ATTENTION`), the agentic branch bridges those before + calling `run_eval`; `append_lm_eval_summary` itself runs automatically inside `run_eval()` + (same `EVAL_ONLY=true && IS_AGENTIC` auto-staging as single-node), not as a separate call. - Concurrency uses workflow-provided `EVAL_CONC` when set, otherwise falls back to max of `BENCH_MAX_CONCURRENCY` (x-separated values) - Eval artifacts copied to `/run_logs/slurm_job-*/eval_results/` - `runners/launch_mi355x-amds.sh` skips benchmark result collection when `EVAL_ONLY=true` and uses `find` to locate eval results diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index cad7cb09ad..de98935c8c 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -118,6 +118,30 @@ def test_kimi_vendor_skips_unused_model_context_loading() -> None: assert "UNEXPECTED_CONTEXT_LOAD" not in result.stdout +def test_kimi_failure_preserves_rc_without_eval_only() -> None: + script = r''' +set -u +source "$BENCHMARK_LIB" +run_kimi_vendor_eval() { return 7; } +export EVAL_FRAMEWORK=kimi-vendor +export EVAL_CONCURRENT_REQUESTS="" +export EVAL_MAX_MODEL_LEN=16384 +export IS_AGENTIC=0 +unset EVAL_ONLY +run_eval --port 8888 +''' + result = subprocess.run( + ["bash", "-c", script], + env={**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB)}, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 7 + assert "unbound variable" not in result.stderr + + def test_recipe_lm_eval_arg_still_lm_eval_on_fixed_seqlen(): @@ -509,6 +533,50 @@ def test_summary_metadata_prefers_explicit_suite_then_task_basename( assert explicit["eval_suite"] == "kimi_tool_call_schema" +def test_env_is_true_is_case_insensitive_and_unset_safe() -> None: + script = r''' +set -u +source "$BENCHMARK_LIB" +for value in TrUe yEs oN 1 false 0; do + if _env_is_true "$value"; then + echo true + else + echo false + fi +done +for empty_call in with-argument without-argument; do + if [ "$empty_call" = "with-argument" ]; then + _env_is_true "" + else + _env_is_true + fi + if [ "$?" -eq 0 ]; then + echo true + else + echo false + fi +done +''' + result = subprocess.run( + ["bash", "-c", script], + env={**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB)}, + text=True, + capture_output=True, + check=True, + ) + + assert result.stdout.splitlines() == [ + "true", + "true", + "true", + "true", + "false", + "false", + "false", + "false", + ] + + _MODAL_CREDS_SCRIPT = r''' source "$BENCHMARK_LIB" diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index 5b2dee4108..815eefb01a 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -289,10 +289,9 @@ def mark_eval_entries(matrix_values: list[dict], include_agentic: bool = False) - Single-node: run GSM8K through the same lm-eval path as fixed-sequence 8k1k evals, marking the highest-conc entry per (model, runner, framework, precision) group. - - Multi-node: same policy as the fixed-seq-len multi-node case above - (highest eligible conc per distinct parallelism config, via - eval-conc), using SWE-bench since it doesn't support batched - concurrencies. + - Multi-node: run GSM8K through the same lm-eval path, selecting the + highest eligible concurrency per distinct parallelism config via + eval-conc. """ from collections import defaultdict @@ -360,8 +359,7 @@ def _eligible_eval_concs(entry): ag_sn_groups = defaultdict(list) # Multi-node agentic: same "highest eligible conc per distinct # parallelism config" policy as the fixed-seq-len mn_groups above. - # SWE-bench doesn't support batched concurrencies (unlike lm-eval), - # so exactly one conc is picked per group, never the full list. + # The selected eval subset uses exactly one conc per group. ag_mn_groups = defaultdict(list) for i, entry in enumerate(matrix_values): if entry.get(Fields.SCENARIO_TYPE.value) != 'agentic-coding': @@ -402,12 +400,10 @@ def mark_all_eval_entries(matrix_values: list[dict]) -> list[dict]: Evals only run at 8k1k (matching mark_eval_entries), so entries at other sequence lengths (e.g. 1k1k) are passed through untouched rather than expanded into eval rows. - Single-node agentic entries use GSM8K through the same lm-eval path as - fixed-sequence 8k1k evals. Multi-node agentic entries use SWE-bench, - which doesn't support batched concurrencies (unlike lm-eval): multi-node - agentic rows with the same topology are merged (to recombine any chunking - split), but only the highest resulting conc is marked for eval via - eval-conc, not the full list. + Single- and multi-node agentic entries use GSM8K through lm-eval. + Multi-node agentic rows with the same topology are merged (to recombine + any chunking split), but only the highest resulting conc is marked for + eval via eval-conc, matching the default agentic selection policy. Multi-node fixed-seq-len rows with the same engine topology are merged into one eval row whose full concurrency list is run sequentially against the same engine. From 294e39d51753ca27c9ee99d310a96e8614a68ffb Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:37:45 -0500 Subject: [PATCH 12/24] fix: harden verifier review paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：强化验证器评审路径。 --- .github/workflows/benchmark-tmpl.yml | 2 +- .github/workflows/e2e-tests.yml | 10 ++-- .github/workflows/run-sweep.yml | 12 ++--- benchmarks/benchmark_lib.sh | 13 +++++ .../agentic/glm5.2_fp4_mi355x_sglang_mtp.sh | 1 - utils/collect_eval_results.py | 4 +- utils/evals/EVALS.md | 13 +++-- utils/evals/kimi_vendor_eval.py | 27 +++++++---- utils/evals/test_kimi_vendor_eval.py | 8 ++++ utils/evals/test_run_eval_dispatch.py | 35 ++++++++++++++ utils/test_collect_eval_results.py | 48 ++++++++++++++++++- 11 files changed, 142 insertions(+), 31 deletions(-) diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index afafe2585f..68aeaf7a4a 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -91,7 +91,7 @@ on: required: false default: "lm-eval" eval-suite: - description: "Suite interpreted by the selected eval runner" + description: "Kimi Vendor Verifier suite; leave empty for other eval runners" type: string required: false default: "" diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 1b0aa95cbb..7b483a57b1 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -46,12 +46,12 @@ on: type: string default: "" eval-framework: - description: "Agentic eval runner (lm-eval, swebench, or kimi-vendor)" + description: "Single-node agentic eval runner (lm-eval, swebench, or kimi-vendor)" required: false type: string default: "lm-eval" eval-suite: - description: "Agentic eval suite interpreted by the selected runner" + description: "Single-node Kimi Vendor Verifier suite; empty for other runners" required: false type: string default: "" @@ -136,12 +136,12 @@ on: type: string default: "" eval-framework: - description: "Agentic eval runner (lm-eval, swebench, or kimi-vendor)" + description: "Single-node agentic eval runner (lm-eval, swebench, or kimi-vendor)" required: false type: string default: "lm-eval" eval-suite: - description: "Agentic eval suite interpreted by the selected runner" + description: "Single-node Kimi Vendor Verifier suite; empty for other runners" required: false type: string default: "" @@ -256,7 +256,7 @@ jobs: CMD+=(--evals-only) fi RAW_CONFIG_JSON=$("${CMD[@]}") - CONFIG_JSON=$(python3 -c 'import json,sys; data=json.load(sys.stdin); rows=[row for family in ("single_node","multi_node") for group in data.get(family,{}).values() for row in group]; rows.extend(row for family in ("evals","agentic_evals","multinode_evals") for row in data.get(family,[])); print(json.dumps(rows))' <<<"$RAW_CONFIG_JSON") + CONFIG_JSON=$(python3 -c 'import json,sys; data=json.load(sys.stdin); rows=[row for family in ("single_node","multi_node") for group in data.get(family,{}).values() for row in group]; rows.extend(row for family in ("evals","agentic_evals","multinode_evals","multinode_agentic_evals") for row in data.get(family,[])); print(json.dumps(rows))' <<<"$RAW_CONFIG_JSON") else GENERATE_COMMAND="${{ inputs.generate-cli-command || github.event.inputs.generate-cli-command }}" if [ -z "$GENERATE_COMMAND" ]; then diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index b8e95f0806..62d013934b 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -853,12 +853,12 @@ jobs: eval-only: true eval-conc: ${{ matrix.config['eval-all-concs'] && join(matrix.config.conc, ' ') || matrix.config['eval-conc'] }} - # Multi-node agentic (SWE-bench) eval rows carry the agentic input shape, - # so they are dispatched with sweep-multi-node-agentic's inputs rather - # than sweep-multi-node-evals' fixed-seq-len inputs (isl/osl/max-model-len, - # which agentic rows don't have). SWE-bench doesn't support batched - # concurrencies (unlike lm-eval), so eval-conc is always a single value, - # never the joined-list form sweep-multi-node-evals uses. + # Multi-node agentic GSM8K eval rows carry the agentic input shape, so + # they are dispatched with sweep-multi-node-agentic's inputs rather than + # sweep-multi-node-evals' fixed-seq-len inputs (isl/osl/max-model-len, + # which agentic rows don't have). Agentic selection uses one highest + # eval-conc per topology; fixed-sequence --all-evals rows may instead pass + # a joined concurrency list to lm-eval. sweep-multi-node-agentic-evals: needs: [setup, canary-select, canary-sweep] if: >- diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 6c198ea230..63edf03fe9 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1815,6 +1815,19 @@ run_eval() { local framework="${EVAL_FRAMEWORK:-${cli_framework:-$scenario_default}}" + case "${EVAL_SUITE:-}" in + "") ;; + *[!A-Za-z0-9_.-]*) + echo "ERROR: EVAL_SUITE may contain only letters, digits, '.', '_', and '-'" >&2 + return 2 + ;; + esac + + if [ -n "${EVAL_SUITE:-}" ] && [ "$framework" != "kimi-vendor" ]; then + echo "ERROR: EVAL_SUITE is only supported with EVAL_FRAMEWORK=kimi-vendor" >&2 + return 2 + fi + # Kimi Vendor Verifier uses a fixed request budget and does not consume # EVAL_MAX_MODEL_LEN, so avoid loading model configuration for that path. if [ "$framework" != "kimi-vendor" ] && [ -z "${EVAL_MAX_MODEL_LEN:-}" ]; then diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh index d7ed7a359d..62ec5db23a 100644 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh @@ -4,7 +4,6 @@ set -x source "$(dirname "$0")/../../benchmark_lib.sh" - export EVAL_FRAMEWORK="lm-eval" check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION diff --git a/utils/collect_eval_results.py b/utils/collect_eval_results.py index 1070a9305e..28d9ab11c9 100644 --- a/utils/collect_eval_results.py +++ b/utils/collect_eval_results.py @@ -408,7 +408,7 @@ def main(): f"{pct(r['score'])}{se(r['score_se'])}", f"{pct(r['em_strict'])}{se(r['em_strict_se'])}", f"{pct(r['em_flexible'])}{se(r['em_flexible_se'])}", - r['n_eff'] or '', + r['n_eff'] if r['n_eff'] is not None else '', r['model'], ] for r in single_node_rows @@ -446,7 +446,7 @@ def main(): f"{pct(r['score'])}{se(r['score_se'])}", f"{pct(r['em_strict'])}{se(r['em_strict_se'])}", f"{pct(r['em_flexible'])}{se(r['em_flexible_se'])}", - r['n_eff'] or '', + r['n_eff'] if r['n_eff'] is not None else '', r['model'], ] for r in multinode_rows diff --git a/utils/evals/EVALS.md b/utils/evals/EVALS.md index f808ce9d88..58237912f0 100644 --- a/utils/evals/EVALS.md +++ b/utils/evals/EVALS.md @@ -50,11 +50,16 @@ The default eval framework is [lm-evaluation-harness](https://github.com/Eleuthe The Phase 1 Kimi smoke is opt-in and single-node only. Select `eval-framework: kimi-vendor` and `eval-suite: kimi_tool_call_schema` on -`e2e-tests.yml`, or invoke it after a server is ready: +`e2e-tests.yml`, or invoke it from the repository root after a server is ready: ```bash -EVAL_FRAMEWORK=kimi-vendor EVAL_SUITE=kimi_tool_call_schema \ - run_eval --port "$PORT" +source benchmarks/benchmark_lib.sh +export EVAL_FRAMEWORK=kimi-vendor +export EVAL_SUITE=kimi_tool_call_schema +export EVAL_RESULT_DIR="$(mktemp -d /tmp/eval_out-XXXXXX)" +run_eval --port "$PORT" +append_lm_eval_summary +python3 utils/evals/validate_scores.py ``` The framework selects a provider-specific subprocess adapter, while the suite @@ -218,7 +223,7 @@ cat ./evals/agg_eval_all.json | jq '[.[] | select(.hw == "B200")]' | `RUN_EVAL` | `false` | Enable eval after throughput benchmark | | `EVAL_ONLY` | `false` | Skip throughput, only run evals (set by workflow) | | `EVAL_FRAMEWORK` | `lm-eval` | Eval runner (`lm-eval`, `swebench`, or `kimi-vendor`) | -| `EVAL_SUITE` | basename of `EVAL_TASKS_DIR`, else `gsm8k` | Runner-specific suite selector and artifact identity; the workflow `eval-suite` input sets it explicitly | +| `EVAL_SUITE` | basename of `EVAL_TASKS_DIR`, else `gsm8k` | Provider suite selector and artifact identity. External override is currently supported only by `kimi-vendor`; other runners derive it from their task | | `EVAL_TASKS_DIR` | `utils/evals/gsm8k.yaml` | Path to lm-eval task YAML | | `EVAL_RESULT_DIR` | `/tmp/eval_out-*` | Output directory for eval results | | `EVAL_MAX_MODEL_LEN` | `16384` | Max context for eval (set by `compute_eval_context_length`) | diff --git a/utils/evals/kimi_vendor_eval.py b/utils/evals/kimi_vendor_eval.py index 4fe93fd973..4f3debc6fb 100755 --- a/utils/evals/kimi_vendor_eval.py +++ b/utils/evals/kimi_vendor_eval.py @@ -74,16 +74,19 @@ def _project_report(model: str, report: Any) -> tuple[dict[str, Any], bool]: total = summary.get("total") by_status = _mapping(summary.get("by_status"), "report.summary.by_status") - passed = by_status.get("passed", 0) - if ( - not isinstance(total, int) - or isinstance(total, bool) - or not isinstance(passed, int) - or isinstance(passed, bool) - or passed < 0 - or passed > 2 - ): + if not isinstance(total, int) or isinstance(total, bool): raise ValueError("report summary contains invalid counts") + for status, count in by_status.items(): + if ( + status not in {"passed", "failed"} + or not isinstance(count, int) + or isinstance(count, bool) + or count < 0 + ): + raise ValueError("report summary contains invalid counts") + if sum(by_status.values()) != total: + raise ValueError("report summary does not match total") + passed = by_status.get("passed", 0) modes: list[str] = [] result_passes = 0 @@ -91,7 +94,11 @@ def _project_report(model: str, report: Any) -> tuple[dict[str, Any], bool]: record = _mapping(result, f"report.results[{index}]") mode = record.get("mode") status = record.get("status") - if not isinstance(mode, str) or not isinstance(status, str): + if ( + not isinstance(mode, str) + or not isinstance(status, str) + or status not in {"passed", "failed"} + ): raise ValueError(f"report.results[{index}] has invalid mode or status") modes.append(mode) result_passes += status == "passed" diff --git a/utils/evals/test_kimi_vendor_eval.py b/utils/evals/test_kimi_vendor_eval.py index fe8394efc8..e133d3b735 100644 --- a/utils/evals/test_kimi_vendor_eval.py +++ b/utils/evals/test_kimi_vendor_eval.py @@ -25,6 +25,12 @@ def _report(stream_status: str = "passed") -> dict[str, Any]: } +def _report_with_inconsistent_counts() -> dict[str, Any]: + report = _report("failed") + report["summary"]["by_status"]["failed"] = 2 + return report + + def _result(output_dir: Path) -> dict[str, Any]: paths = list(output_dir.glob(kve.COMPATIBILITY_GLOB)) assert len(paths) == 1 @@ -134,6 +140,8 @@ def fake_run( (None, "FileNotFoundError"), ("{bad-json", "JSONDecodeError"), (OSError("boom"), "OSError"), + (json.dumps(_report("skipped")), "ValueError"), + (json.dumps(_report_with_inconsistent_counts()), "ValueError"), ( subprocess.TimeoutExpired("pytest", kve.DEFAULT_TIMEOUT_SECONDS), "TimeoutExpired", diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index de98935c8c..0f20df2934 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -170,6 +170,24 @@ def test_run_eval_rejects_missing_framework_value(): assert "--framework requires a value" in result.stderr +def test_run_eval_rejects_unsafe_suite_name() -> None: + result = _run_invalid_call( + "EVAL_SUITE='kimi\"suite' run_eval --framework kimi-vendor" + ) + + assert result.returncode == 2 + assert "EVAL_SUITE may contain only" in result.stderr + + +def test_run_eval_rejects_suite_override_for_lm_eval() -> None: + result = _run_invalid_call( + "EVAL_SUITE=gpqa_diamond run_eval --framework lm-eval" + ) + + assert result.returncode == 2 + assert "only supported with EVAL_FRAMEWORK=kimi-vendor" in result.stderr + + def test_kimi_vendor_rejects_batched_concurrency() -> None: result = _run_invalid_call( "EVAL_MAX_MODEL_LEN=16384 " @@ -1081,3 +1099,20 @@ def test_agentic_eval_workflow_forwards_runner_contract() -> None: assert forwarded["spec-decoding"] == "${{ matrix.config.spec-decoding }}" assert forwarded["eval-framework"] == "${{ inputs.eval-framework }}" assert forwarded["eval-suite"] == "${{ inputs.eval-suite }}" + + + +def test_trusted_changelog_matrix_keeps_multinode_agentic_evals() -> None: + workflow = yaml.safe_load(E2E_WORKFLOW.read_text()) + get_jobs = next( + step + for step in workflow["jobs"]["get-jobs"]["steps"] + if step.get("id") == "get-jobs" + ) + flatten_command = next( + line + for line in get_jobs["run"].splitlines() + if "rows.extend" in line + ) + + assert '"multinode_agentic_evals"' in flatten_command \ No newline at end of file diff --git a/utils/test_collect_eval_results.py b/utils/test_collect_eval_results.py index fa7f9901dc..41cd5cf3d0 100644 --- a/utils/test_collect_eval_results.py +++ b/utils/test_collect_eval_results.py @@ -1,9 +1,15 @@ """Tests for eval result aggregation.""" import json +import sys from pathlib import Path -from collect_eval_results import EVAL_RESULT_FORMAT, build_row, collect_eval_rows +from collect_eval_results import ( + EVAL_RESULT_FORMAT, + build_row, + collect_eval_rows, + main as collect_main, +) from evals.kimi_vendor_eval import RESULT_FORMAT as KIMI_VENDOR_RESULT_FORMAT @@ -144,4 +150,42 @@ def test_collect_eval_rows_accepts_neutral_result_format(tmp_path: Path) -> None assert len(rows) == 1 assert rows[0]["score"] == 1.0 - assert rows[0]["eval_suite"] == "provider_smoke" \ No newline at end of file + assert rows[0]["eval_suite"] == "provider_smoke" + + +def test_main_renders_zero_effective_samples( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + for name, is_multinode in (("single", False), ("multi", True)): + artifact_dir = tmp_path / f"eval_{name}" + artifact_dir.mkdir() + (artifact_dir / "meta_env.json").write_text(json.dumps({ + "is_multinode": is_multinode, + "eval_suite": "gsm8k", + })) + result_path = artifact_dir / f"results_{name}.json" + _write_lm_eval_result(result_path, 0.0) + result = json.loads(result_path.read_text()) + result["n-samples"]["gsm8k"]["effective"] = 0 + result_path.write_text(json.dumps(result)) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + sys, + "argv", + ["collect_eval_results.py", str(tmp_path), "zero-samples"], + ) + + collect_main() + + task_rows = [ + line + for line in capsys.readouterr().out.splitlines() + if "| gsm8k " in line + ] + assert len(task_rows) == 2 + for row in task_rows: + cells = [cell.strip() for cell in row.split("|")[1:-1]] + assert cells[-2] == "0" \ No newline at end of file From ce37f468c03ad2c4f65c808b544a5f2cd45667f8 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:36:29 -0500 Subject: [PATCH 13/24] fix: scope eval state and links MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：限定评估套件状态的作用域，并修正双语 AgentX 文档中的源码行号链接。 --- benchmarks/benchmark_lib.sh | 2 ++ docs/eval-agentx-procedures.md | 14 ++++++------ docs/eval-agentx-procedures_zh.md | 14 ++++++------ utils/evals/test_run_eval_dispatch.py | 33 +++++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 14 deletions(-) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 63edf03fe9..0043fb901b 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1789,6 +1789,8 @@ run_swebench_eval() { run_eval() { local cli_framework="" local forwarded=() + # Keep runner-selected suite identity scoped to this invocation. + local EVAL_SUITE="${EVAL_SUITE:-}" while [[ $# -gt 0 ]]; do case "$1" in diff --git a/docs/eval-agentx-procedures.md b/docs/eval-agentx-procedures.md index 66d96838cb..263927d908 100644 --- a/docs/eval-agentx-procedures.md +++ b/docs/eval-agentx-procedures.md @@ -97,7 +97,7 @@ Set `EVAL_ONLY=true` **before server launch**. It is not merely a switch inside 4. Throughput returns immediately or is skipped. 5. `run_eval` and artifact staging run. -Relevant implementation: [context setup](../benchmarks/benchmark_lib.sh#L1049-L1078), [eval dispatch and failure policy](../benchmarks/benchmark_lib.sh#L1789-L1908), and [workflow inputs](../.github/workflows/benchmark-tmpl.yml#L79-L97). +Relevant implementation: [context setup](../benchmarks/benchmark_lib.sh#L1049-L1078), [eval dispatch and failure policy](../benchmarks/benchmark_lib.sh#L1789-L1923), and [workflow inputs](../.github/workflows/benchmark-tmpl.yml#L79-L97). Do not toggle `EVAL_ONLY` after a throughput-sized server is already running and assume the context changed. Restart through the recipe. In eval-only mode an eval failure is returned after available artifacts are staged. In a workflow, upload happens with `always()` before score validation so failed evidence survives ([single-node upload and gate](../.github/workflows/benchmark-tmpl.yml#L399-L417), [multi-node upload and gate](../.github/workflows/benchmark-multinode-tmpl.yml#L466-L488)). @@ -121,7 +121,7 @@ The batch runner creates a fresh temporary output directory per point, stages fi - `completed_eval_concs`: eval and staging both succeeded. - `failed_eval_concs`: either eval or staging failed. -A failed point is deferred so artifacts from every attempted point can upload. The post-upload validator then fails the job. Batched mode accepts positive integers and supports only `lm-eval`. See [`run_eval` batching](../benchmarks/benchmark_lib.sh#L1824-L1885), [artifact suffixing](../benchmarks/benchmark_lib.sh#L1163-L1222), and [manifest validation](../utils/evals/validate_scores.py#L72-L171). +A failed point is deferred so artifacts from every attempted point can upload. The post-upload validator then fails the job. Batched mode accepts positive integers and supports only `lm-eval`. See [`run_eval` batching](../benchmarks/benchmark_lib.sh#L1839-L1900), [artifact suffixing](../benchmarks/benchmark_lib.sh#L1163-L1222), and [manifest validation](../utils/evals/validate_scores.py#L72-L171). For multi-node `all-evals`, the workflow constructs `EVAL_CONC` by joining the topology's concurrency list ([dispatch](../.github/workflows/e2e-tests.yml#L397-L400)). Never compare a point if its `_conc` result or completed-manifest entry is missing. @@ -173,7 +173,7 @@ Retain `meta_env.json`, `results*.json`, and `sample*.jsonl`. Agentic SWE-bench ## 7. Run AgentX: fast feedback versus canonical evidence -AgentX is AIPerf `inferencex-agentx-mvp` trace replay, not a fixed-token synthetic benchmark. The checked-in default uses ten additional warmup requests per trajectory lane and the recipe's configured profile duration. `agentx-fast` forces one warmup request per lane and a 1,200-second profile. It affects single- and multi-node AgentX throughput only. Fixed-sequence throughput and evals remain canonical. Fast runs are not eligible for artifact reuse ([workflow policy](../.github/workflows/README.md#agentx-fast-mode), [fast replay settings](../benchmarks/benchmark_lib.sh#L2089-L2113)). +AgentX is AIPerf `inferencex-agentx-mvp` trace replay, not a fixed-token synthetic benchmark. The checked-in default uses ten additional warmup requests per trajectory lane and the recipe's configured profile duration. `agentx-fast` forces one warmup request per lane and a 1,200-second profile. It affects single- and multi-node AgentX throughput only. Fixed-sequence throughput and evals remain canonical. Fast runs are not eligible for artifact reuse ([workflow policy](../.github/workflows/README.md#agentx-fast-mode), [fast replay settings](../benchmarks/benchmark_lib.sh#L2104-L2128)). Targeted canonical run (configured duration and warmup, with fast and duration overrides omitted): @@ -206,11 +206,11 @@ gh workflow run e2e-tests.yml --repo SemiAnalysisAI/InferenceX --ref "$REF" \ For a publishable SWE-bench score, omit `eval-limit`. Do not use `single-shot`, which is only a debugging escape hatch. SWE-bench generation/scoring controls and its `0.50` full-split threshold are documented next to the implementation in [`utils/evals/EVALS.md`](../utils/evals/EVALS.md#swe-bench-lite---framework-swebench). -Treat fast results as bring-up evidence, never as a replacement for the canonical candidate. A duration below 900 seconds or `AIPERF_UNSAFE_OVERRIDE=true` adds AIPerf's `--unsafe-override` and flags the submission invalid. Use it only for smoke diagnosis ([source](../benchmarks/benchmark_lib.sh#L2251-L2253)). After a fast run is healthy, run the exact candidate canonically before claiming benchmark success. +Treat fast results as bring-up evidence, never as a replacement for the canonical candidate. A duration below 900 seconds or `AIPERF_UNSAFE_OVERRIDE=true` adds AIPerf's `--unsafe-override` and flags the submission invalid. Use it only for smoke diagnosis ([source](../benchmarks/benchmark_lib.sh#L2266-L2268)). After a fast run is healthy, run the exact candidate canonically before claiming benchmark success. ## 8. Preserve trace and run provenance -AgentX defaults to recorded assistant-response replay. Live server outputs are measured but discarded when constructing later turns. Set `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1` only for an explicitly different live-assistant experiment. The selected trace corpus is model-family dependent unless `WEKA_LOADER_OVERRIDE` pins it. The resolver logs both loader and Hugging Face dataset ([trace resolution](../benchmarks/benchmark_lib.sh#L2008-L2087), [replay semantics](../benchmarks/benchmark_lib.sh#L2089-L2255)). +AgentX defaults to recorded assistant-response replay. Live server outputs are measured but discarded when constructing later turns. Set `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1` only for an explicitly different live-assistant experiment. The selected trace corpus is model-family dependent unless `WEKA_LOADER_OVERRIDE` pins it. The resolver logs both loader and Hugging Face dataset ([trace resolution](../benchmarks/benchmark_lib.sh#L2023-L2102), [replay semantics](../benchmarks/benchmark_lib.sh#L2104-L2270)). Capture orchestration provenance immediately: @@ -242,7 +242,7 @@ For each concurrency retain: - server/frontend logs and every metrics endpoint represented. - run URL/ID, attempt, head SHA, recipe/config identity, image, topology, fast flag, and any override. -The runner writes the command before replay and validates raw results after aggregation ([execution path](../benchmarks/benchmark_lib.sh#L2305-L2345)). Aggregation preserves dataset provenance and hardware/model/topology fields ([aggregate construction](../utils/agentic/aggregation/process_agentic_result.py#L194-L272)). Raw workflow uploads intentionally omit very large `inputs.json` and `profile_export_raw.jsonl`. If those are required for an investigation, preserve them from the live allocation before cleanup ([single-node artifact contract](../.github/workflows/benchmark-tmpl.yml#L349-L358), [multi-node contract](../.github/workflows/benchmark-multinode-tmpl.yml#L455-L464)). +The runner writes the command before replay and validates raw results after aggregation ([execution path](../benchmarks/benchmark_lib.sh#L2320-L2360)). Aggregation preserves dataset provenance and hardware/model/topology fields ([aggregate construction](../utils/agentic/aggregation/process_agentic_result.py#L194-L272)). Raw workflow uploads intentionally omit very large `inputs.json` and `profile_export_raw.jsonl`. If those are required for an investigation, preserve them from the live allocation before cleanup ([single-node artifact contract](../.github/workflows/benchmark-tmpl.yml#L349-L358), [multi-node contract](../.github/workflows/benchmark-multinode-tmpl.yml#L455-L464)). ## 9. Debug long AgentX runs from live evidence @@ -291,7 +291,7 @@ curl -fsS '' | \ rg -i 'request|queue|cache|token|prefill|decode|error|fail' ``` -Track trends over repeated samples: running/waiting requests, KV usage, prefix hits, input/output token rates, completed/cancelled/errored requests, frontend routing balance, and disaggregated KV transfer. AIPerf records endpoint identity for every server series ([metrics wiring](../benchmarks/benchmark_lib.sh#L2221-L2245)). +Track trends over repeated samples: running/waiting requests, KV usage, prefix hits, input/output token rates, completed/cancelled/errored requests, frontend routing balance, and disaggregated KV transfer. AIPerf records endpoint identity for every server series ([metrics wiring](../benchmarks/benchmark_lib.sh#L2236-L2260)). Use phase markers, not total Slurm age: diff --git a/docs/eval-agentx-procedures_zh.md b/docs/eval-agentx-procedures_zh.md index c4c8733e69..66c205782e 100644 --- a/docs/eval-agentx-procedures_zh.md +++ b/docs/eval-agentx-procedures_zh.md @@ -97,7 +97,7 @@ python3 utils/evals/validate_scores.py --model-prefix "$MODEL_PREFIX" 4. 吞吐量路径立即返回或被跳过。 5. 运行 `run_eval` 和 artifact staging。 -相关实现：[context 设置](../benchmarks/benchmark_lib.sh#L1049-L1078)、[eval 分派与失败策略](../benchmarks/benchmark_lib.sh#L1789-L1908) 和[工作流输入](../.github/workflows/benchmark-tmpl.yml#L79-L97)。 +相关实现：[context 设置](../benchmarks/benchmark_lib.sh#L1049-L1078)、[eval 分派与失败策略](../benchmarks/benchmark_lib.sh#L1789-L1923) 和[工作流输入](../.github/workflows/benchmark-tmpl.yml#L79-L97)。 不要在吞吐量规格的服务已经运行后才切换 `EVAL_ONLY`，并假定 context 会随之变化。应通过 recipe 重启。Eval-only 模式会在暂存已有 artifact 后返回 eval 失败；在工作流中，上传步骤使用 `always()`，并位于分数校验前，因此失败证据仍会保留（[单节点上传与 gate](../.github/workflows/benchmark-tmpl.yml#L399-L417)、[多节点上传与 gate](../.github/workflows/benchmark-multinode-tmpl.yml#L466-L488)）。 @@ -121,7 +121,7 @@ python3 utils/evals/validate_scores.py --expected-concs '16 32 64' - `completed_eval_concs`：eval 与 staging 均成功的点； - `failed_eval_concs`：eval 或 staging 失败的点。 -失败点会延迟报错，使所有已尝试点的 artifact 都能上传；随后 post-upload validator 会使作业失败。批量模式只接受正整数，且仅支持 `lm-eval`。参见 [`run_eval` batching](../benchmarks/benchmark_lib.sh#L1824-L1885)、[artifact 后缀处理](../benchmarks/benchmark_lib.sh#L1163-L1222) 和[manifest 校验](../utils/evals/validate_scores.py#L72-L171)。 +失败点会延迟报错，使所有已尝试点的 artifact 都能上传；随后 post-upload validator 会使作业失败。批量模式只接受正整数，且仅支持 `lm-eval`。参见 [`run_eval` batching](../benchmarks/benchmark_lib.sh#L1839-L1900)、[artifact 后缀处理](../benchmarks/benchmark_lib.sh#L1163-L1222) 和[manifest 校验](../utils/evals/validate_scores.py#L72-L171)。 对于多节点 `all-evals`，工作流通过连接拓扑的并发列表构造 `EVAL_CONC`（[分派](../.github/workflows/e2e-tests.yml#L397-L400)）。如果缺少某点的 `_conc` 结果或 completed manifest 条目，绝不能比较该点。 @@ -173,7 +173,7 @@ gh run download "$RUN_ID" --repo SemiAnalysisAI/InferenceX \ ## 7. 运行 AgentX：快速反馈与 canonical 证据 -AgentX 是 AIPerf `inferencex-agentx-mvp` trace replay，不是固定 token 的合成 benchmark。仓库默认设置对每条 trajectory lane 额外执行十个 warmup 请求，并使用 recipe 配置的 profile 时长。`agentx-fast` 强制每条 lane 只运行一个 warmup 请求，并将 profile 设为 1,200 秒。它只影响单节点和多节点 AgentX 吞吐量；定长序列吞吐量与 eval 保持 canonical。Fast 运行不符合 artifact reuse 条件（[工作流策略](../.github/workflows/README.md#agentx-fast-mode)、[fast replay 设置](../benchmarks/benchmark_lib.sh#L2089-L2113)）。 +AgentX 是 AIPerf `inferencex-agentx-mvp` trace replay，不是固定 token 的合成 benchmark。仓库默认设置对每条 trajectory lane 额外执行十个 warmup 请求，并使用 recipe 配置的 profile 时长。`agentx-fast` 强制每条 lane 只运行一个 warmup 请求，并将 profile 设为 1,200 秒。它只影响单节点和多节点 AgentX 吞吐量；定长序列吞吐量与 eval 保持 canonical。Fast 运行不符合 artifact reuse 条件（[工作流策略](../.github/workflows/README.md#agentx-fast-mode)、[fast replay 设置](../benchmarks/benchmark_lib.sh#L2104-L2128)）。 目标 canonical 运行（使用配置的 duration 和 warmup；不要加 fast 或 duration override）： @@ -206,11 +206,11 @@ gh workflow run e2e-tests.yml --repo SemiAnalysisAI/InferenceX --ref "$REF" \ 要得到可发布的 SWE-bench 分数，省略 `eval-limit`；不要使用 `single-shot`，它只是诊断逃生选项。SWE-bench generation/scoring 控制项以及完整 split 的 `0.50` 阈值在实现旁的 [`utils/evals/EVALS.md`](../utils/evals/EVALS.md#swe-bench-lite---framework-swebench) 中说明。 -Fast 结果只能作为 bring-up 证据，绝不能替代 canonical candidate。小于 900 秒的 duration 或 `AIPERF_UNSAFE_OVERRIDE=true` 会添加 AIPerf 的 `--unsafe-override` 并将 submission 标记为无效；只能用于 smoke 诊断（[源码](../benchmarks/benchmark_lib.sh#L2251-L2253)）。Fast 运行健康后，必须对完全相同的 candidate 进行 canonical 运行，才能宣称 benchmark 成功。 +Fast 结果只能作为 bring-up 证据，绝不能替代 canonical candidate。小于 900 秒的 duration 或 `AIPERF_UNSAFE_OVERRIDE=true` 会添加 AIPerf 的 `--unsafe-override` 并将 submission 标记为无效；只能用于 smoke 诊断（[源码](../benchmarks/benchmark_lib.sh#L2266-L2268)）。Fast 运行健康后，必须对完全相同的 candidate 进行 canonical 运行，才能宣称 benchmark 成功。 ## 8. 保留 trace 与运行 provenance -AgentX 默认 replay 已记录的 assistant response。实时服务输出会被测量，但构造后续 turn 时会丢弃。只有在明确要进行不同的 live-assistant 实验时，才设置 `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1`。除非用 `WEKA_LOADER_OVERRIDE` 固定，否则所选 trace corpus 依赖模型 family；resolver 会同时记录 loader 与 Hugging Face dataset（[trace 解析](../benchmarks/benchmark_lib.sh#L2008-L2087)、[replay 语义](../benchmarks/benchmark_lib.sh#L2089-L2255)）。 +AgentX 默认 replay 已记录的 assistant response。实时服务输出会被测量，但构造后续 turn 时会丢弃。只有在明确要进行不同的 live-assistant 实验时，才设置 `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1`。除非用 `WEKA_LOADER_OVERRIDE` 固定，否则所选 trace corpus 依赖模型 family；resolver 会同时记录 loader 与 Hugging Face dataset（[trace 解析](../benchmarks/benchmark_lib.sh#L2023-L2102)、[replay 语义](../benchmarks/benchmark_lib.sh#L2104-L2270)）。 立即记录 orchestration provenance： @@ -242,7 +242,7 @@ gh run download "$RUN_ID" --repo SemiAnalysisAI/InferenceX \ - server/frontend 日志以及所代表的每个 metrics endpoint； - run URL/ID、attempt、head SHA、recipe/config 标识、image、topology、fast 标志和所有 override。 -Runner 会在 replay 前写入命令，并在聚合后校验原始结果（[执行路径](../benchmarks/benchmark_lib.sh#L2305-L2345)）。聚合会保留 dataset provenance 以及硬件/模型/拓扑字段（[aggregate 构造](../utils/agentic/aggregation/process_agentic_result.py#L194-L272)）。工作流的 raw upload 会有意排除体积很大的 `inputs.json` 和 `profile_export_raw.jsonl`；如果调查需要这些文件，应在清理前从实时 allocation 保存（[单节点 artifact 约定](../.github/workflows/benchmark-tmpl.yml#L349-L358)、[多节点约定](../.github/workflows/benchmark-multinode-tmpl.yml#L455-L464)）。 +Runner 会在 replay 前写入命令，并在聚合后校验原始结果（[执行路径](../benchmarks/benchmark_lib.sh#L2320-L2360)）。聚合会保留 dataset provenance 以及硬件/模型/拓扑字段（[aggregate 构造](../utils/agentic/aggregation/process_agentic_result.py#L194-L272)）。工作流的 raw upload 会有意排除体积很大的 `inputs.json` 和 `profile_export_raw.jsonl`；如果调查需要这些文件，应在清理前从实时 allocation 保存（[单节点 artifact 约定](../.github/workflows/benchmark-tmpl.yml#L349-L358)、[多节点约定](../.github/workflows/benchmark-multinode-tmpl.yml#L455-L464)）。 ## 9. 用实时证据调试长时间 AgentX 运行 @@ -291,7 +291,7 @@ curl -fsS '' | \ rg -i 'request|queue|cache|token|prefill|decode|error|fail' ``` -通过重复 sample 跟踪趋势：running/waiting request、KV usage、prefix hit、input/output token rate、completed/cancelled/errored request、frontend routing balance，以及 disaggregated KV transfer。AIPerf 会为每条 server series 记录 endpoint identity（[metrics 接线](../benchmarks/benchmark_lib.sh#L2221-L2245)）。 +通过重复 sample 跟踪趋势：running/waiting request、KV usage、prefix hit、input/output token rate、completed/cancelled/errored request、frontend routing balance，以及 disaggregated KV transfer。AIPerf 会为每条 server series 记录 endpoint identity（[metrics 接线](../benchmarks/benchmark_lib.sh#L2236-L2260)）。 应使用 phase marker，而不是 Slurm 总运行时间： diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index 0f20df2934..fae4aab2de 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -188,6 +188,39 @@ def test_run_eval_rejects_suite_override_for_lm_eval() -> None: assert "only supported with EVAL_FRAMEWORK=kimi-vendor" in result.stderr +def test_run_eval_scopes_runner_selected_suite_to_one_call() -> None: + script = r''' +source "$BENCHMARK_LIB" +run_kimi_vendor_eval() { + export EVAL_SUITE=kimi_tool_call_schema + echo "DISPATCH=kimi-vendor SUITE=$EVAL_SUITE" +} +run_lm_eval() { echo "DISPATCH=lm-eval SUITE=${EVAL_SUITE:-unset}"; } +export EVAL_MAX_MODEL_LEN=16384 +export EVAL_CONCURRENT_REQUESTS="" +export EVAL_ONLY=false +export IS_AGENTIC=0 +unset EVAL_SUITE +export EVAL_FRAMEWORK=kimi-vendor +run_eval --port 8888 +export EVAL_FRAMEWORK=lm-eval +run_eval --port 8888 +printf 'FINAL_SUITE=%s\n' "${EVAL_SUITE-unset}" +''' + result = subprocess.run( + ["bash", "-c", script], + env={**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB)}, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert "DISPATCH=kimi-vendor SUITE=kimi_tool_call_schema" in result.stdout + assert "DISPATCH=lm-eval SUITE=unset" in result.stdout + assert "FINAL_SUITE=unset" in result.stdout + + def test_kimi_vendor_rejects_batched_concurrency() -> None: result = _run_invalid_call( "EVAL_MAX_MODEL_LEN=16384 " From 405dc0e2186f62f4150014c405d677694576a2fb Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:59:54 -0500 Subject: [PATCH 14/24] ci: exclude faulty b300 node from slurm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：将发生不可纠正 NVLink 错误的 b300-017 节点排除在 Slurm 分配之外。 --- .github/workflows/benchmark-tmpl.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 68aeaf7a4a..27f933dec5 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -209,7 +209,7 @@ env: MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} # These b300 nodes are currently broken. - SALLOC_EXCLUDE: 'b300-005,b300-006' + SALLOC_EXCLUDE: 'b300-005,b300-006,b300-017' permissions: contents: read From 847982ffefea00377a0b69ad0234663b83bcf397 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 11 Aug 2026 18:10:04 -0500 Subject: [PATCH 15/24] ci: apply B300 node exclusions to allocations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：让 B300 启动器将 SALLOC_EXCLUDE 传递给 salloc，避免调度到已知故障节点。 --- runners/launch_b300-nv.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index cad9ba30aa..5fced02ffb 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -481,8 +481,21 @@ else export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}" - SALLOC_TIME_LIMIT="${SALLOC_TIME_LIMIT:-480}" - salloc --partition=$SLURM_PARTITION --account=$SLURM_ACCOUNT -N 1 --gres=gpu:$GPU_COUNT --exclusive --mem=0 --time="$SALLOC_TIME_LIMIT" --no-shell --job-name="$RUNNER_NAME" + SALLOC_ARGS=( + --partition="$SLURM_PARTITION" + --account="$SLURM_ACCOUNT" + -N 1 + --gres="gpu:$GPU_COUNT" + --exclusive + --mem=0 + --time="${SALLOC_TIME_LIMIT:-480}" + --no-shell + --job-name="$RUNNER_NAME" + ) + if [[ -n "${SALLOC_EXCLUDE:-}" ]]; then + SALLOC_ARGS+=(--exclude="$SALLOC_EXCLUDE") + fi + salloc "${SALLOC_ARGS[@]}" JOB_ID=$(squeue --name="$RUNNER_NAME" -u "$USER" -h -o %A | head -n1) srun --jobid=$JOB_ID \ From 15b08571445d4e8cf14112a73ed4fe7ab9bc60ca Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 11 Aug 2026 20:11:38 -0500 Subject: [PATCH 16/24] feat: enable multinode kimi verifier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：启用多节点 Kimi 验证器 --- .../workflows/benchmark-multinode-tmpl.yml | 14 ++ .github/workflows/e2e-tests.yml | 11 +- benchmarks/benchmark_lib.sh | 10 +- benchmarks/multi_node/agentic_srt.sh | 5 +- .../agentic/agg-b200-tp8pp2-agentic.yaml | 2 +- .../agg-gb200-tep16-balanced-agentic.yaml | 3 +- .../agg-gb200-tp16-latency-agentic.yaml | 3 +- runners/inject_synthetic_acceptance.py | 52 +++-- runners/launch_b200-dgxc.sh | 5 + runners/launch_gb200-nv.sh | 11 +- runners/launch_h200-dgxc-slurm.sh | 5 + runners/patch_srt_eval_dispatch.py | 103 +++++++++ runners/synthetic_injectors/vllm.py | 41 +++- runners/test_slurm_utils.py | 197 ++++++++++++++++++ utils/evals/EVALS.md | 12 +- utils/evals/test_run_eval_dispatch.py | 58 ++++-- 16 files changed, 468 insertions(+), 64 deletions(-) create mode 100755 runners/patch_srt_eval_dispatch.py diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 6b5319c667..675394838a 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -133,6 +133,16 @@ on: type: boolean required: false default: false + eval-framework: + description: "Eval runner (lm-eval, swebench, or kimi-vendor)" + type: string + required: false + default: "lm-eval" + eval-suite: + description: "Kimi Vendor Verifier suite; leave empty for other eval runners" + type: string + required: false + default: "" eval-conc: description: "Concurrency value or space-separated list for eval requests (overrides default max-of-conc-list)" type: string @@ -232,6 +242,8 @@ env: DECODE_HARDWARE: ${{ inputs.decode-hardware }} RUN_EVAL: ${{ inputs.run-eval }} EVAL_ONLY: ${{ inputs.eval-only }} + EVAL_FRAMEWORK: ${{ inputs.eval-framework }} + EVAL_SUITE: ${{ inputs.eval-suite }} EVAL_CONC: ${{ inputs.eval-conc }} EVAL_LIMIT: ${{ inputs.eval-limit }} SWEBENCH_GEN_MODE: ${{ inputs.swebench-gen-mode }} @@ -471,6 +483,7 @@ jobs: path: | meta_env.json results*.json + *_vendor_report.json sample*.jsonl agent_preds.json predictions.jsonl @@ -492,6 +505,7 @@ jobs: run: | rm -f meta_env.json || true rm -f results*.json || true + rm -f *_vendor_report.json || true rm -f sample*.jsonl || true rm -f agent_preds.json predictions.jsonl swebench_report_*.json *.traj* || true diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 7b483a57b1..a0a31b5e91 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -46,12 +46,12 @@ on: type: string default: "" eval-framework: - description: "Single-node agentic eval runner (lm-eval, swebench, or kimi-vendor)" + description: "Agentic eval runner (lm-eval, swebench, or kimi-vendor)" required: false type: string default: "lm-eval" eval-suite: - description: "Single-node Kimi Vendor Verifier suite; empty for other runners" + description: "Kimi Vendor Verifier suite; empty for other runners" required: false type: string default: "" @@ -136,12 +136,12 @@ on: type: string default: "" eval-framework: - description: "Single-node agentic eval runner (lm-eval, swebench, or kimi-vendor)" + description: "Agentic eval runner (lm-eval, swebench, or kimi-vendor)" required: false type: string default: "lm-eval" eval-suite: - description: "Single-node Kimi Vendor Verifier suite; empty for other runners" + description: "Kimi Vendor Verifier suite; empty for other runners" required: false type: string default: "" @@ -280,6 +280,7 @@ jobs: MULTI_AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x and not x.get('run-eval', False)]))" | score_matrix multi-agentic) MULTI_AGENTIC_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x and x.get('run-eval', False)]))" | score_matrix multi-agentic-eval) SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix single) + EVALS=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix eval) MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix multi) MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix multi-eval) { @@ -598,6 +599,8 @@ jobs: eval-conc: ${{ matrix.config['eval-conc'] }} eval-limit: ${{ inputs.eval-limit }} swebench-gen-mode: ${{ inputs.swebench-gen-mode }} + eval-framework: ${{ inputs.eval-framework }} + eval-suite: ${{ inputs.eval-suite }} scenario-type: agentic-coding ref: ${{ inputs.ref }} diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 0043fb901b..7b6454652a 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -929,13 +929,6 @@ _run_kimi_tool_call_schema_eval() { esac done - case "${IS_MULTINODE:-false}" in - true|1) - echo "ERROR: Kimi tool-call schema eval supports single-node only" >&2 - export EVAL_RESULT_DIR="" - return 2 - ;; - esac local model_name="${MODEL_NAME:-${MODEL:-}}" local adapter_path="${INFERENCEX_REPO_ROOT}/utils/evals/kimi_vendor_eval.py" @@ -1816,6 +1809,9 @@ run_eval() { fi local framework="${EVAL_FRAMEWORK:-${cli_framework:-$scenario_default}}" + if [ "$framework" = "kimi-vendor" ] && [ -z "${EVAL_SUITE:-}" ]; then + EVAL_SUITE="kimi_tool_call_schema" + fi case "${EVAL_SUITE:-}" in "") ;; diff --git a/benchmarks/multi_node/agentic_srt.sh b/benchmarks/multi_node/agentic_srt.sh index 79a36da524..dea0881327 100644 --- a/benchmarks/multi_node/agentic_srt.sh +++ b/benchmarks/multi_node/agentic_srt.sh @@ -26,8 +26,6 @@ for concurrency in "${CONCURRENCIES[@]}"; do fi done -resolve_trace_source -install_agentic_deps wait_for_agentic_servers_idle() { local timeout_seconds="${AIPERF_DRAIN_TIMEOUT_SECONDS:-1800}" @@ -98,6 +96,9 @@ raise SystemExit(f"Agentic servers did not drain within {timeout_seconds} second PY } +resolve_trace_source +install_agentic_deps + # The AgentX scenario's first-turn cache-bust marker includes AIPerf's unique # per-invocation benchmark ID. Each point therefore gets a disjoint KV keyspace # while its own warmup and profile phases share markers. This makes sequential diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index a0207d65f2..e6ed84715d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -80,7 +80,7 @@ backend: TILELANG_CLEANUP_TEMP_FILES: "1" UCX_MEMTYPE_CACHE: "n" UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_6:1,mlx5_7:1" HF_HUB_CACHE: "/hf_hub_cache" HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" vllm_config: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml index bf135c0e33..e22dae3f38 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml @@ -101,7 +101,7 @@ backend: max-num-seqs: 32 max-num-batched-tokens: 8192 speculative-config: '{"method":"dspark","model":"/tmp/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' - compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96]}' + compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96],"pass_config":{"fuse_allreduce_rms":false}}' block-size: 64 language-model-only: true disable-custom-all-reduce: true @@ -111,7 +111,6 @@ backend: reasoning-parser: "kimi_k3" dyn-reasoning-parser: "kimi_k3" no-enable-flashinfer-autotune: true - compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' sbatch_directives: cpus-per-task: "144" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml index baed3f19e7..2566aa62f4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml @@ -98,7 +98,7 @@ backend: max-num-seqs: 8 max-num-batched-tokens: 8192 speculative-config: '{"method":"dspark","model":"/tmp/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' - compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24]}' + compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24],"pass_config":{"fuse_allreduce_rms":false}}' block-size: 64 language-model-only: true disable-custom-all-reduce: true @@ -108,7 +108,6 @@ backend: reasoning-parser: "kimi_k3" dyn-reasoning-parser: "kimi_k3" no-enable-flashinfer-autotune: true - compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' sbatch_directives: cpus-per-task: "144" diff --git a/runners/inject_synthetic_acceptance.py b/runners/inject_synthetic_acceptance.py index 80d2740e79..1382454d13 100644 --- a/runners/inject_synthetic_acceptance.py +++ b/runners/inject_synthetic_acceptance.py @@ -1,18 +1,13 @@ #!/usr/bin/env python3 -"""Inject synthetic acceptance parameters into an srt-slurm recipe (generic driver). +"""Configure speculative acceptance in an srt-slurm recipe. -This is the framework-agnostic half of the synthetic-acceptance mechanism. It -decides *whether* to inject (the ``SYNTHETIC_ACCEPTANCE`` flag) and *what* mean -acceptance length to inject, then delegates the actual recipe rewrite to a -framework-specific backend (see ``runners/synthetic_injectors/``). - -The script is a no-op (exit 0, file untouched) when: - - SYNTHETIC_ACCEPTANCE is unset/false, -so existing callers that do not opt in get exactly the previous behavior. When -enabled it requires a backend registered for the given framework; the vLLM -backend is added in a follow-up framework-support change. +Eval-only runs remove synthetic acceptance so generated text is checked against +the target model. Throughput runs inject a configured synthetic acceptance +length only when ``SYNTHETIC_ACCEPTANCE=true``. Framework-specific rewriting +lives under ``runners/synthetic_injectors/``. Environment variables: + EVAL_ONLY "true" to restore real target verification SYNTHETIC_ACCEPTANCE "true" to enable (default: "false") SYNTHETIC_ACCEPTANCE_LENGTH target mean acceptance length; if unset, it is auto-resolved from the reference AL YAML using @@ -72,7 +67,9 @@ def _lookup_al(model_block, num_spec_tokens): if isinstance(model_block, dict): # Thinking matrix form: pick the requested mode, then index by level. if any(str(k).startswith("thinking") for k in model_block): - mode = os.environ.get("THINKING_MODE", "thinking_on").strip() or "thinking_on" + mode = ( + os.environ.get("THINKING_MODE", "thinking_on").strip() or "thinking_on" + ) mode_block = model_block.get(mode) if mode_block is None: sys.exit( @@ -115,7 +112,9 @@ def _resolve_al(config_text, injector, ref_yaml): al = _lookup_al(model_block, num_spec_tokens) if al is None: - sys.exit(f"ERROR: num_spec_tokens={num_spec_tokens} not found for {key} in {ref_yaml}") + sys.exit( + f"ERROR: num_spec_tokens={num_spec_tokens} not found for {key} in {ref_yaml}" + ) _log( f"Auto-resolved AL={al} from {ref_yaml} " @@ -125,14 +124,30 @@ def _resolve_al(config_text, injector, ref_yaml): def inject(config_file, framework): + injector = get_injector(framework) + if _enabled("EVAL_ONLY"): - print("[Synthetic AL] EVAL_ONLY=true: keeping real MTP recipe") + if injector is None or not hasattr(injector, "rewrite_real"): + print( + f"[Synthetic AL] EVAL_ONLY=true: no real-acceptance rewriter " + f"for FRAMEWORK='{framework}'" + ) + return 0 + + with open(config_file) as f: + content = f.read() + new_content, count = injector.rewrite_real(content, _log) + if count: + with open(config_file, "w") as f: + f.write(new_content) + _log(f"Restored real acceptance in {count} speculative-config entries") + else: + _log("EVAL_ONLY=true: recipe already uses real acceptance") return 0 if not _enabled("SYNTHETIC_ACCEPTANCE"): return 0 - injector = get_injector(framework) if injector is None: sys.exit( "ERROR: SYNTHETIC_ACCEPTANCE=true but no synthetic-acceptance " @@ -145,7 +160,12 @@ def inject(config_file, framework): al = _resolve_al( content, injector, - os.path.join(os.path.dirname(__file__), "..", "benchmarks", "speedbench-reference-al.yaml"), + os.path.join( + os.path.dirname(__file__), + "..", + "benchmarks", + "speedbench-reference-al.yaml", + ), ) _log(f"Injecting synthetic acceptance (length={al}) into {config_file}") diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 2cd2f2ee81..7e1c5d1b50 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -225,6 +225,9 @@ if [[ "$IS_MULTINODE" == "true" ]]; then cd "$SRT_REPO_DIR" || exit 1 git checkout sa-submission-q2-2026 fi + if [[ "${EVAL_FRAMEWORK:-lm-eval}" == "kimi-vendor" ]]; then + python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" || exit 1 + fi echo "Installing srtctl..." export UV_INSTALL_DIR="$GITHUB_WORKSPACE/.local/bin" @@ -364,6 +367,8 @@ EOF # so large-model loads (e.g. DSR1-FP8 ~680GB off shared FS) finish in time. # Uses ${CONFIG_FILE%%:*} because CONFIG_FILE may carry an :override[N] suffix. sed -i 's/^ max_attempts: [0-9]*/ max_attempts: 720/' "${CONFIG_FILE%%:*}" + python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \ + "${CONFIG_FILE%%:*}" "$FRAMEWORK" || exit 1 SRTCTL_PREFLIGHT_ARGS=() # Kimi K2.6 weights are staged on the Slurm compute nodes, not the login node. diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 34d7c4c806..870b684725 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -488,6 +488,9 @@ else git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" fi +if [[ "${EVAL_FRAMEWORK:-lm-eval}" == "kimi-vendor" ]]; then + python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" || exit 1 +fi echo "Installing srtctl..." curl -LsSf https://astral.sh/uv/install.sh | sh @@ -635,10 +638,10 @@ if command -v squeue >/dev/null 2>&1; then fi sed -i "s/^name:.*/name: \"${SRT_SLURM_JOB_NAME}\"/" "$CONFIG_PATH" -# Optionally inject synthetic acceptance into the recipe's speculative-config -# when SYNTHETIC_ACCEPTANCE=true (no-op otherwise). Must run after the name -# override and before srtctl apply so the rendered job picks it up. -python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" "$CONFIG_PATH" "$FRAMEWORK" +# Restore real acceptance for eval-only jobs, or inject synthetic acceptance +# when a throughput run explicitly enables it. +python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \ + "$CONFIG_PATH" "$FRAMEWORK" || exit 1 # Don't leak the login-node venv to the compute-node orchestrator. sbatch's # default --export=ALL propagates VIRTUAL_ENV (set by `source diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index 8991e04182..622fcb66f1 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -84,6 +84,9 @@ if [[ "$IS_MULTINODE" == "true" ]]; then cd "$SRT_REPO_DIR" git checkout sa-submission-q2-2026 fi + if [[ "${EVAL_FRAMEWORK:-lm-eval}" == "kimi-vendor" ]]; then + python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" + fi echo "Installing srtctl..." curl -LsSf https://astral.sh/uv/install.sh | sh @@ -206,6 +209,8 @@ EOF sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" sed -i '/^health_check:/,/^[^ ]/{ /^health_check:/d; /^ /d; }' "$CONFIG_PATH" printf '\nhealth_check:\n max_attempts: 720\n interval_seconds: 10\n' >> "$CONFIG_PATH" + python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \ + "$CONFIG_PATH" "$FRAMEWORK" WORKLOAD_TAG="${ISL}x${OSL}" if [[ "$IS_AGENTIC" == "1" ]]; then WORKLOAD_TAG="agentic" diff --git a/runners/patch_srt_eval_dispatch.py b/runners/patch_srt_eval_dispatch.py new file mode 100755 index 0000000000..daa52d9ab0 --- /dev/null +++ b/runners/patch_srt_eval_dispatch.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Enable InferenceX-selected eval dispatch in an srt-slurm checkout.""" + +from __future__ import annotations + +import sys +from pathlib import Path + + +DO_SWEEP_ENV_BLOCK = """ "EVAL_ONLY", + "IS_MULTINODE",""" +DO_SWEEP_ENV_REPLACEMENT = """ "EVAL_ONLY", + "EVAL_FRAMEWORK", + "EVAL_SUITE", + "IS_MULTINODE",""" +LM_EVAL_COMMAND = 'run_eval --framework lm-eval --port "$PORT" || eval_rc=$?' +GENERIC_EVAL_COMMAND = 'run_eval --port "$PORT" || eval_rc=$?' +EVAL_ARTIFACT_COPY = """cp -v results*.json /logs/eval_results/ 2>/dev/null || true +cp -v sample*.jsonl /logs/eval_results/ 2>/dev/null || true""" +KIMI_ARTIFACT_COPY = """cp -v results*.json /logs/eval_results/ 2>/dev/null || true +cp -v *_vendor_report.json /logs/eval_results/ 2>/dev/null || true +cp -v sample*.jsonl /logs/eval_results/ 2>/dev/null || true""" + + +def prepare_replacements( + path: Path, + replacements: tuple[tuple[str, str], ...], +) -> tuple[str, str, bool]: + """Validate source replacements without mutating the checkout.""" + original = path.read_text() + content = original + changed = False + for old, new in replacements: + old_count = content.count(old) + new_count = content.count(new) + if old_count == 1 and new_count == 0: + content = content.replace(old, new, 1) + changed = True + elif old_count != 0 or new_count != 1: + raise RuntimeError( + f"invalid patch state in {path}: old anchor count={old_count}, " + f"replacement count={new_count}" + ) + return original, content, changed + + +def patch_checkout(root: Path) -> list[Path]: + """Patch both post-eval sources after validating the complete checkout.""" + patches = ( + ( + root / "src/srtctl/cli/do_sweep.py", + ((DO_SWEEP_ENV_BLOCK, DO_SWEEP_ENV_REPLACEMENT),), + ), + ( + root / "src/srtctl/benchmarks/scripts/lm-eval/bench.sh", + ( + (LM_EVAL_COMMAND, GENERIC_EVAL_COMMAND), + (EVAL_ARTIFACT_COPY, KIMI_ARTIFACT_COPY), + ), + ), + ) + staged = [ + (path, *prepare_replacements(path, replacements)) + for path, replacements in patches + ] + changed = [] + written = [] + try: + for path, original, replacement, needs_write in staged: + if needs_write: + path.write_text(replacement) + written.append((path, original)) + changed.append(path) + except OSError: + for path, original in reversed(written): + path.write_text(original) + raise + return changed + + +def main(argv: list[str]) -> int: + """Patch the checkout named on the command line.""" + if len(argv) != 2: + print(f"Usage: {argv[0]} SRT_SLURM_CHECKOUT", file=sys.stderr) + return 2 + root = Path(argv[1]).resolve() + try: + changed = patch_checkout(root) + except (OSError, RuntimeError) as error: + print( + f"ERROR: failed to patch srt-slurm eval dispatch: {error}", file=sys.stderr + ) + return 1 + if changed: + for path in changed: + print(f"Patched srt-slurm eval dispatch: {path}") + else: + print("srt-slurm eval dispatch is already patched") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/runners/synthetic_injectors/vllm.py b/runners/synthetic_injectors/vllm.py index e71a5df2b4..d36740195c 100644 --- a/runners/synthetic_injectors/vllm.py +++ b/runners/synthetic_injectors/vllm.py @@ -1,13 +1,8 @@ -"""vLLM synthetic-acceptance backend (FRAMEWORK=dynamo-vllm). +"""vLLM speculative-acceptance recipe rewriting. -Rewrites every ``speculative-config: ''`` entry in an srt-slurm recipe to -use synthetic rejection sampling: it adds ``rejection_sample_method=synthetic`` -and ``synthetic_acceptance_length=`` to the JSON so the engine emits a -controlled mean acceptance length instead of running the real draft model. - -Registered under the "dynamo-vllm" framework key at import time, so importing -the ``synthetic_injectors`` package is enough for the generic driver to resolve -this backend. +Throughput opt-ins can inject synthetic acceptance. Eval-only runs restore real +block verification so model outputs remain valid for accuracy checks. The +backend is registered for both direct vLLM and Dynamo-vLLM recipes. """ import json @@ -57,7 +52,9 @@ def _replace(match): new_content, count = _SPEC_CONFIG_RE.subn(_replace, content) if count: - after = [ln.strip() for ln in new_content.splitlines() if _SPEC_CONFIG_RE.search(ln)] + after = [ + ln.strip() for ln in new_content.splitlines() if _SPEC_CONFIG_RE.search(ln) + ] if after: log("After:") for ln in after: @@ -66,4 +63,28 @@ def _replace(match): return new_content, count +def rewrite_real(content, log): + """Restore real block verification in every synthetic config entry.""" + modified = 0 + + def _replace(match): + nonlocal modified + spec = json.loads(match.group(1)) + if ( + spec.get("rejection_sample_method") != "synthetic" + and "synthetic_acceptance_length" not in spec + ): + return match.group(0) + spec["rejection_sample_method"] = "block" + spec.pop("synthetic_acceptance_length", None) + modified += 1 + return "speculative-config: '" + json.dumps(spec, separators=(",", ":")) + "'" + + new_content = _SPEC_CONFIG_RE.sub(_replace, content) + if modified: + log("Restored real block verification for eval-only mode") + return new_content, modified + + register("dynamo-vllm", sys.modules[__name__]) +register("vllm", sys.modules[__name__]) diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index c027cc9b2f..bfce571816 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -1,9 +1,14 @@ +import json +import os import subprocess from pathlib import Path +import yaml REPO_ROOT = Path(__file__).resolve().parents[1] SLURM_UTILS = REPO_ROOT / "runners" / "slurm_utils.sh" +PATCH_SRT_EVAL = REPO_ROOT / "runners" / "patch_srt_eval_dispatch.py" +INJECT_ACCEPTANCE = REPO_ROOT / "runners" / "inject_synthetic_acceptance.py" def run_bash(command: str, *args: Path | str) -> subprocess.CompletedProcess[str]: @@ -55,3 +60,195 @@ def test_copy_agentic_results_fails_when_aggregate_is_missing( assert result.returncode != 0 assert "no run_conc*.json results found" in result.stderr + + +def test_patch_srt_eval_dispatch_forwards_selection_and_is_idempotent( + tmp_path: Path, +) -> None: + do_sweep = tmp_path / "src/srtctl/cli/do_sweep.py" + eval_script = tmp_path / "src/srtctl/benchmarks/scripts/lm-eval/bench.sh" + do_sweep.parent.mkdir(parents=True) + eval_script.parent.mkdir(parents=True) + do_sweep.write_text( + " for var in [\n" + ' "RUN_EVAL",\n' + ' "EVAL_ONLY",\n' + ' "IS_MULTINODE",\n' + " ]:\n" + ) + eval_script.write_text( + 'run_eval --framework lm-eval --port "$PORT" || eval_rc=$?\n' + "cp -v results*.json /logs/eval_results/ 2>/dev/null || true\n" + "cp -v sample*.jsonl /logs/eval_results/ 2>/dev/null || true\n" + ) + + first = subprocess.run( + ["python3", str(PATCH_SRT_EVAL), str(tmp_path)], + check=False, + capture_output=True, + text=True, + ) + second = subprocess.run( + ["python3", str(PATCH_SRT_EVAL), str(tmp_path)], + check=False, + capture_output=True, + text=True, + ) + + assert first.returncode == 0, first.stderr + assert second.returncode == 0, second.stderr + assert do_sweep.read_text().count('"EVAL_FRAMEWORK"') == 1 + assert do_sweep.read_text().count('"EVAL_SUITE"') == 1 + assert 'run_eval --port "$PORT"' in eval_script.read_text() + assert "--framework lm-eval" not in eval_script.read_text() + assert "*_vendor_report.json" in eval_script.read_text() + assert "already patched" in second.stdout + + +def test_patch_srt_eval_dispatch_preflights_before_writing(tmp_path: Path) -> None: + do_sweep = tmp_path / "src/srtctl/cli/do_sweep.py" + eval_script = tmp_path / "src/srtctl/benchmarks/scripts/lm-eval/bench.sh" + do_sweep.parent.mkdir(parents=True) + eval_script.parent.mkdir(parents=True) + original_do_sweep = ' "EVAL_ONLY",\n "IS_MULTINODE",\n' + original_eval_script = "unsupported eval hook\n" + do_sweep.write_text(original_do_sweep) + eval_script.write_text(original_eval_script) + + result = subprocess.run( + ["python3", str(PATCH_SRT_EVAL), str(tmp_path)], + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert do_sweep.read_text() == original_do_sweep + assert eval_script.read_text() == original_eval_script + + +def test_patch_srt_eval_dispatch_rejects_mixed_patch_state(tmp_path: Path) -> None: + do_sweep = tmp_path / "src/srtctl/cli/do_sweep.py" + eval_script = tmp_path / "src/srtctl/benchmarks/scripts/lm-eval/bench.sh" + do_sweep.parent.mkdir(parents=True) + eval_script.parent.mkdir(parents=True) + original_do_sweep = ( + ' "EVAL_ONLY",\n' + ' "IS_MULTINODE",\n' + ' "EVAL_ONLY",\n' + ' "EVAL_FRAMEWORK",\n' + ' "EVAL_SUITE",\n' + ' "IS_MULTINODE",\n' + ) + original_eval_script = ( + 'run_eval --framework lm-eval --port "$PORT" || eval_rc=$?\n' + "cp -v results*.json /logs/eval_results/ 2>/dev/null || true\n" + "cp -v sample*.jsonl /logs/eval_results/ 2>/dev/null || true\n" + ) + do_sweep.write_text(original_do_sweep) + eval_script.write_text(original_eval_script) + + result = subprocess.run( + ["python3", str(PATCH_SRT_EVAL), str(tmp_path)], + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert "invalid patch state" in result.stderr + assert do_sweep.read_text() == original_do_sweep + assert eval_script.read_text() == original_eval_script + + +def test_eval_only_restores_real_vllm_acceptance(tmp_path: Path) -> None: + recipe = tmp_path / "recipe.yaml" + recipe.write_text( + "speculative-config: " + """'{\"method\":\"dspark\",\"num_speculative_tokens\":2,""" + """\"rejection_sample_method\":\"synthetic\",""" + """\"synthetic_acceptance_length\":2.51}'\n""" + ) + env = { + **os.environ, + "EVAL_ONLY": "true", + "SYNTHETIC_ACCEPTANCE": "true", + } + + result = subprocess.run( + ["python3", str(INJECT_ACCEPTANCE), str(recipe), "vllm"], + env=env, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + rewritten = recipe.read_text() + assert '"rejection_sample_method":"block"' in rewritten + assert "synthetic_acceptance_length" not in rewritten + + +def test_eval_only_acceptance_rewrite_allows_non_speculative_recipe( + tmp_path: Path, +) -> None: + recipe = tmp_path / "recipe.yaml" + original = "backend:\n type: vllm\n" + recipe.write_text(original) + + result = subprocess.run( + ["python3", str(INJECT_ACCEPTANCE), str(recipe), "dynamo-vllm"], + env={**os.environ, "EVAL_ONLY": "true"}, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + assert recipe.read_text() == original + + +def test_nvidia_srt_launchers_prepare_kimi_eval_dispatch() -> None: + launchers = ( + REPO_ROOT / "runners/launch_h200-dgxc-slurm.sh", + REPO_ROOT / "runners/launch_b200-dgxc.sh", + REPO_ROOT / "runners/launch_gb200-nv.sh", + ) + + for launcher in launchers: + content = launcher.read_text() + assert "patch_srt_eval_dispatch.py" in content + assert 'EVAL_FRAMEWORK:-lm-eval}" == "kimi-vendor"' in content + assert "inject_synthetic_acceptance.py" in content + + +def test_gb200_kimi_compilation_config_preserves_all_settings() -> None: + recipes = { + "agg-gb200-tep16-balanced-agentic.yaml": 96, + "agg-gb200-tp16-latency-agentic.yaml": 24, + } + recipe_dir = ( + REPO_ROOT / "benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" + ) + + for filename, largest_capture in recipes.items(): + recipe = yaml.safe_load((recipe_dir / filename).read_text()) + raw_config = recipe["backend"]["vllm_config"]["aggregated"][ + "compilation-config" + ] + compilation_config = json.loads(raw_config) + + assert compilation_config["cudagraph_capture_sizes"][-1] == largest_capture + assert compilation_config["pass_config"]["fuse_allreduce_rms"] is False + + +def test_b200_kimi_recipe_uses_available_roce_devices() -> None: + recipe_path = ( + REPO_ROOT + / "benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" + / "agg-b200-tp8pp2-agentic.yaml" + ) + recipe = yaml.safe_load(recipe_path.read_text()) + devices = recipe["backend"]["aggregated_environment"]["UCX_NET_DEVICES"] + + assert devices == ",".join(f"mlx5_{index}:1" for index in range(8)) diff --git a/utils/evals/EVALS.md b/utils/evals/EVALS.md index 58237912f0..d5de674c62 100644 --- a/utils/evals/EVALS.md +++ b/utils/evals/EVALS.md @@ -48,7 +48,8 @@ runner. Existing jobs continue to use lm-eval with GSM8K by default. The default eval framework is [lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) (`lm-eval`). Agentic eval-only matrix jobs inherit this default and therefore run the same GSM8K task as 8k1k. Explicit agentic runs can still select SWE-bench. -The Phase 1 Kimi smoke is opt-in and single-node only. Select +The Phase 1 Kimi smoke is opt-in. It supports single-node jobs and Kimi K3 +aggregate H200, B200, and GB200 srt-slurm jobs. Select `eval-framework: kimi-vendor` and `eval-suite: kimi_tool_call_schema` on `e2e-tests.yml`, or invoke it from the repository root after a server is ready: @@ -98,7 +99,10 @@ upstream pytest process to 900 seconds. This smoke validates one object-schema tool call. It does not cover tool choice, parallel calls, multi-turn execution, or general agent quality. Multi-value -batched concurrency and multi-node execution are unsupported. +batched concurrency is unsupported. Multi-node aggregate jobs run the same +two-case smoke against their OpenAI-compatible frontend. Eval-only launchers +restore real block verification before submitting recipes that otherwise use +synthetic acceptance for throughput. ### Benchmark script flow @@ -161,10 +165,10 @@ Multi-node evals support two hardware paths: **NVIDIA Slurm multi-node (GB200, GB300, B200, B300, H100, H200)** runs through [srt-slurm](https://github.com/NVIDIA/srt-slurm) on the `sa-submission-q2-2026` branch. - `do_sweep.py` skips the benchmark stage when `EVAL_ONLY=true`, runs `_run_post_eval()` directly - In eval-only mode, uses the full `wait_for_model()` health check (same as benchmark stage) since the benchmark health check was skipped -- `lm-eval` runner (`benchmarks/lm_eval.py`) is invoked by `do_sweep.py` as a post/eval-only step and sources InferenceX's `benchmark_lib.sh` from the mounted workspace (`/infmax-workspace`) +- The registered srt-slurm `lm-eval` post-runner sources InferenceX's `benchmark_lib.sh` from the mounted workspace (`/infmax-workspace`). Kimi-selected launches patch that hook to use generic `run_eval` dispatch while preserving lm-eval as the default. - Eval artifacts written to `/logs/eval_results/` inside the container, collected by launch scripts - NVIDIA Slurm launch scripts always collect server logs for debugging but skip benchmark result collection when `EVAL_ONLY=true` -- Env vars threaded: `RUN_EVAL`, `EVAL_ONLY`, `IS_MULTINODE`, `FRAMEWORK`, `PRECISION`, `MODEL_PREFIX`, `RUNNER_TYPE`, `RESULT_FILENAME`, `SPEC_DECODING`, `ISL`, `OSL`, `PREFILL_TP/EP/NUM_WORKERS/DP_ATTN`, `DECODE_TP/EP/NUM_WORKERS/DP_ATTN`, `MODEL_NAME`, `EVAL_CONC` +- Env vars threaded: `RUN_EVAL`, `EVAL_ONLY`, `EVAL_FRAMEWORK`, `EVAL_SUITE`, `IS_MULTINODE`, `FRAMEWORK`, `PRECISION`, `MODEL_PREFIX`, `RUNNER_TYPE`, `RESULT_FILENAME`, `SPEC_DECODING`, `ISL`, `OSL`, `PREFILL_TP/EP/NUM_WORKERS/DP_ATTN`, `DECODE_TP/EP/NUM_WORKERS/DP_ATTN`, `MODEL_NAME`, `EVAL_CONC` For multi-node `all-evals`, `EVAL_CONC` is a space-separated list. When it contains multiple values, `run_eval` runs those concurrency points sequentially against the same live engine, stages each result with a `_concN` filename suffix, and records expected/completed/failed points in `meta_env.json`. diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index fae4aab2de..bc235c094f 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -11,6 +11,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] BENCHMARK_LIB = REPO_ROOT / "benchmarks" / "benchmark_lib.sh" +MULTINODE_WORKFLOW = REPO_ROOT / ".github/workflows/benchmark-multinode-tmpl.yml" E2E_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "e2e-tests.yml" _SCRIPT = r''' @@ -221,6 +222,31 @@ def test_run_eval_scopes_runner_selected_suite_to_one_call() -> None: assert "FINAL_SUITE=unset" in result.stdout +def test_kimi_default_suite_reaches_eval_only_metadata() -> None: + script = r''' +source "$BENCHMARK_LIB" +run_kimi_vendor_eval() { echo "DISPATCH=$EVAL_SUITE"; } +append_lm_eval_summary() { echo "METADATA=$EVAL_SUITE"; } +export EVAL_FRAMEWORK=kimi-vendor +export EVAL_ONLY=true +export IS_AGENTIC=1 +export EVAL_CONCURRENT_REQUESTS="" +unset EVAL_SUITE +run_eval --port 8888 +''' + result = subprocess.run( + ["bash", "-c", script], + env={**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB)}, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert "DISPATCH=kimi_tool_call_schema" in result.stdout + assert "METADATA=kimi_tool_call_schema" in result.stdout + + def test_kimi_vendor_rejects_batched_concurrency() -> None: result = _run_invalid_call( "EVAL_MAX_MODEL_LEN=16384 " @@ -239,15 +265,6 @@ def test_kimi_vendor_rejects_unsupported_suite() -> None: assert "unsupported Kimi Vendor Verifier suite 'gsm8k'" in result.stderr -def test_kimi_vendor_rejects_multinode() -> None: - for value in ("true", "1"): - result = _run_invalid_call( - f"EVAL_SUITE=kimi_tool_call_schema IS_MULTINODE={value} " - "run_kimi_vendor_eval" - ) - assert result.returncode == 2 - assert "supports single-node only" in result.stderr - def test_kimi_vendor_setup_failure_writes_compatibility_result( tmp_path: Path, @@ -352,7 +369,9 @@ def test_kimi_vendor_surfaces_failure_artifact_error(tmp_path: Path) -> None: -def test_kimi_vendor_runner_uses_fixed_upstream_contract(tmp_path: Path) -> None: +def test_kimi_vendor_multinode_runner_uses_fixed_upstream_contract( + tmp_path: Path, +) -> None: results_dir = tmp_path / "results" verifier_dir = tmp_path / "verifier" runtime_dir = tmp_path / "runtime" @@ -383,7 +402,7 @@ def test_kimi_vendor_runner_uses_fixed_upstream_contract(tmp_path: Path) -> None "RUNTIME_DIR": str(runtime_dir), "OPENAI_API_KEY": "must-not-be-forwarded", "KV_OFFLOADING": "none", - "IS_MULTINODE": "false", + "IS_MULTINODE": "true", } for key in ( "EVAL_SUITE", @@ -1134,6 +1153,18 @@ def test_agentic_eval_workflow_forwards_runner_contract() -> None: assert forwarded["eval-suite"] == "${{ inputs.eval-suite }}" +def test_multinode_agentic_eval_workflow_forwards_runner_contract() -> None: + workflow = yaml.safe_load(E2E_WORKFLOW.read_text()) + forwarded = workflow["jobs"]["test-sweep-multi-node-agentic-evals"]["with"] + reusable_workflow = yaml.safe_load(MULTINODE_WORKFLOW.read_text()) + + assert forwarded["eval-framework"] == "${{ inputs.eval-framework }}" + assert forwarded["eval-suite"] == "${{ inputs.eval-suite }}" + assert reusable_workflow["env"]["EVAL_FRAMEWORK"] == "${{ inputs.eval-framework }}" + assert reusable_workflow["env"]["EVAL_SUITE"] == "${{ inputs.eval-suite }}" + assert "*_vendor_report.json" in MULTINODE_WORKFLOW.read_text() + + def test_trusted_changelog_matrix_keeps_multinode_agentic_evals() -> None: workflow = yaml.safe_load(E2E_WORKFLOW.read_text()) @@ -1148,4 +1179,7 @@ def test_trusted_changelog_matrix_keeps_multinode_agentic_evals() -> None: if "rows.extend" in line ) - assert '"multinode_agentic_evals"' in flatten_command \ No newline at end of file + assert '"multinode_agentic_evals"' in flatten_command + get_jobs_command = get_jobs["run"] + assert "EVALS=$(" in get_jobs_command + assert "score_matrix eval" in get_jobs_command \ No newline at end of file From 13c5a457ae39c8e252ed57a99ba4bb114ca842da Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 11 Aug 2026 21:43:29 -0500 Subject: [PATCH 17/24] fix: harden Kimi eval runtime failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：强化 Kimi 评估运行时故障处理 --- .../workflows/benchmark-multinode-tmpl.yml | 2 +- benchmarks/benchmark_lib.sh | 221 +++++++++-- utils/collect_eval_results.py | 47 ++- utils/evals/EVALS.md | 11 +- utils/evals/test_batched_eval.py | 100 +++++ utils/evals/test_run_eval_dispatch.py | 345 ++++++++++++++++++ utils/evals/validate_scores.py | 46 +++ utils/test_collect_eval_results.py | 85 +++-- 8 files changed, 796 insertions(+), 61 deletions(-) diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 675394838a..7004faed99 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -479,7 +479,7 @@ jobs: if: ${{ always() && (env.RUN_EVAL == 'true' || inputs.eval-only) }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: eval_${{ env.EXP_NAME }}_${{ env.RESULT_FILENAME }} + name: eval_${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_p${{ env.PREFILL_NUM_WORKERS }}x${{ env.PREFILL_TP }}p${{ env.PREFILL_PP_SIZE }}c${{ env.PREFILL_DCP_SIZE }}k${{ env.PREFILL_PCP_SIZE }}e${{ env.PREFILL_EP }}d${{ env.PREFILL_DP_ATTN }}_d${{ env.DECODE_NUM_WORKERS }}x${{ env.DECODE_TP }}p${{ env.DECODE_PP_SIZE }}c${{ env.DECODE_DCP_SIZE }}k${{ env.DECODE_PCP_SIZE }}e${{ env.DECODE_EP }}d${{ env.DECODE_DP_ATTN }}_kv${{ env.KV_OFFLOADING }}-${{ env.KV_OFFLOAD_BACKEND }}_spec${{ env.SPEC_DECODING }}_c${{ join(fromJson(inputs.conc-list), 'x') }}_${{ runner.name }} path: | meta_env.json results*.json diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 7b6454652a..cef240b274 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -856,31 +856,210 @@ _prepare_kimi_vendor_runtime() { _prepare_kimi_vendor_verifier() { local repo_url="$1" local verifier_ref="$2" - local checkout_dir + local checkout_dir prepare_rc=0 - command -v git >/dev/null 2>&1 || { - echo "ERROR: git is required to fetch Kimi-Vendor-Verifier" >&2 + checkout_dir="$(mktemp -d /tmp/kimi-vendor-verifier-XXXXXX)" || { + echo "ERROR: could not create a temporary directory for Kimi-Vendor-Verifier" >&2 return 1 } - checkout_dir="$(mktemp -d /tmp/kimi-vendor-verifier-XXXXXX)" || return $? - if ! ( - git init -q "$checkout_dir" \ - && git -C "$checkout_dir" remote add origin "$repo_url" \ - && git -C "$checkout_dir" fetch -q --filter=blob:none --depth=1 \ - origin "$verifier_ref" \ - && git -C "$checkout_dir" update-ref HEAD FETCH_HEAD \ - && git -C "$checkout_dir" sparse-checkout set --no-cone \ - /pyproject.toml \ - /tests/conftest.py \ - /tests/__init__.py \ - /tests/tool_call_json_schema/ \ - /testdata/walle_validator_cases/ \ - && git -C "$checkout_dir" checkout -q --detach HEAD - ); then - rm -rf "$checkout_dir" - echo "ERROR: failed to fetch Kimi-Vendor-Verifier at ${verifier_ref}" >&2 - return 1 + + python3 - "$repo_url" "$verifier_ref" "$checkout_dir" <<'PY' || prepare_rc=$? +from pathlib import Path +import re +import socket +import sys +import tarfile +import tempfile +import time +from urllib.parse import quote, urlsplit, urlunsplit +from urllib.request import Request, urlopen + + +repo_url, verifier_ref, checkout_dir_arg = sys.argv[1:] +checkout_dir = Path(checkout_dir_arg) +stage = "derive the pinned archive URL" + + +def archive_member_parts(name): + if not name or "\x00" in name or "\\" in name or name.startswith("/"): + raise ValueError(f"unsafe archive member path: {name!r}") + normalized = name.rstrip("/") + parts = normalized.split("/") + if not normalized or any(part in ("", ".", "..") for part in parts): + raise ValueError(f"unsafe archive member path: {name!r}") + return tuple(parts) + + +try: + if not re.fullmatch(r"[0-9a-fA-F]{40}", verifier_ref): + raise ValueError(f"expected a 40-character commit SHA, got {verifier_ref!r}") + + parsed_repo_url = urlsplit(repo_url) + if parsed_repo_url.scheme not in ("http", "https") or not parsed_repo_url.netloc: + raise ValueError(f"unsupported repository URL: {repo_url!r}") + if parsed_repo_url.query or parsed_repo_url.fragment: + raise ValueError(f"repository URL must not contain a query or fragment: {repo_url!r}") + repo_path = parsed_repo_url.path.rstrip("/") + if repo_path.endswith(".git"): + repo_path = repo_path[:-4] + if not repo_path: + raise ValueError(f"repository URL has no repository path: {repo_url!r}") + archive_path = f"{repo_path}/archive/{quote(verifier_ref, safe='')}.tar.gz" + archive_url = urlunsplit( + (parsed_repo_url.scheme, parsed_repo_url.netloc, archive_path, "", "") + ) + + stage = f"download {archive_url}" + request = Request( + archive_url, + headers={"User-Agent": "InferenceX-Kimi-Vendor-Verifier"}, + ) + with tempfile.TemporaryFile() as archive_file: + downloaded = 0 + deadline = time.monotonic() + 60 + with urlopen(request, timeout=60) as response: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("archive download exceeded the 60-second deadline") + sock = getattr(getattr(response, "fp", None), "raw", None) + sock = getattr(sock, "_sock", None) + if sock is not None: + sock.settimeout(max(0.001, remaining)) + try: + chunk = response.read(1024 * 1024) + except socket.timeout as error: + raise TimeoutError( + "archive download exceeded the 60-second deadline" + ) from error + if not chunk: + break + downloaded += len(chunk) + if downloaded > 128 * 1024 * 1024: + raise ValueError("archive download exceeds the 128 MiB safety limit") + archive_file.write(chunk) + if downloaded == 0: + raise ValueError("downloaded archive is empty") + archive_file.seek(0) + + stage = "validate the downloaded archive" + required_files = { + "pyproject.toml", + "tests/conftest.py", + "tests/__init__.py", + "tests/tool_call_json_schema/conftest.py", + "tests/tool_call_json_schema/__init__.py", + "tests/tool_call_json_schema/test_tool_call_json_schema.py", + "tests/tool_call_json_schema/validator.py", + "testdata/walle_validator_cases/validator_cases/TestAdditionalProperties/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestAnyOf/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestBasicTypes/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestDefs/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestDescription/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestEnforcerCases/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestID/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestKeywordsValidation/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestNestedDefsDepth/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestNumberFormat/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestRangeConstraints/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestRefInProperties/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestReferences/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestRequired/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestSingleTypeInArray/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestTypeLocation/valid.jsonl", + } + selected_files = {} + archive_roots = set() + member_count = 0 + archive_size = 0 + selected_size = 0 + + with tarfile.open(fileobj=archive_file, mode="r|gz") as archive: + for member in archive: + member_count += 1 + if member_count > 100_000: + raise ValueError("archive contains more than 100000 members") + if member.size < 0: + raise ValueError( + f"archive member has a negative size: {member.name!r}" + ) + archive_size += member.size + if archive_size > 512 * 1024 * 1024: + raise ValueError("expanded archive exceeds the 512 MiB safety limit") + + parts = archive_member_parts(member.name) + archive_roots.add(parts[0]) + if len(archive_roots) > 1: + roots = ", ".join(sorted(archive_roots)) + raise ValueError(f"archive has multiple roots: {roots}") + if not (member.isdir() or member.isfile()): + raise ValueError( + f"archive member has unsafe type: {member.name!r}" + ) + if len(parts) == 1: + continue + + relative_path = "/".join(parts[1:]) + if relative_path not in required_files: + continue + if relative_path in selected_files: + raise ValueError( + f"archive contains duplicate selected path: {relative_path!r}" + ) + if not member.isfile(): + raise ValueError( + f"required path is not a regular file: {relative_path}" + ) + selected_size += member.size + if selected_size > 256 * 1024 * 1024: + raise ValueError( + "selected archive subset exceeds the 256 MiB safety limit" + ) + source = archive.extractfile(member) + if source is None: + raise ValueError(f"could not read archive member: {member.name!r}") + with source: + content = source.read(member.size + 1) + if len(content) != member.size: + raise ValueError( + f"archive member size mismatch: {member.name!r}" + ) + selected_files[relative_path] = content + + if member_count == 0: + raise ValueError("archive contains no members") + if len(archive_roots) != 1: + raise ValueError("archive does not have exactly one root") + missing_files = sorted(required_files - selected_files.keys()) + if missing_files: + raise ValueError( + "archive is missing required files: " + ", ".join(missing_files) + ) + + stage = "extract the verified archive subset" + if any(checkout_dir.iterdir()): + raise ValueError(f"checkout directory is not empty: {checkout_dir}") + for relative_path, content in selected_files.items(): + destination = checkout_dir.joinpath(*relative_path.split("/")) + destination.parent.mkdir(parents=True, exist_ok=True) + with destination.open("xb") as output: + output.write(content) +except Exception as error: + print( + f"ERROR: failed to {stage} for Kimi-Vendor-Verifier " + f"at {verifier_ref}: {error}", + file=sys.stderr, + ) + raise SystemExit(1) +PY + + if [ "$prepare_rc" -ne 0 ]; then + if ! rm -rf "$checkout_dir"; then + echo "ERROR: failed to remove partial Kimi-Vendor-Verifier directory ${checkout_dir}" >&2 + fi + return "$prepare_rc" fi + printf '%s\n' "$checkout_dir" } diff --git a/utils/collect_eval_results.py b/utils/collect_eval_results.py index 28d9ab11c9..545ef3a855 100644 --- a/utils/collect_eval_results.py +++ b/utils/collect_eval_results.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import sys import json +import math import re from pathlib import Path from typing import Any, Dict, List, Optional, Tuple @@ -72,10 +73,10 @@ def result_concurrency(path: Path) -> Optional[int]: def detect_lm_eval_jsons(d: Path, batched: bool = False) -> List[Path]: - """Return collector-compatible eval result JSONs from one artifact directory. + """Return the latest collector-compatible eval result JSONs. - Legacy lm-eval artifacts contribute their latest result file. Batched - artifacts contribute the latest result file for each `_concN` suffix. + Result filenames contain sortable timestamps. Mtime remains a fallback for + legacy names, with the filename as a deterministic tie-breaker. """ immediate_jsons = set(d.glob('results*.json')) immediate_jsons.update( @@ -83,6 +84,17 @@ def detect_lm_eval_jsons(d: Path, batched: bool = False) -> List[Path]: ) lm_paths = [] + def recency_key(path: Path) -> Tuple[str, int, str]: + match = re.search( + r"\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}(?:\.\d+)?", + path.name, + ) + return ( + match.group(0) if match else "", + path.stat().st_mtime_ns, + path.name, + ) + for p in immediate_jsons: data = load_json(p) if not isinstance(data, dict): @@ -93,7 +105,7 @@ def detect_lm_eval_jsons(d: Path, batched: bool = False) -> List[Path]: if not lm_paths: return [] if not batched: - return [max(lm_paths, key=lambda path: path.stat().st_mtime)] + return [max(lm_paths, key=recency_key)] latest_by_conc: Dict[int, Path] = {} for path in lm_paths: @@ -101,15 +113,28 @@ def detect_lm_eval_jsons(d: Path, batched: bool = False) -> List[Path]: if conc is None: continue current = latest_by_conc.get(conc) - if current is None or path.stat().st_mtime > current.stat().st_mtime: + if current is None or recency_key(path) > recency_key(current): latest_by_conc[conc] = path return [latest_by_conc[conc] for conc in sorted(latest_by_conc)] -def detect_eval_jsons(d: Path) -> Tuple[Optional[Path], Optional[Path]]: - """Return the latest legacy lm-eval JSON and deprecated second slot.""" - lm_paths = detect_lm_eval_jsons(d) - return (lm_paths[0] if lm_paths else None), None +def has_invalid_effective_count(data: Dict[str, Any], task: str) -> bool: + """Return whether a task has an explicitly invalid effective count.""" + if 'n-samples' not in data: + return False + sample_counts = data['n-samples'] + if not isinstance(sample_counts, dict) or task not in sample_counts: + return True + task_samples = sample_counts[task] + if not isinstance(task_samples, dict) or 'effective' not in task_samples: + return True + effective = task_samples['effective'] + return ( + isinstance(effective, bool) + or not isinstance(effective, (int, float)) + or not math.isfinite(effective) + or effective <= 0 + ) def extract_lm_metrics(json_path: Path) -> List[Dict[str, Any]]: @@ -124,6 +149,8 @@ def extract_lm_metrics(json_path: Path) -> List[Dict[str, Any]]: - Values from results[task][metric,filter] """ data = load_json(json_path) or {} + if 'integration_error' in data: + return [] results = data.get('results', {}) configs = data.get('configs', {}) @@ -133,6 +160,8 @@ def extract_lm_metrics(json_path: Path) -> List[Dict[str, Any]]: extracted = [] for task in results.keys(): + if has_invalid_effective_count(data, task): + continue task_results = results[task] task_config = configs.get(task, {}) diff --git a/utils/evals/EVALS.md b/utils/evals/EVALS.md index d5de674c62..74c4974c1c 100644 --- a/utils/evals/EVALS.md +++ b/utils/evals/EVALS.md @@ -75,10 +75,11 @@ launch their existing `*_mtp.sh` server instead of silently falling back to STP. The smoke runs the unmodified [MoonshotAI/Kimi-Vendor-Verifier](https://github.com/MoonshotAI/Kimi-Vendor-Verifier) -at commit `b9ed3a6665bdff2c943246f7d2903cd003d6ddd6`. Each run creates a fresh -sparse checkout containing the upstream pytest configuration, tool-call schema -tests, and bundled Walle cases. InferenceX does not install the verifier package -or reimplement its request, streaming, retry, or validation logic. +at commit `b9ed3a6665bdff2c943246f7d2903cd003d6ddd6`. Each run downloads the +fresh pinned GitHub source archive and safely extracts only the upstream pytest +configuration, tool-call schema tests, and bundled Walle cases. InferenceX does +not install the verifier package or reimplement its request, streaming, retry, +or validation logic. Python 3.12 or newer is required. The runner installs the minimal pinned runtime (`httpx[http2]`, `openai`, `jsonschema`, and `pytest`) into a temporary isolated @@ -137,7 +138,7 @@ Key eval functions in `benchmarks/benchmark_lib.sh`: | `append_lm_eval_summary` | Writes `meta_env.json` and moves eval artifacts to workspace | | `_install_lm_eval_deps` | Installs lm-eval dependencies | | `_prepare_kimi_vendor_runtime` | Installs the pinned verifier dependencies in an isolated temp path | -| `_prepare_kimi_vendor_verifier` | Fetches a fresh pinned sparse checkout | +| `_prepare_kimi_vendor_verifier` | Downloads and safely extracts a fresh subset of the pinned source archive | | `_patch_lm_eval` | Patches lm-eval for reasoning tokens and TRT compatibility | | `compute_eval_context_length` | Computes eval context length (requested benchmark context, capped at model native max) | | `get_native_max_context_length` | Extracts model's native max context length from HF config | diff --git a/utils/evals/test_batched_eval.py b/utils/evals/test_batched_eval.py index a5d6df0085..56c219e558 100644 --- a/utils/evals/test_batched_eval.py +++ b/utils/evals/test_batched_eval.py @@ -227,6 +227,106 @@ def test_validate_scores_checks_threshold_for_every_concurrency( assert "FAIL: [conc=4] gsm8k exact_match,strict-match" in captured.err +def test_validate_scores_reports_integration_failure_without_thresholding( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + result_path = tmp_path / "results_test.json" + result_path.write_text(json.dumps({ + "integration_error": { + "type": "RuntimeError", + "message": "vendor verifier checkout failed", + }, + "results": { + "gsm8k": { + "exact_match,strict-match": 0.0, + }, + }, + "n-samples": {"gsm8k": {"effective": 0}}, + })) + monkeypatch.setattr(sys, "argv", [ + "validate_scores.py", + "--meta-env", + str(tmp_path / "meta_env.json"), + "--results-glob", + str(result_path), + ]) + + assert validate_scores_main() == 1 + captured = capsys.readouterr() + assert "integration failure: RuntimeError: vendor verifier checkout failed" in captured.err + assert "gsm8k exact_match,strict-match" not in captured.err + + +def test_validate_scores_rejects_invalid_effective_count_without_thresholding( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + result_path = tmp_path / "results_test.json" + result_path.write_text(json.dumps({ + "results": { + "gsm8k": { + "exact_match,strict-match": 0.0, + }, + "other": { + "exact_match,strict-match": 0.0, + }, + "nonfinite": { + "exact_match,strict-match": 1.0, + }, + }, + "n-samples": { + "gsm8k": {"effective": "unknown"}, + "other": {"effective": 0}, + "nonfinite": {"effective": float("inf")}, + }, + })) + monkeypatch.setattr(sys, "argv", [ + "validate_scores.py", + "--meta-env", + str(tmp_path / "meta_env.json"), + "--results-glob", + str(result_path), + ]) + + assert validate_scores_main() == 1 + captured = capsys.readouterr() + assert "gsm8k invalid effective sample count: 'unknown'" in captured.err + assert "gsm8k exact_match,strict-match" not in captured.err + assert "other invalid effective sample count: 0" in captured.err + assert "other exact_match,strict-match" not in captured.err + + assert "nonfinite invalid effective sample count: inf" in captured.err + assert "nonfinite exact_match,strict-match" not in captured.err + +def test_validate_scores_accepts_legacy_result_without_effective_count( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + result_path = tmp_path / "results_test.json" + result_path.write_text(json.dumps({ + "results": { + "gsm8k": { + "exact_match,strict-match": 1.0, + }, + }, + })) + monkeypatch.setattr(sys, "argv", [ + "validate_scores.py", + "--meta-env", + str(tmp_path / "meta_env.json"), + "--results-glob", + str(result_path), + ]) + + assert validate_scores_main() == 0 + captured = capsys.readouterr() + assert "PASS: gsm8k exact_match,strict-match" in captured.out + + def test_amd_multinode_container_forwards_eval_concurrency_list() -> None: job_slurm = ( Path(__file__).resolve().parents[2] diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index bc235c094f..6344ee9247 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -1,10 +1,16 @@ from __future__ import annotations +import io import json import os +import re import stat import subprocess +import tarfile +import threading +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path import yaml @@ -317,6 +323,166 @@ def test_kimi_vendor_setup_failure_writes_compatibility_result( assert not (results_dir / "kimi_vendor_report.json").exists() +_KIMI_VERIFIER_REQUIRED_FILES = { + "pyproject.toml", + "tests/conftest.py", + "tests/__init__.py", + "tests/tool_call_json_schema/conftest.py", + "tests/tool_call_json_schema/__init__.py", + "tests/tool_call_json_schema/test_tool_call_json_schema.py", + "tests/tool_call_json_schema/validator.py", + *{ + f"testdata/walle_validator_cases/validator_cases/{case}/valid.jsonl" + for case in ( + "TestAdditionalProperties", + "TestAnyOf", + "TestBasicTypes", + "TestDefs", + "TestDescription", + "TestEnforcerCases", + "TestID", + "TestKeywordsValidation", + "TestNestedDefsDepth", + "TestNumberFormat", + "TestRangeConstraints", + "TestRefInProperties", + "TestReferences", + "TestRequired", + "TestSingleTypeInArray", + "TestTypeLocation", + ) + }, +} + + +def _kimi_verifier_archive( + *, + missing: str | None = None, + unsafe_member: tarfile.TarInfo | None = None, +) -> bytes: + output = io.BytesIO() + with tarfile.open(fileobj=output, mode="w:gz") as archive: + for relative_path in sorted(_KIMI_VERIFIER_REQUIRED_FILES - {missing}): + payload = relative_path.encode() + member = tarfile.TarInfo(f"verifier-pinned/{relative_path}") + member.size = len(payload) + archive.addfile(member, io.BytesIO(payload)) + extra = b"must not be extracted" + member = tarfile.TarInfo("verifier-pinned/README.md") + member.size = len(extra) + archive.addfile(member, io.BytesIO(extra)) + if unsafe_member is not None: + archive.addfile( + unsafe_member, + io.BytesIO(b"unsafe") if unsafe_member.isfile() else None, + ) + return output.getvalue() + + +@contextmanager +def _serve_archive(payload: bytes): + request_paths = [] + class ArchiveHandler(BaseHTTPRequestHandler): + def do_GET(self): + request_paths.append(self.path) + self.send_response(200) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), ArchiveHandler) + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + yield ( + f"http://127.0.0.1:{server.server_port}/owner/verifier.git", + request_paths, + ) + finally: + server.shutdown() + thread.join() + server.server_close() + + +def _prepare_local_kimi_verifier( + tmp_path: Path, + payload: bytes, + verifier_ref: str = "1" * 40, +) -> tuple[subprocess.CompletedProcess[str], Path, list[str]]: + checkout = tmp_path / "checkout" + script = r''' +source "$BENCHMARK_LIB" +git() { echo "git must not be invoked" >&2; return 127; } +mktemp() { mkdir "$CHECKOUT"; printf '%s\n' "$CHECKOUT"; } +_prepare_kimi_vendor_verifier "$REPO_URL" "$VERIFIER_REF" +''' + with _serve_archive(payload) as (repo_url, request_paths): + result = subprocess.run( + ["bash", "-c", script], + env={ + **os.environ, + "BENCHMARK_LIB": str(BENCHMARK_LIB), + "CHECKOUT": str(checkout), + "REPO_URL": repo_url, + "VERIFIER_REF": verifier_ref, + }, + text=True, + capture_output=True, + ) + return result, checkout, request_paths + + +def test_kimi_vendor_verifier_fetches_expected_subset_without_git(tmp_path: Path) -> None: + result, checkout, request_paths = _prepare_local_kimi_verifier( + tmp_path, + _kimi_verifier_archive(), + ) + verifier_ref = "1" * 40 + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == str(checkout) + assert request_paths == [ + f"/owner/verifier/archive/{verifier_ref}.tar.gz", + ] + assert { + path.relative_to(checkout).as_posix() + for path in checkout.rglob("*") + if path.is_file() + } == _KIMI_VERIFIER_REQUIRED_FILES + assert "git must not be invoked" not in result.stderr + + +def test_kimi_vendor_verifier_removes_partial_checkout_when_member_missing( + tmp_path: Path, +) -> None: + missing = "tests/tool_call_json_schema/validator.py" + result, checkout, _ = _prepare_local_kimi_verifier( + tmp_path, + _kimi_verifier_archive(missing=missing), + ) + + assert result.returncode == 1 + assert missing in result.stderr + assert not checkout.exists() + + +def test_kimi_vendor_verifier_rejects_unsafe_archive_members(tmp_path: Path) -> None: + unsafe = tarfile.TarInfo("verifier-pinned/../../escaped") + unsafe.size = len(b"unsafe") + result, checkout, _ = _prepare_local_kimi_verifier( + tmp_path, + _kimi_verifier_archive(unsafe_member=unsafe), + ) + + assert result.returncode == 1 + assert "unsafe archive member path" in result.stderr + assert not checkout.exists() + assert not (tmp_path / "escaped").exists() + + def test_kimi_vendor_dependency_install_is_isolated(tmp_path: Path) -> None: runtime_dir = tmp_path / "runtime" script = r''' @@ -1017,6 +1183,185 @@ def test_agentic_eval_limit_full_runs_whole_split(tmp_path): assert "GEN_RC=0" in res.stdout, res.stdout + res.stderr +def test_multinode_eval_artifact_names_are_bounded_and_distinct() -> None: + workflow = yaml.safe_load(MULTINODE_WORKFLOW.read_text()) + upload = next( + step + for step in workflow["jobs"]["benchmark"]["steps"] + if step.get("name") == "Upload eval results (if any)" + ) + expression = upload["with"]["name"] + assert expression.startswith("eval_") + assert "RESULT_FILENAME" not in expression + + targets = [ + { + "EXP_NAME": "kimik3_p2x16ep32dpa_d0x16ep32dpa_conc12", + "PRECISION": "fp4", + "FRAMEWORK": "vllm", + "PREFILL_NUM_WORKERS": "2", + "PREFILL_TP": "16", + "PREFILL_PP_SIZE": "1", + "SPEC_DECODING": "mtp", + "PREFILL_DCP_SIZE": "1", + "PREFILL_PCP_SIZE": "1", + "PREFILL_EP": "32", + "PREFILL_DP_ATTN": "true", + "DECODE_NUM_WORKERS": "0", + "DECODE_TP": "16", + "DECODE_PP_SIZE": "1", + "DECODE_DCP_SIZE": "1", + "DECODE_PCP_SIZE": "1", + "DECODE_EP": "32", + "DECODE_DP_ATTN": "true", + "KV_OFFLOADING": "none", + "KV_OFFLOAD_BACKEND": "", + "conc-list": ["1", "12", "16"], + "runner.name": "h200-dgxc-slurm_00", + }, + { + "EXP_NAME": "kimik3_p4x8ep32dpa_d0x8ep32dpa_conc12", + "PRECISION": "fp4", + "FRAMEWORK": "vllm", + "PREFILL_NUM_WORKERS": "4", + "PREFILL_TP": "8", + "PREFILL_PP_SIZE": "1", + "PREFILL_DCP_SIZE": "1", + "PREFILL_PCP_SIZE": "1", + "PREFILL_EP": "32", + "PREFILL_DP_ATTN": "true", + "DECODE_NUM_WORKERS": "0", + "SPEC_DECODING": "mtp", + "DECODE_TP": "8", + "DECODE_PP_SIZE": "1", + "DECODE_DCP_SIZE": "1", + "DECODE_PCP_SIZE": "1", + "DECODE_EP": "32", + "DECODE_DP_ATTN": "true", + "KV_OFFLOADING": "none", + "KV_OFFLOAD_BACKEND": "", + "conc-list": ["1", "12", "16"], + "runner.name": "h200-dgxc-slurm_01", + }, + { + "EXP_NAME": "kimik3_p4x8ep32dpa_d0x8ep32dpa_conc12_kvdram-vllm-simple", + "PRECISION": "fp4", + "FRAMEWORK": "vllm", + "PREFILL_NUM_WORKERS": "4", + "PREFILL_TP": "8", + "PREFILL_PP_SIZE": "1", + "PREFILL_DCP_SIZE": "1", + "PREFILL_PCP_SIZE": "1", + "PREFILL_EP": "32", + "PREFILL_DP_ATTN": "true", + "DECODE_NUM_WORKERS": "0", + "DECODE_TP": "8", + "DECODE_PP_SIZE": "1", + "DECODE_DCP_SIZE": "1", + "DECODE_PCP_SIZE": "1", + "DECODE_EP": "32", + "SPEC_DECODING": "mtp", + "DECODE_DP_ATTN": "true", + "KV_OFFLOADING": "dram", + "KV_OFFLOAD_BACKEND": "vllm-simple", + "conc-list": ["1", "12", "16"], + "runner.name": "h200-dgxc-slurm_02", + }, + { + "EXP_NAME": "kimik3_p1x8_d0x8_conc16", + "PRECISION": "fp4", + "FRAMEWORK": "dynamo-vllm", + "PREFILL_NUM_WORKERS": "1", + "PREFILL_TP": "8", + "PREFILL_PP_SIZE": "2", + "PREFILL_DCP_SIZE": "1", + "PREFILL_PCP_SIZE": "1", + "PREFILL_EP": "1", + "PREFILL_DP_ATTN": "false", + "DECODE_NUM_WORKERS": "0", + "DECODE_TP": "8", + "DECODE_PP_SIZE": "2", + "DECODE_DCP_SIZE": "1", + "DECODE_PCP_SIZE": "1", + "DECODE_EP": "1", + "DECODE_DP_ATTN": "false", + "SPEC_DECODING": "mtp", + "KV_OFFLOADING": "none", + "KV_OFFLOAD_BACKEND": "", + "conc-list": ["1", "16"], + "runner.name": "b200-dgxc_00", + }, + { + "EXP_NAME": "kimik3_p1x16ep16_d0x16ep16_conc16", + "PRECISION": "fp4", + "FRAMEWORK": "dynamo-vllm", + "PREFILL_NUM_WORKERS": "1", + "PREFILL_TP": "16", + "PREFILL_PP_SIZE": "1", + "PREFILL_DCP_SIZE": "1", + "PREFILL_PCP_SIZE": "1", + "PREFILL_EP": "16", + "PREFILL_DP_ATTN": "false", + "DECODE_NUM_WORKERS": "0", + "DECODE_TP": "16", + "DECODE_PP_SIZE": "1", + "DECODE_DCP_SIZE": "1", + "DECODE_PCP_SIZE": "1", + "DECODE_EP": "16", + "DECODE_DP_ATTN": "false", + "KV_OFFLOADING": "none", + "SPEC_DECODING": "mtp", + "KV_OFFLOAD_BACKEND": "", + "conc-list": ["16"], + "runner.name": "gb200-nv_00", + }, + { + "EXP_NAME": "kimik3_p1x16_d0x16_conc1", + "PRECISION": "fp4", + "FRAMEWORK": "dynamo-vllm", + "PREFILL_NUM_WORKERS": "1", + "PREFILL_TP": "16", + "PREFILL_PP_SIZE": "1", + "PREFILL_DCP_SIZE": "1", + "PREFILL_PCP_SIZE": "1", + "PREFILL_EP": "1", + "PREFILL_DP_ATTN": "false", + "DECODE_NUM_WORKERS": "0", + "DECODE_TP": "16", + "DECODE_PP_SIZE": "1", + "DECODE_DCP_SIZE": "1", + "DECODE_PCP_SIZE": "1", + "DECODE_EP": "1", + "DECODE_DP_ATTN": "false", + "KV_OFFLOADING": "none", + "KV_OFFLOAD_BACKEND": "", + "SPEC_DECODING": "mtp", + "conc-list": ["1"], + "runner.name": "gb200-nv_01", + }, + ] + non_mtp_twin = {**targets[3], "SPEC_DECODING": "none"} + + def render(values: dict[str, object]) -> str: + name = expression + name = re.sub( + r"\$\{\{ join\(fromJson\(inputs\.conc-list\), 'x'\) \}\}", + "x".join(values["conc-list"]), + name, + ) + for key, value in values.items(): + if key != "conc-list": + name = name.replace(f"${{{{ env.{key} }}}}", str(value)) + name = name.replace("${{ runner.name }}", str(values["runner.name"])) + assert "${{" not in name + return name + + names = [render(target) for target in targets] + assert len(names) == len(set(names)) == 6 + assert render(targets[3]) != render(non_mtp_twin) + assert all(name.startswith("eval_") and len(name.encode()) <= 256 for name in names) + + _GENMODE_SCRIPT = r''' source "$BENCHMARK_LIB" 2>/dev/null diff --git a/utils/evals/validate_scores.py b/utils/evals/validate_scores.py index ba7fc13962..bf4b391728 100644 --- a/utils/evals/validate_scores.py +++ b/utils/evals/validate_scores.py @@ -5,6 +5,7 @@ import argparse import glob import json +import math import os import re import sys @@ -68,6 +69,34 @@ def resolve_threshold(config: dict, prefix: str | None, task: str, fallback: flo return default[task], "default" return fallback, "min-score" +def invalid_effective_count(data: dict, task: str) -> tuple[bool, object]: + """Return whether an explicitly present effective count is invalid.""" + if "n-samples" not in data: + return False, None + sample_counts = data["n-samples"] + if not isinstance(sample_counts, dict) or task not in sample_counts: + return True, sample_counts + task_samples = sample_counts[task] + if not isinstance(task_samples, dict) or "effective" not in task_samples: + return True, task_samples + effective = task_samples["effective"] + invalid = ( + isinstance(effective, bool) + or not isinstance(effective, (int, float)) + or not math.isfinite(effective) + or effective <= 0 + ) + return invalid, effective + + +def integration_error_message(error: object) -> str: + """Render the structured integration error fields for a direct failure.""" + if isinstance(error, dict): + error_type = error.get("type", "unknown") + message = error.get("message", "") + return f"{error_type}: {message}" + return f"unknown: {error}" + def validate_batch_manifest( meta_env_path: str, @@ -277,7 +306,24 @@ def main() -> int: conc_label = f"[conc={match.group(1)}] " if match else "" with open(f) as fh: data = json.load(fh) + if "integration_error" in data: + print( + f"FAIL: {conc_label}integration failure: " + f"{integration_error_message(data['integration_error'])}", + file=sys.stderr, + ) + failed = True + continue for task, metrics in data.get("results", {}).items(): + invalid_effective, effective = invalid_effective_count(data, task) + if invalid_effective: + print( + f"FAIL: {conc_label}{task} invalid effective sample count: " + f"{effective!r}", + file=sys.stderr, + ) + failed = True + continue min_score, source = resolve_threshold(config, prefix, task, args.min_score) for name, val in metrics.items(): if not name.startswith(args.metric_prefix) or "stderr" in name: diff --git a/utils/test_collect_eval_results.py b/utils/test_collect_eval_results.py index 41cd5cf3d0..3cac4d6b48 100644 --- a/utils/test_collect_eval_results.py +++ b/utils/test_collect_eval_results.py @@ -1,14 +1,12 @@ """Tests for eval result aggregation.""" import json -import sys from pathlib import Path from collect_eval_results import ( EVAL_RESULT_FORMAT, build_row, collect_eval_rows, - main as collect_main, ) from evals.kimi_vendor_eval import RESULT_FORMAT as KIMI_VENDOR_RESULT_FORMAT @@ -153,39 +151,76 @@ def test_collect_eval_rows_accepts_neutral_result_format(tmp_path: Path) -> None assert rows[0]["eval_suite"] == "provider_smoke" -def test_main_renders_zero_effective_samples( +def test_collect_eval_rows_excludes_integration_and_sample_failures( tmp_path: Path, - monkeypatch, - capsys, ) -> None: - for name, is_multinode in (("single", False), ("multi", True)): + for name, invalid in ( + ("integration", "integration"), + ("zero", 0), + ("nonnumeric", "unknown"), + ("nonfinite", float("nan")), + ("malformed", []), + ): artifact_dir = tmp_path / f"eval_{name}" artifact_dir.mkdir() (artifact_dir / "meta_env.json").write_text(json.dumps({ - "is_multinode": is_multinode, "eval_suite": "gsm8k", })) result_path = artifact_dir / f"results_{name}.json" _write_lm_eval_result(result_path, 0.0) result = json.loads(result_path.read_text()) - result["n-samples"]["gsm8k"]["effective"] = 0 + if invalid == "integration": + result["integration_error"] = { + "type": "RuntimeError", + "message": "vendor verifier checkout failed", + } + else: + result["n-samples"]["gsm8k"]["effective"] = invalid result_path.write_text(json.dumps(result)) - monkeypatch.chdir(tmp_path) - monkeypatch.setattr( - sys, - "argv", - ["collect_eval_results.py", str(tmp_path), "zero-samples"], - ) + assert collect_eval_rows(tmp_path) == [] + + +def test_collect_eval_rows_accepts_legacy_missing_effective_count( + tmp_path: Path, +) -> None: + artifact_dir = tmp_path / "eval_legacy" + artifact_dir.mkdir() + (artifact_dir / "meta_env.json").write_text(json.dumps({ + "eval_suite": "gsm8k", + })) + result_path = artifact_dir / "results_legacy.json" + _write_lm_eval_result(result_path, 0.9) + result = json.loads(result_path.read_text()) + result.pop("n-samples") + result_path.write_text(json.dumps(result)) - collect_main() - - task_rows = [ - line - for line in capsys.readouterr().out.splitlines() - if "| gsm8k " in line - ] - assert len(task_rows) == 2 - for row in task_rows: - cells = [cell.strip() for cell in row.split("|")[1:-1]] - assert cells[-2] == "0" \ No newline at end of file + rows = collect_eval_rows(tmp_path) + + assert len(rows) == 1 + assert rows[0]["score"] == 0.9 + assert rows[0]["n_eff"] is None + + +def test_collect_eval_rows_does_not_resurrect_stale_valid_result( + tmp_path: Path, +) -> None: + artifact_dir = tmp_path / "eval_retry" + artifact_dir.mkdir() + (artifact_dir / "meta_env.json").write_text(json.dumps({ + "eval_suite": "gsm8k", + })) + stale_path = artifact_dir / "results_older.json" + _write_lm_eval_result(stale_path, 1.0) + current_path = artifact_dir / "results_current.json" + _write_lm_eval_result(current_path, 0.0) + result = json.loads(current_path.read_text()) + result["integration_error"] = { + "type": "RuntimeError", + "message": "vendor verifier checkout failed", + } + current_path.write_text(json.dumps(result)) + stale_path.touch() + current_path.touch() + + assert collect_eval_rows(tmp_path) == [] \ No newline at end of file From 134906e56d97408b8b457f06e60fce6fc16030ba Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Tue, 11 Aug 2026 23:36:04 -0500 Subject: [PATCH 18/24] test: capture Kimi response diagnostics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：捕获 Kimi 工具调用响应诊断数据。 --- .../workflows/benchmark-multinode-tmpl.yml | 7 + .github/workflows/e2e-tests.yml | 11 ++ benchmarks/benchmark_lib.sh | 5 + utils/evals/kimi_vendor_eval.py | 157 +++++++++++++++++- utils/evals/test_kimi_vendor_eval.py | 67 ++++++++ 5 files changed, 246 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 7004faed99..4f2462d137 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -148,6 +148,11 @@ on: type: string required: false default: "" + kimi-vendor-diagnostic: + description: "Capture ordered raw Kimi tool-call responses" + type: boolean + required: false + default: false eval-limit: description: "Eval instance count: empty/full = whole split (default); N = first-N smoke slice" required: false @@ -246,6 +251,7 @@ env: EVAL_SUITE: ${{ inputs.eval-suite }} EVAL_CONC: ${{ inputs.eval-conc }} EVAL_LIMIT: ${{ inputs.eval-limit }} + KIMI_VENDOR_DIAGNOSTIC: ${{ inputs.kimi-vendor-diagnostic && '1' || '0' }} SWEBENCH_GEN_MODE: ${{ inputs.swebench-gen-mode }} # GPU/multi-node runners lack Docker for SWE-bench scoring. SWEBENCH_USE_MODAL: 'true' @@ -484,6 +490,7 @@ jobs: meta_env.json results*.json *_vendor_report.json + eval_kimi_tool_call_diagnostic.json sample*.jsonl agent_preds.json predictions.jsonl diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index a0a31b5e91..8d354e6027 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -55,6 +55,11 @@ on: required: false type: string default: "" + kimi-vendor-diagnostic: + description: "Capture ordered raw Kimi tool-call responses" + required: false + type: boolean + default: false swebench-gen-mode: description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)." required: false @@ -145,6 +150,11 @@ on: required: false type: string default: "" + kimi-vendor-diagnostic: + description: "Capture ordered raw Kimi tool-call responses" + required: false + type: boolean + default: false swebench-gen-mode: description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)." required: false @@ -601,6 +611,7 @@ jobs: swebench-gen-mode: ${{ inputs.swebench-gen-mode }} eval-framework: ${{ inputs.eval-framework }} eval-suite: ${{ inputs.eval-suite }} + kimi-vendor-diagnostic: ${{ inputs.kimi-vendor-diagnostic }} scenario-type: agentic-coding ref: ${{ inputs.ref }} diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index cef240b274..21bae9908e 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1150,6 +1150,10 @@ _run_kimi_tool_call_schema_eval() { fi local eval_rc=0 + local diagnostic_args=() + if [ "${KIMI_VENDOR_DIAGNOSTIC:-0}" = "1" ]; then + diagnostic_args=(--diagnostic) + fi PYTHONPATH="${runtime_dir}${PYTHONPATH:+:${PYTHONPATH}}" \ python3 "$adapter_path" \ --verifier-dir "$checkout_dir" \ @@ -1157,6 +1161,7 @@ _run_kimi_tool_call_schema_eval() { --api-key EMPTY \ --model "$model_name" \ --output-dir "$results_dir" \ + "${diagnostic_args[@]}" \ || eval_rc=$? _cleanup_kimi_vendor_eval "$runtime_dir" "$checkout_dir" return "$eval_rc" diff --git a/utils/evals/kimi_vendor_eval.py b/utils/evals/kimi_vendor_eval.py index 4f3debc6fb..9f791b318e 100755 --- a/utils/evals/kimi_vendor_eval.py +++ b/utils/evals/kimi_vendor_eval.py @@ -4,9 +4,10 @@ from __future__ import annotations import argparse +import importlib import json -import subprocess import sys +import subprocess from collections.abc import Mapping, Sequence from datetime import datetime, timezone from pathlib import Path @@ -19,6 +20,145 @@ DEFAULT_TIMEOUT_SECONDS = 900 RESULT_FORMAT = "inferencex-eval-v1" ADAPTER_NAME = "kimi-vendor-verifier" +DIAGNOSTIC_REPORT_FILENAME = "eval_kimi_tool_call_diagnostic.json" +DIAGNOSTIC_MODES = ( + "non-stream", + "non-stream", + "stream", + "stream", + "stream", + "non-stream", +) + + +def _jsonable(value: Any) -> Any: + """Convert an OpenAI response model into JSON-compatible data.""" + if hasattr(value, "model_dump"): + return value.model_dump(mode="json") + if isinstance(value, Mapping): + return {str(key): _jsonable(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_jsonable(item) for item in value] + if value is None or isinstance(value, (bool, int, float, str)): + return value + return repr(value) + + +class _CapturingStream: + """Record stream chunks while preserving the upstream iterator contract.""" + + def __init__(self, stream: Any, chunks: list[Any]) -> None: + self._stream = stream + self._chunks = chunks + + def __iter__(self): + for chunk in self._stream: + self._chunks.append(_jsonable(chunk)) + yield chunk + + +class _CapturingCompletions: + """Delegate OpenAI requests while retaining their raw responses.""" + + def __init__(self, completions: Any, record: dict[str, Any]) -> None: + self._completions = completions + self._record = record + + def create(self, **request: Any) -> Any: + response = self._completions.create(**request) + if request.get("stream"): + chunks: list[Any] = [] + self._record["raw_chunks"] = chunks + return _CapturingStream(response, chunks) + self._record["raw_response"] = _jsonable(response) + return response + + +class _CapturingClient: + """Expose the OpenAI chat interface expected by the upstream helper.""" + + def __init__(self, client: Any, record: dict[str, Any]) -> None: + chat_type = type("_CapturingChat", (), {}) + self.chat = chat_type() + self.chat.completions = _CapturingCompletions(client.chat.completions, record) + + +def run_diagnostic_sequence( + *, + verifier_dir: Path, + base_url: str, + api_key: str, + model: str, + output_dir: Path, +) -> None: + """Run ordered stock-helper requests and preserve raw response evidence.""" + sys.path.insert(0, str(verifier_dir)) + try: + validator = importlib.import_module("tests.tool_call_json_schema.validator") + cases = validator.load_cases( + verifier_dir / "testdata" / "walle_validator_cases" / "validator_cases" + ) + selected = validator.select_cases( + cases, + selection="object", + requested_cases=set(), + max_cases=1, + ) + if len(selected) != 1: + raise ValueError( + f"diagnostic expected one selected case, found {len(selected)}" + ) + case, schema, selection_reason = selected[0] + client = validator.make_client(base_url, api_key, 120) + records: list[dict[str, Any]] = [] + try: + for index, mode in enumerate(DIAGNOSTIC_MODES, start=1): + record: dict[str, Any] = {"index": index, "mode": mode} + response = validator.send_tool_schema( + _CapturingClient(client, record), + model, + schema, + 2048, + False, + "none", + stream=mode == "stream", + ) + valid, validation_message = validator.validate_arguments( + schema, response.arguments + ) + record.update( + { + "accepted": response.accepted, + "message": response.message, + "arguments": response.arguments, + "arguments_valid": valid, + "validation_message": validation_message, + "http_status": response.http_status, + "error_type": response.error_type, + } + ) + records.append(record) + finally: + client.close() + + report = { + "model": model, + "base_url": base_url, + "case": { + "suite": case.suite, + "line": case.line, + "selection_reason": selection_reason, + "schema": schema, + }, + "sequence": list(DIAGNOSTIC_MODES), + "results": records, + } + (output_dir / DIAGNOSTIC_REPORT_FILENAME).write_text( + json.dumps(report, indent=2) + "\n", + encoding="utf-8", + ) + finally: + sys.path.pop(0) def prepare_compatibility_path(output_dir: Path) -> Path: @@ -167,6 +307,7 @@ def run_evaluation( model: str, output_dir: Path, timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS, + diagnostic: bool = False, ) -> bool: """Run upstream pytest and always attempt to publish a compatibility result.""" output_dir.mkdir(parents=True, exist_ok=True) @@ -179,6 +320,14 @@ def run_evaluation( try: native_report.unlink(missing_ok=True) + if diagnostic: + run_diagnostic_sequence( + verifier_dir=verifier_dir, + base_url=base_url, + api_key=api_key, + model=model, + output_dir=output_dir, + ) completed = subprocess.run( build_pytest_command( base_url=base_url, @@ -237,6 +386,11 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: "--timeout-seconds", type=_positive_int, default=DEFAULT_TIMEOUT_SECONDS ) parser.add_argument("--integration-error") + parser.add_argument( + "--diagnostic", + action="store_true", + help="Run ordered raw request diagnostics before the unchanged verifier", + ) args = parser.parse_args(argv) if args.integration_error is None: missing = [ @@ -276,6 +430,7 @@ def main(argv: Sequence[str] | None = None) -> int: model=args.model, output_dir=args.output_dir, timeout_seconds=args.timeout_seconds, + diagnostic=args.diagnostic, ) return 0 if passed else 1 diff --git a/utils/evals/test_kimi_vendor_eval.py b/utils/evals/test_kimi_vendor_eval.py index e133d3b735..048b23d156 100644 --- a/utils/evals/test_kimi_vendor_eval.py +++ b/utils/evals/test_kimi_vendor_eval.py @@ -236,3 +236,70 @@ def test_cli_setup_failure_writes_zero_score_artifact(tmp_path: Path) -> None: assert _score(output_dir) == 0.0 assert projected["integration_error"]["message"] == "checkout failed" assert _n_eff(output_dir) == 0 + + +def test_diagnostic_sequence_captures_raw_responses( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + verifier_dir = tmp_path / "verifier" + case_dir = verifier_dir / "testdata" / "walle_validator_cases" / "validator_cases" + case_dir.mkdir(parents=True) + output_dir = tmp_path / "output" + output_dir.mkdir() + + class FakeResponse: + accepted = True + message = "tool call returned" + arguments = '{"value": {}}' + http_status = None + error_type = None + + class FakeClient: + def __init__(self) -> None: + self.chat = SimpleNamespace( + completions=SimpleNamespace(create=lambda **kwargs: None) + ) + + def close(self) -> None: + pass + + fake_validator = SimpleNamespace( + load_cases=lambda path: ["case"], + select_cases=lambda cases, **kwargs: [ + ( + SimpleNamespace(suite="TestAdditionalProperties", line=1), + {"type": "object"}, + "object_parameter_schema", + ) + ], + make_client=lambda *args: FakeClient(), + send_tool_schema=lambda client, + model, + schema, + max_tokens, + thinking, + think_mode, + *, + stream: FakeResponse(), + validate_arguments=lambda schema, arguments: (True, "valid"), + ) + monkeypatch.setattr( + kve.importlib, + "import_module", + lambda name: fake_validator, + ) + + kve.run_diagnostic_sequence( + verifier_dir=verifier_dir, + base_url="http://localhost/v1", + api_key="EMPTY", + model="model-a", + output_dir=output_dir, + ) + + report = json.loads((output_dir / kve.DIAGNOSTIC_REPORT_FILENAME).read_text()) + assert report["sequence"] == list(kve.DIAGNOSTIC_MODES) + assert [record["mode"] for record in report["results"]] == list( + kve.DIAGNOSTIC_MODES + ) + assert all(record["arguments_valid"] for record in report["results"]) From 23851fb80ff82729611b32f72cf5432f3d6d092d Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:30:45 -0500 Subject: [PATCH 19/24] Revert "test: capture Kimi response diagnostics" This reverts commit 134906e56d97408b8b457f06e60fce6fc16030ba. --- .../workflows/benchmark-multinode-tmpl.yml | 7 - .github/workflows/e2e-tests.yml | 11 -- benchmarks/benchmark_lib.sh | 5 - utils/evals/kimi_vendor_eval.py | 157 +----------------- utils/evals/test_kimi_vendor_eval.py | 67 -------- 5 files changed, 1 insertion(+), 246 deletions(-) diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 4f2462d137..7004faed99 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -148,11 +148,6 @@ on: type: string required: false default: "" - kimi-vendor-diagnostic: - description: "Capture ordered raw Kimi tool-call responses" - type: boolean - required: false - default: false eval-limit: description: "Eval instance count: empty/full = whole split (default); N = first-N smoke slice" required: false @@ -251,7 +246,6 @@ env: EVAL_SUITE: ${{ inputs.eval-suite }} EVAL_CONC: ${{ inputs.eval-conc }} EVAL_LIMIT: ${{ inputs.eval-limit }} - KIMI_VENDOR_DIAGNOSTIC: ${{ inputs.kimi-vendor-diagnostic && '1' || '0' }} SWEBENCH_GEN_MODE: ${{ inputs.swebench-gen-mode }} # GPU/multi-node runners lack Docker for SWE-bench scoring. SWEBENCH_USE_MODAL: 'true' @@ -490,7 +484,6 @@ jobs: meta_env.json results*.json *_vendor_report.json - eval_kimi_tool_call_diagnostic.json sample*.jsonl agent_preds.json predictions.jsonl diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 8d354e6027..a0a31b5e91 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -55,11 +55,6 @@ on: required: false type: string default: "" - kimi-vendor-diagnostic: - description: "Capture ordered raw Kimi tool-call responses" - required: false - type: boolean - default: false swebench-gen-mode: description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)." required: false @@ -150,11 +145,6 @@ on: required: false type: string default: "" - kimi-vendor-diagnostic: - description: "Capture ordered raw Kimi tool-call responses" - required: false - type: boolean - default: false swebench-gen-mode: description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)." required: false @@ -611,7 +601,6 @@ jobs: swebench-gen-mode: ${{ inputs.swebench-gen-mode }} eval-framework: ${{ inputs.eval-framework }} eval-suite: ${{ inputs.eval-suite }} - kimi-vendor-diagnostic: ${{ inputs.kimi-vendor-diagnostic }} scenario-type: agentic-coding ref: ${{ inputs.ref }} diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 21bae9908e..cef240b274 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1150,10 +1150,6 @@ _run_kimi_tool_call_schema_eval() { fi local eval_rc=0 - local diagnostic_args=() - if [ "${KIMI_VENDOR_DIAGNOSTIC:-0}" = "1" ]; then - diagnostic_args=(--diagnostic) - fi PYTHONPATH="${runtime_dir}${PYTHONPATH:+:${PYTHONPATH}}" \ python3 "$adapter_path" \ --verifier-dir "$checkout_dir" \ @@ -1161,7 +1157,6 @@ _run_kimi_tool_call_schema_eval() { --api-key EMPTY \ --model "$model_name" \ --output-dir "$results_dir" \ - "${diagnostic_args[@]}" \ || eval_rc=$? _cleanup_kimi_vendor_eval "$runtime_dir" "$checkout_dir" return "$eval_rc" diff --git a/utils/evals/kimi_vendor_eval.py b/utils/evals/kimi_vendor_eval.py index 9f791b318e..4f3debc6fb 100755 --- a/utils/evals/kimi_vendor_eval.py +++ b/utils/evals/kimi_vendor_eval.py @@ -4,10 +4,9 @@ from __future__ import annotations import argparse -import importlib import json -import sys import subprocess +import sys from collections.abc import Mapping, Sequence from datetime import datetime, timezone from pathlib import Path @@ -20,145 +19,6 @@ DEFAULT_TIMEOUT_SECONDS = 900 RESULT_FORMAT = "inferencex-eval-v1" ADAPTER_NAME = "kimi-vendor-verifier" -DIAGNOSTIC_REPORT_FILENAME = "eval_kimi_tool_call_diagnostic.json" -DIAGNOSTIC_MODES = ( - "non-stream", - "non-stream", - "stream", - "stream", - "stream", - "non-stream", -) - - -def _jsonable(value: Any) -> Any: - """Convert an OpenAI response model into JSON-compatible data.""" - if hasattr(value, "model_dump"): - return value.model_dump(mode="json") - if isinstance(value, Mapping): - return {str(key): _jsonable(item) for key, item in value.items()} - if isinstance(value, (list, tuple)): - return [_jsonable(item) for item in value] - if value is None or isinstance(value, (bool, int, float, str)): - return value - return repr(value) - - -class _CapturingStream: - """Record stream chunks while preserving the upstream iterator contract.""" - - def __init__(self, stream: Any, chunks: list[Any]) -> None: - self._stream = stream - self._chunks = chunks - - def __iter__(self): - for chunk in self._stream: - self._chunks.append(_jsonable(chunk)) - yield chunk - - -class _CapturingCompletions: - """Delegate OpenAI requests while retaining their raw responses.""" - - def __init__(self, completions: Any, record: dict[str, Any]) -> None: - self._completions = completions - self._record = record - - def create(self, **request: Any) -> Any: - response = self._completions.create(**request) - if request.get("stream"): - chunks: list[Any] = [] - self._record["raw_chunks"] = chunks - return _CapturingStream(response, chunks) - self._record["raw_response"] = _jsonable(response) - return response - - -class _CapturingClient: - """Expose the OpenAI chat interface expected by the upstream helper.""" - - def __init__(self, client: Any, record: dict[str, Any]) -> None: - chat_type = type("_CapturingChat", (), {}) - self.chat = chat_type() - self.chat.completions = _CapturingCompletions(client.chat.completions, record) - - -def run_diagnostic_sequence( - *, - verifier_dir: Path, - base_url: str, - api_key: str, - model: str, - output_dir: Path, -) -> None: - """Run ordered stock-helper requests and preserve raw response evidence.""" - sys.path.insert(0, str(verifier_dir)) - try: - validator = importlib.import_module("tests.tool_call_json_schema.validator") - cases = validator.load_cases( - verifier_dir / "testdata" / "walle_validator_cases" / "validator_cases" - ) - selected = validator.select_cases( - cases, - selection="object", - requested_cases=set(), - max_cases=1, - ) - if len(selected) != 1: - raise ValueError( - f"diagnostic expected one selected case, found {len(selected)}" - ) - case, schema, selection_reason = selected[0] - client = validator.make_client(base_url, api_key, 120) - records: list[dict[str, Any]] = [] - try: - for index, mode in enumerate(DIAGNOSTIC_MODES, start=1): - record: dict[str, Any] = {"index": index, "mode": mode} - response = validator.send_tool_schema( - _CapturingClient(client, record), - model, - schema, - 2048, - False, - "none", - stream=mode == "stream", - ) - valid, validation_message = validator.validate_arguments( - schema, response.arguments - ) - record.update( - { - "accepted": response.accepted, - "message": response.message, - "arguments": response.arguments, - "arguments_valid": valid, - "validation_message": validation_message, - "http_status": response.http_status, - "error_type": response.error_type, - } - ) - records.append(record) - finally: - client.close() - - report = { - "model": model, - "base_url": base_url, - "case": { - "suite": case.suite, - "line": case.line, - "selection_reason": selection_reason, - "schema": schema, - }, - "sequence": list(DIAGNOSTIC_MODES), - "results": records, - } - (output_dir / DIAGNOSTIC_REPORT_FILENAME).write_text( - json.dumps(report, indent=2) + "\n", - encoding="utf-8", - ) - finally: - sys.path.pop(0) def prepare_compatibility_path(output_dir: Path) -> Path: @@ -307,7 +167,6 @@ def run_evaluation( model: str, output_dir: Path, timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS, - diagnostic: bool = False, ) -> bool: """Run upstream pytest and always attempt to publish a compatibility result.""" output_dir.mkdir(parents=True, exist_ok=True) @@ -320,14 +179,6 @@ def run_evaluation( try: native_report.unlink(missing_ok=True) - if diagnostic: - run_diagnostic_sequence( - verifier_dir=verifier_dir, - base_url=base_url, - api_key=api_key, - model=model, - output_dir=output_dir, - ) completed = subprocess.run( build_pytest_command( base_url=base_url, @@ -386,11 +237,6 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: "--timeout-seconds", type=_positive_int, default=DEFAULT_TIMEOUT_SECONDS ) parser.add_argument("--integration-error") - parser.add_argument( - "--diagnostic", - action="store_true", - help="Run ordered raw request diagnostics before the unchanged verifier", - ) args = parser.parse_args(argv) if args.integration_error is None: missing = [ @@ -430,7 +276,6 @@ def main(argv: Sequence[str] | None = None) -> int: model=args.model, output_dir=args.output_dir, timeout_seconds=args.timeout_seconds, - diagnostic=args.diagnostic, ) return 0 if passed else 1 diff --git a/utils/evals/test_kimi_vendor_eval.py b/utils/evals/test_kimi_vendor_eval.py index 048b23d156..e133d3b735 100644 --- a/utils/evals/test_kimi_vendor_eval.py +++ b/utils/evals/test_kimi_vendor_eval.py @@ -236,70 +236,3 @@ def test_cli_setup_failure_writes_zero_score_artifact(tmp_path: Path) -> None: assert _score(output_dir) == 0.0 assert projected["integration_error"]["message"] == "checkout failed" assert _n_eff(output_dir) == 0 - - -def test_diagnostic_sequence_captures_raw_responses( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - verifier_dir = tmp_path / "verifier" - case_dir = verifier_dir / "testdata" / "walle_validator_cases" / "validator_cases" - case_dir.mkdir(parents=True) - output_dir = tmp_path / "output" - output_dir.mkdir() - - class FakeResponse: - accepted = True - message = "tool call returned" - arguments = '{"value": {}}' - http_status = None - error_type = None - - class FakeClient: - def __init__(self) -> None: - self.chat = SimpleNamespace( - completions=SimpleNamespace(create=lambda **kwargs: None) - ) - - def close(self) -> None: - pass - - fake_validator = SimpleNamespace( - load_cases=lambda path: ["case"], - select_cases=lambda cases, **kwargs: [ - ( - SimpleNamespace(suite="TestAdditionalProperties", line=1), - {"type": "object"}, - "object_parameter_schema", - ) - ], - make_client=lambda *args: FakeClient(), - send_tool_schema=lambda client, - model, - schema, - max_tokens, - thinking, - think_mode, - *, - stream: FakeResponse(), - validate_arguments=lambda schema, arguments: (True, "valid"), - ) - monkeypatch.setattr( - kve.importlib, - "import_module", - lambda name: fake_validator, - ) - - kve.run_diagnostic_sequence( - verifier_dir=verifier_dir, - base_url="http://localhost/v1", - api_key="EMPTY", - model="model-a", - output_dir=output_dir, - ) - - report = json.loads((output_dir / kve.DIAGNOSTIC_REPORT_FILENAME).read_text()) - assert report["sequence"] == list(kve.DIAGNOSTIC_MODES) - assert [record["mode"] for record in report["results"]] == list( - kve.DIAGNOSTIC_MODES - ) - assert all(record["arguments_valid"] for record in report["results"]) From 9a661a60c2394bba27358648614544116d49c7f0 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:50:56 -0500 Subject: [PATCH 20/24] test: capture deterministic Kimi diagnostics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：采集确定性的 Kimi 工具调用诊断数据。 --- .../workflows/benchmark-multinode-tmpl.yml | 26 ++ .github/workflows/benchmark-tmpl.yml | 26 ++ .github/workflows/e2e-tests.yml | 60 +++ benchmarks/benchmark_lib.sh | 10 + utils/evals/kimi_vendor_eval.py | 351 +++++++++++++++++- utils/evals/test_kimi_vendor_eval.py | 194 ++++++++++ 6 files changed, 666 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 7004faed99..06a58745d9 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -143,6 +143,26 @@ on: type: string required: false default: "" + kimi-tool-call-diagnostic: + description: "Run temporary ordered Kimi tool-call diagnostics" + type: boolean + required: false + default: false + kimi-tool-call-diagnostic-temperature: + description: "Temperature for temporary Kimi diagnostics" + type: string + required: false + default: "0" + kimi-tool-call-diagnostic-seed: + description: "Seed for temporary Kimi diagnostics" + type: string + required: false + default: "1" + kimi-tool-call-diagnostic-sequence: + description: "Comma-separated unary/stream diagnostic order" + type: string + required: false + default: "unary,unary,stream,stream,stream,unary" eval-conc: description: "Concurrency value or space-separated list for eval requests (overrides default max-of-conc-list)" type: string @@ -244,6 +264,10 @@ env: EVAL_ONLY: ${{ inputs.eval-only }} EVAL_FRAMEWORK: ${{ inputs.eval-framework }} EVAL_SUITE: ${{ inputs.eval-suite }} + KIMI_TOOL_CALL_DIAGNOSTIC: ${{ inputs.kimi-tool-call-diagnostic }} + KIMI_TOOL_CALL_DIAGNOSTIC_TEMPERATURE: ${{ inputs.kimi-tool-call-diagnostic-temperature }} + KIMI_TOOL_CALL_DIAGNOSTIC_SEED: ${{ inputs.kimi-tool-call-diagnostic-seed }} + KIMI_TOOL_CALL_DIAGNOSTIC_SEQUENCE: ${{ inputs.kimi-tool-call-diagnostic-sequence }} EVAL_CONC: ${{ inputs.eval-conc }} EVAL_LIMIT: ${{ inputs.eval-limit }} SWEBENCH_GEN_MODE: ${{ inputs.swebench-gen-mode }} @@ -484,6 +508,7 @@ jobs: meta_env.json results*.json *_vendor_report.json + eval_kimi_tool_call_diagnostic.json sample*.jsonl agent_preds.json predictions.jsonl @@ -506,6 +531,7 @@ jobs: rm -f meta_env.json || true rm -f results*.json || true rm -f *_vendor_report.json || true + rm -f eval_kimi_tool_call_diagnostic.json || true rm -f sample*.jsonl || true rm -f agent_preds.json predictions.jsonl swebench_report_*.json *.traj* || true diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 27f933dec5..612bb005f2 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -95,6 +95,26 @@ on: type: string required: false default: "" + kimi-tool-call-diagnostic: + description: "Run temporary ordered Kimi tool-call diagnostics" + type: boolean + required: false + default: false + kimi-tool-call-diagnostic-temperature: + description: "Temperature for temporary Kimi diagnostics" + type: string + required: false + default: "0" + kimi-tool-call-diagnostic-seed: + description: "Seed for temporary Kimi diagnostics" + type: string + required: false + default: "1" + kimi-tool-call-diagnostic-sequence: + description: "Comma-separated unary/stream diagnostic order" + type: string + required: false + default: "unary,unary,stream,stream,stream,unary" random-range-ratio: required: false type: string @@ -185,6 +205,10 @@ env: EVAL_ONLY: ${{ inputs.eval-only }} EVAL_FRAMEWORK: ${{ inputs.eval-framework }} EVAL_SUITE: ${{ inputs.eval-suite }} + KIMI_TOOL_CALL_DIAGNOSTIC: ${{ inputs.kimi-tool-call-diagnostic }} + KIMI_TOOL_CALL_DIAGNOSTIC_TEMPERATURE: ${{ inputs.kimi-tool-call-diagnostic-temperature }} + KIMI_TOOL_CALL_DIAGNOSTIC_SEED: ${{ inputs.kimi-tool-call-diagnostic-seed }} + KIMI_TOOL_CALL_DIAGNOSTIC_SEQUENCE: ${{ inputs.kimi-tool-call-diagnostic-sequence }} # Agentic-coding env. Fixed-seq-len jobs leave these empty. SCENARIO_TYPE: ${{ inputs.scenario-type }} SCENARIO_SUBDIR: ${{ inputs.scenario-type == 'agentic-coding' && 'agentic/' || 'fixed_seq_len/' }} @@ -405,6 +429,7 @@ jobs: meta_env.json results*.json *_vendor_report.json + eval_kimi_tool_call_diagnostic.json sample*.jsonl agent_preds.json predictions.jsonl @@ -423,6 +448,7 @@ jobs: # Remove any eval results JSONs that were moved into workspace rm -f results*.json || true rm -f -- ./*_vendor_report.json || true + rm -f eval_kimi_tool_call_diagnostic.json || true rm -f sample*.jsonl || true rm -f agent_preds.json predictions.jsonl swebench_report_*.json *.traj* || true diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index a0a31b5e91..2e52a9289e 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -55,6 +55,26 @@ on: required: false type: string default: "" + kimi-tool-call-diagnostic: + description: "Run temporary ordered Kimi tool-call diagnostics" + required: false + type: boolean + default: false + kimi-tool-call-diagnostic-temperature: + description: "Temperature for temporary Kimi diagnostics" + required: false + type: string + default: "0" + kimi-tool-call-diagnostic-seed: + description: "Seed for temporary Kimi diagnostics" + required: false + type: string + default: "1" + kimi-tool-call-diagnostic-sequence: + description: "Comma-separated unary/stream diagnostic order" + required: false + type: string + default: "unary,unary,stream,stream,stream,unary" swebench-gen-mode: description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)." required: false @@ -145,6 +165,26 @@ on: required: false type: string default: "" + kimi-tool-call-diagnostic: + description: "Run temporary ordered Kimi tool-call diagnostics" + required: false + type: boolean + default: false + kimi-tool-call-diagnostic-temperature: + description: "Temperature for temporary Kimi diagnostics" + required: false + type: string + default: "0" + kimi-tool-call-diagnostic-seed: + description: "Seed for temporary Kimi diagnostics" + required: false + type: string + default: "1" + kimi-tool-call-diagnostic-sequence: + description: "Comma-separated unary/stream diagnostic order" + required: false + type: string + default: "unary,unary,stream,stream,stream,unary" swebench-gen-mode: description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)." required: false @@ -398,6 +438,12 @@ jobs: run-eval: true eval-only: true eval-conc: ${{ matrix.config['eval-all-concs'] && join(matrix.config.conc, ' ') || matrix.config['eval-conc'] }} + eval-framework: ${{ inputs.eval-framework }} + eval-suite: ${{ inputs.eval-suite }} + kimi-tool-call-diagnostic: ${{ inputs.kimi-tool-call-diagnostic }} + kimi-tool-call-diagnostic-temperature: ${{ inputs.kimi-tool-call-diagnostic-temperature }} + kimi-tool-call-diagnostic-seed: ${{ inputs.kimi-tool-call-diagnostic-seed }} + kimi-tool-call-diagnostic-sequence: ${{ inputs.kimi-tool-call-diagnostic-sequence }} ref: ${{ inputs.ref }} test-sweep-agentic: @@ -484,6 +530,10 @@ jobs: swebench-gen-mode: ${{ inputs.swebench-gen-mode }} eval-framework: ${{ inputs.eval-framework }} eval-suite: ${{ inputs.eval-suite }} + kimi-tool-call-diagnostic: ${{ inputs.kimi-tool-call-diagnostic }} + kimi-tool-call-diagnostic-temperature: ${{ inputs.kimi-tool-call-diagnostic-temperature }} + kimi-tool-call-diagnostic-seed: ${{ inputs.kimi-tool-call-diagnostic-seed }} + kimi-tool-call-diagnostic-sequence: ${{ inputs.kimi-tool-call-diagnostic-sequence }} scenario-type: agentic-coding ref: ${{ inputs.ref }} @@ -601,6 +651,10 @@ jobs: swebench-gen-mode: ${{ inputs.swebench-gen-mode }} eval-framework: ${{ inputs.eval-framework }} eval-suite: ${{ inputs.eval-suite }} + kimi-tool-call-diagnostic: ${{ inputs.kimi-tool-call-diagnostic }} + kimi-tool-call-diagnostic-temperature: ${{ inputs.kimi-tool-call-diagnostic-temperature }} + kimi-tool-call-diagnostic-seed: ${{ inputs.kimi-tool-call-diagnostic-seed }} + kimi-tool-call-diagnostic-sequence: ${{ inputs.kimi-tool-call-diagnostic-sequence }} scenario-type: agentic-coding ref: ${{ inputs.ref }} @@ -679,6 +733,12 @@ jobs: run-eval: true eval-only: true eval-limit: ${{ inputs.eval-limit }} + eval-framework: ${{ inputs.eval-framework }} + eval-suite: ${{ inputs.eval-suite }} + kimi-tool-call-diagnostic: ${{ inputs.kimi-tool-call-diagnostic }} + kimi-tool-call-diagnostic-temperature: ${{ inputs.kimi-tool-call-diagnostic-temperature }} + kimi-tool-call-diagnostic-seed: ${{ inputs.kimi-tool-call-diagnostic-seed }} + kimi-tool-call-diagnostic-sequence: ${{ inputs.kimi-tool-call-diagnostic-sequence }} ref: ${{ inputs.ref }} collect-results: diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index cef240b274..e36f840c51 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1113,6 +1113,15 @@ _run_kimi_tool_call_schema_eval() { local adapter_path="${INFERENCEX_REPO_ROOT}/utils/evals/kimi_vendor_eval.py" local runtime_dir="" local checkout_dir="" + local diagnostic_args=() + if [[ "${KIMI_TOOL_CALL_DIAGNOSTIC:-false}" == "true" ]]; then + diagnostic_args+=(--diagnostic) + diagnostic_args+=( + --diagnostic-temperature "${KIMI_TOOL_CALL_DIAGNOSTIC_TEMPERATURE:-0}" + --diagnostic-seed "${KIMI_TOOL_CALL_DIAGNOSTIC_SEED:-1}" + --diagnostic-sequence "${KIMI_TOOL_CALL_DIAGNOSTIC_SEQUENCE:-unary,unary,stream,stream,stream,unary}" + ) + fi mkdir -p "$results_dir" || return $? export EVAL_RESULT_DIR="$results_dir" @@ -1157,6 +1166,7 @@ _run_kimi_tool_call_schema_eval() { --api-key EMPTY \ --model "$model_name" \ --output-dir "$results_dir" \ + "${diagnostic_args[@]}" \ || eval_rc=$? _cleanup_kimi_vendor_eval "$runtime_dir" "$checkout_dir" return "$eval_rc" diff --git a/utils/evals/kimi_vendor_eval.py b/utils/evals/kimi_vendor_eval.py index 4f3debc6fb..8da0eb9884 100755 --- a/utils/evals/kimi_vendor_eval.py +++ b/utils/evals/kimi_vendor_eval.py @@ -4,6 +4,8 @@ from __future__ import annotations import argparse +import importlib +import importlib.metadata import json import subprocess import sys @@ -19,6 +21,301 @@ DEFAULT_TIMEOUT_SECONDS = 900 RESULT_FORMAT = "inferencex-eval-v1" ADAPTER_NAME = "kimi-vendor-verifier" +DIAGNOSTIC_REPORT_FILENAME = "eval_kimi_tool_call_diagnostic.json" +DEFAULT_DIAGNOSTIC_TEMPERATURE = 0.0 +DEFAULT_DIAGNOSTIC_SEED = 1 +DEFAULT_DIAGNOSTIC_SEQUENCE = ( + "unary", + "unary", + "stream", + "stream", + "stream", + "unary", +) + + +def _utc_timestamp() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _jsonable(value: Any) -> Any: + """Convert OpenAI response models into JSON-compatible evidence.""" + if hasattr(value, "model_dump"): + return value.model_dump(mode="json") + if isinstance(value, Mapping): + return {str(key): _jsonable(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_jsonable(item) for item in value] + if value is None or isinstance(value, (bool, int, float, str)): + return value + return repr(value) + + +def _response_ids(value: Any) -> list[str]: + ids: list[str] = [] + if isinstance(value, Mapping): + response_id = value.get("id") + if isinstance(response_id, str) and response_id not in ids: + ids.append(response_id) + for item in value.values(): + for nested_id in _response_ids(item): + if nested_id not in ids: + ids.append(nested_id) + elif isinstance(value, list): + for item in value: + for nested_id in _response_ids(item): + if nested_id not in ids: + ids.append(nested_id) + return ids + + +class _CapturingStream: + """Record stream chunks while preserving the upstream iterator contract.""" + + def __init__(self, stream: Any, record: dict[str, Any]) -> None: + self._stream = stream + self._record = record + + def __iter__(self): + try: + for chunk in self._stream: + raw_chunk = _jsonable(chunk) + self._record["raw_chunks"].append(raw_chunk) + for response_id in _response_ids(raw_chunk): + if response_id not in self._record["response_ids"]: + self._record["response_ids"].append(response_id) + yield chunk + except Exception as exc: + self._record["transport_error"] = { + "type": type(exc).__name__, + "message": str(exc), + } + raise + + +class _CapturingCompletions: + """Delegate OpenAI requests while retaining exact payloads and raw replies.""" + + def __init__( + self, + completions: Any, + record: dict[str, Any], + request_overrides: Mapping[str, Any], + ) -> None: + self._completions = completions + self._record = record + self._request_overrides = request_overrides + + def create(self, **request: Any) -> Any: + request.update(self._request_overrides) + self._record["request_payload"] = _jsonable(request) + self._record["request_started_at"] = _utc_timestamp() + try: + response = self._completions.create(**request) + except Exception as exc: + self._record["response_received_at"] = _utc_timestamp() + self._record["transport_error"] = { + "type": type(exc).__name__, + "message": str(exc), + } + raise + self._record["response_received_at"] = _utc_timestamp() + if request.get("stream"): + self._record["raw_chunks"] = [] + return _CapturingStream(response, self._record) + raw_response = _jsonable(response) + self._record["raw_response"] = raw_response + self._record["response_ids"] = _response_ids(raw_response) + return response + + +class _CapturingClient: + """Expose the OpenAI chat interface expected by the upstream helper.""" + + def __init__( + self, + client: Any, + record: dict[str, Any], + request_overrides: Mapping[str, Any], + ) -> None: + chat_type = type("_CapturingChat", (), {}) + self.chat = chat_type() + self.chat.completions = _CapturingCompletions( + client.chat.completions, record, request_overrides + ) + + +def _runtime_versions() -> dict[str, str]: + versions: dict[str, str] = {"python": sys.version} + for distribution in ("openai", "httpx", "jsonschema"): + try: + versions[distribution] = importlib.metadata.version(distribution) + except importlib.metadata.PackageNotFoundError: + versions[distribution] = "not-installed" + return versions + +def _capture_version_endpoint(base_url: str) -> dict[str, Any]: + version_url = f"{base_url.removesuffix('/v1').rstrip('/')}/version" + started_at = _utc_timestamp() + try: + httpx = importlib.import_module("httpx") + response = httpx.get(version_url, timeout=5.0) + safe_headers = { + key: value + for key, value in response.headers.items() + if key.lower() + in { + "content-type", + "date", + "server", + "x-request-id", + "x-sglang-version", + } + } + return { + "url": version_url, + "started_at": started_at, + "completed_at": _utc_timestamp(), + "status_code": response.status_code, + "headers": safe_headers, + "body": response.text, + } + except Exception as exc: + return { + "url": version_url, + "started_at": started_at, + "completed_at": _utc_timestamp(), + "error": {"type": type(exc).__name__, "message": str(exc)}, + } + + +def run_diagnostic_sequence( + *, + verifier_dir: Path, + base_url: str, + api_key: str, + model: str, + output_dir: Path, + temperature: float = DEFAULT_DIAGNOSTIC_TEMPERATURE, + seed: int = DEFAULT_DIAGNOSTIC_SEED, + sequence: Sequence[str] = DEFAULT_DIAGNOSTIC_SEQUENCE, +) -> None: + """Run ordered stock-helper requests and preserve client-visible evidence.""" + report_path = output_dir / DIAGNOSTIC_REPORT_FILENAME + records: list[dict[str, Any]] = [] + report: dict[str, Any] = { + "model": model, + "base_url": base_url, + "started_at": _utc_timestamp(), + "controls": { + "temperature": temperature, + "seed": seed, + "sequence": list(sequence), + }, + "runtime_versions": _runtime_versions(), + "version_endpoint": _capture_version_endpoint(base_url), + "results": records, + } + sys.path.insert(0, str(verifier_dir)) + client: Any = None + try: + validator = importlib.import_module("tests.tool_call_json_schema.validator") + report["parser"] = { + "module": validator.__name__, + "file": str(Path(validator.__file__).resolve()), + } + cases = validator.load_cases( + verifier_dir / "testdata" / "walle_validator_cases" / "validator_cases" + ) + selected = validator.select_cases( + cases, + selection="object", + requested_cases=set(), + max_cases=1, + ) + if len(selected) != 1: + raise ValueError( + f"diagnostic expected one selected case, found {len(selected)}" + ) + case, schema, selection_reason = selected[0] + report["case"] = { + "suite": case.suite, + "line": case.line, + "selection_reason": selection_reason, + "schema": schema, + } + client = validator.make_client(base_url, api_key, 120) + mode_occurrences = {"unary": 0, "stream": 0} + for index, mode in enumerate(sequence, start=1): + mode_occurrences[mode] += 1 + record: dict[str, Any] = { + "index": index, + "mode": mode, + "mode_occurrence": mode_occurrences[mode], + "temperature_state": ( + "cold" if mode_occurrences[mode] == 1 else "warm" + ), + "sampling_controls": { + "temperature": temperature, + "seed": seed, + }, + "started_at": _utc_timestamp(), + "response_ids": [], + } + records.append(record) + try: + response = validator.send_tool_schema( + _CapturingClient( + client, + record, + {"temperature": temperature, "seed": seed}, + ), + model, + schema, + 2048, + False, + "none", + stream=mode == "stream", + ) + valid, validation_message = validator.validate_arguments( + schema, response.arguments + ) + record["parser_output"] = { + "accepted": response.accepted, + "message": response.message, + "arguments": response.arguments, + "arguments_valid": valid, + "validation_message": validation_message, + "http_status": response.http_status, + "error_type": response.error_type, + } + except Exception as exc: + record["diagnostic_error"] = { + "type": type(exc).__name__, + "message": str(exc), + } + finally: + record["completed_at"] = _utc_timestamp() + except Exception as exc: + report["setup_error"] = { + "type": type(exc).__name__, + "message": str(exc), + } + finally: + if client is not None: + try: + client.close() + except Exception as exc: + report["client_close_error"] = { + "type": type(exc).__name__, + "message": str(exc), + } + report["completed_at"] = _utc_timestamp() + report_path.write_text( + json.dumps(report, indent=2) + "\n", + encoding="utf-8", + ) + sys.path.pop(0) def prepare_compatibility_path(output_dir: Path) -> Path: @@ -167,6 +464,10 @@ def run_evaluation( model: str, output_dir: Path, timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS, + diagnostic: bool = False, + diagnostic_temperature: float = DEFAULT_DIAGNOSTIC_TEMPERATURE, + diagnostic_seed: int = DEFAULT_DIAGNOSTIC_SEED, + diagnostic_sequence: Sequence[str] = DEFAULT_DIAGNOSTIC_SEQUENCE, ) -> bool: """Run upstream pytest and always attempt to publish a compatibility result.""" output_dir.mkdir(parents=True, exist_ok=True) @@ -176,9 +477,25 @@ def run_evaluation( integration_error: BaseException | None = None compatibility = _compatibility_result(model, 0.0, n_samples=0) complete_pass = False - try: native_report.unlink(missing_ok=True) + if diagnostic: + try: + run_diagnostic_sequence( + verifier_dir=verifier_dir, + base_url=base_url, + api_key=api_key, + model=model, + output_dir=output_dir, + temperature=diagnostic_temperature, + seed=diagnostic_seed, + sequence=diagnostic_sequence, + ) + except Exception as exc: + print( + f"WARNING: Kimi diagnostic collection failed: {exc}", + file=sys.stderr, + ) completed = subprocess.run( build_pytest_command( base_url=base_url, @@ -223,6 +540,14 @@ def _positive_int(value: str) -> int: raise argparse.ArgumentTypeError("must be a positive integer") return parsed +def _diagnostic_sequence(value: str) -> tuple[str, ...]: + sequence = tuple(item.strip() for item in value.split(",") if item.strip()) + if not sequence or any(mode not in {"unary", "stream"} for mode in sequence): + raise argparse.ArgumentTypeError( + "must be a comma-separated sequence of unary and stream" + ) + return sequence + def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser( @@ -237,6 +562,26 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: "--timeout-seconds", type=_positive_int, default=DEFAULT_TIMEOUT_SECONDS ) parser.add_argument("--integration-error") + parser.add_argument( + "--diagnostic", + action="store_true", + help="Run ordered raw request diagnostics before the unchanged verifier", + ) + parser.add_argument( + "--diagnostic-temperature", + type=float, + default=DEFAULT_DIAGNOSTIC_TEMPERATURE, + ) + parser.add_argument( + "--diagnostic-seed", + type=int, + default=DEFAULT_DIAGNOSTIC_SEED, + ) + parser.add_argument( + "--diagnostic-sequence", + type=_diagnostic_sequence, + default=DEFAULT_DIAGNOSTIC_SEQUENCE, + ) args = parser.parse_args(argv) if args.integration_error is None: missing = [ @@ -276,6 +621,10 @@ def main(argv: Sequence[str] | None = None) -> int: model=args.model, output_dir=args.output_dir, timeout_seconds=args.timeout_seconds, + diagnostic=args.diagnostic, + diagnostic_temperature=args.diagnostic_temperature, + diagnostic_seed=args.diagnostic_seed, + diagnostic_sequence=args.diagnostic_sequence, ) return 0 if passed else 1 diff --git a/utils/evals/test_kimi_vendor_eval.py b/utils/evals/test_kimi_vendor_eval.py index e133d3b735..85aef1e9c6 100644 --- a/utils/evals/test_kimi_vendor_eval.py +++ b/utils/evals/test_kimi_vendor_eval.py @@ -236,3 +236,197 @@ def test_cli_setup_failure_writes_zero_score_artifact(tmp_path: Path) -> None: assert _score(output_dir) == 0.0 assert projected["integration_error"]["message"] == "checkout failed" assert _n_eff(output_dir) == 0 + + +def test_diagnostic_defaults_are_explicit_and_stock_command_is_unchanged( + tmp_path: Path, +) -> None: + args = kve.parse_args( + [ + "--verifier-dir", + str(tmp_path), + "--base-url", + "http://localhost/v1", + "--model", + "model-a", + "--output-dir", + str(tmp_path / "output"), + "--diagnostic", + ] + ) + assert args.diagnostic_temperature == 0 + assert args.diagnostic_seed == 1 + assert args.diagnostic_sequence == ( + "unary", + "unary", + "stream", + "stream", + "stream", + "unary", + ) + assert "--diagnostic" not in kve.build_pytest_command( + base_url="http://localhost/v1", + api_key="EMPTY", + model="model-a", + report_path=tmp_path / "report.json", + ) + + +def test_diagnostic_captures_payload_sequence_and_raw_modes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + verifier_dir = tmp_path / "verifier" + (verifier_dir / "testdata/walle_validator_cases/validator_cases").mkdir( + parents=True + ) + output_dir = tmp_path / "output" + output_dir.mkdir() + requests: list[dict[str, Any]] = [] + + class Raw: + def __init__(self, value: dict[str, Any]) -> None: + self.value = value + + def model_dump(self, *, mode: str) -> dict[str, Any]: + assert mode == "json" + return self.value + + class Completions: + def create(self, **request: Any) -> Any: + requests.append(request) + raw = { + "id": f"response-{len(requests)}", + "choices": [ + { + "message": { + "reasoning_content": "reasoning", + "tool_calls": [], + } + } + ], + } + if request.get("stream"): + return iter([Raw(raw)]) + return Raw(raw) + + client = SimpleNamespace( + chat=SimpleNamespace(completions=Completions()), + close=lambda: None, + ) + + def send_tool_schema( + capturing_client: Any, + model: str, + schema: Any, + max_tokens: int, + thinking: bool, + think_mode: str, + *, + stream: bool, + ) -> SimpleNamespace: + response = capturing_client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": "identical"}], + tools=[{"type": "function", "function": {"parameters": schema}}], + max_tokens=max_tokens, + stream=stream, + ) + if stream: + list(response) + return SimpleNamespace( + accepted=True, + message="parsed", + arguments="{}", + http_status=None, + error_type=None, + ) + + fake_validator = SimpleNamespace( + __name__="tests.tool_call_json_schema.validator", + __file__=str(verifier_dir / "validator.py"), + load_cases=lambda path: ["case"], + select_cases=lambda cases, **kwargs: [ + ( + SimpleNamespace(suite="suite", line=1), + {"type": "object"}, + "object_parameter_schema", + ) + ], + make_client=lambda *args: client, + send_tool_schema=send_tool_schema, + validate_arguments=lambda schema, arguments: (True, "valid"), + ) + monkeypatch.setattr(kve.importlib, "import_module", lambda name: fake_validator) + monkeypatch.setattr( + kve, + "_capture_version_endpoint", + lambda base_url: {"status_code": 200, "body": "v1"}, + ) + + kve.run_diagnostic_sequence( + verifier_dir=verifier_dir, + base_url="http://localhost/v1", + api_key="EMPTY", + model="model-a", + output_dir=output_dir, + temperature=0.25, + seed=7, + sequence=("unary", "stream", "unary"), + ) + + report = json.loads((output_dir / kve.DIAGNOSTIC_REPORT_FILENAME).read_text()) + assert report["controls"] == { + "temperature": 0.25, + "seed": 7, + "sequence": ["unary", "stream", "unary"], + } + assert [row["mode"] for row in report["results"]] == [ + "unary", + "stream", + "unary", + ] + assert [row["temperature_state"] for row in report["results"]] == [ + "cold", + "cold", + "warm", + ] + assert all( + request["temperature"] == 0.25 and request["seed"] == 7 + for request in requests + ) + assert report["results"][0]["request_payload"] == requests[0] + assert report["results"][0]["raw_response"]["choices"][0]["message"][ + "reasoning_content" + ] == "reasoning" + assert report["results"][1]["raw_chunks"][0]["id"] == "response-2" + assert report["results"][1]["response_ids"] == ["response-2"] + assert report["results"][2]["parser_output"]["arguments_valid"] is True + + +def test_diagnostic_failure_does_not_change_stock_score( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + output_dir = tmp_path / "output" + + def broken_diagnostic(**kwargs: Any) -> None: + raise RuntimeError("diagnostic failed") + + def fake_run( + command: list[str], *, cwd: Path, check: bool, timeout: int + ) -> SimpleNamespace: + Path(command[command.index("--tool-json-report") + 1]).write_text( + json.dumps(_report()) + ) + return SimpleNamespace(returncode=0) + + monkeypatch.setattr(kve, "run_diagnostic_sequence", broken_diagnostic) + monkeypatch.setattr(kve.subprocess, "run", fake_run) + assert kve.run_evaluation( + verifier_dir=tmp_path, + base_url="http://localhost/v1", + api_key="EMPTY", + model="model-a", + output_dir=output_dir, + diagnostic=True, + ) + assert _score(output_dir) == 1.0 \ No newline at end of file From 53b3ca822ba78d23ac38e3e94cf7a15cfea817bc Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Wed, 12 Aug 2026 01:20:33 -0500 Subject: [PATCH 21/24] fix: enable Kimi structural tool constraints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：启用 Kimi 结构化工具调用约束。 --- .../agg-gb200-dep16-throughput-agentic.yaml | 1 + ...throughput-vllm-simple-offload-agentic.yaml | 1 + .../agg-gb200-tep16-balanced-agentic.yaml | 1 + .../agg-gb200-tp16-latency-agentic.yaml | 1 + runners/test_slurm_utils.py | 18 ++++++++++++++++++ 5 files changed, 22 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-agentic.yaml index e7de6389a1..3ab5688e08 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-agentic.yaml @@ -116,6 +116,7 @@ backend: enable-prefix-caching: true scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" dyn-tool-call-parser: "kimi_k3" + dyn-enable-structural-tag: true reasoning-parser: "kimi_k3" dyn-reasoning-parser: "kimi_k3" no-enable-flashinfer-autotune: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-vllm-simple-offload-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-vllm-simple-offload-agentic.yaml index 703262fc3b..d80a3d1e49 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-vllm-simple-offload-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-vllm-simple-offload-agentic.yaml @@ -119,6 +119,7 @@ backend: kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" dyn-tool-call-parser: "kimi_k3" + dyn-enable-structural-tag: true reasoning-parser: "kimi_k3" dyn-reasoning-parser: "kimi_k3" no-enable-flashinfer-autotune: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml index e22dae3f38..6c73e65705 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml @@ -108,6 +108,7 @@ backend: enable-prefix-caching: true scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" dyn-tool-call-parser: "kimi_k3" + dyn-enable-structural-tag: true reasoning-parser: "kimi_k3" dyn-reasoning-parser: "kimi_k3" no-enable-flashinfer-autotune: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml index 2566aa62f4..bb1fc550e5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml @@ -105,6 +105,7 @@ backend: enable-prefix-caching: true scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" dyn-tool-call-parser: "kimi_k3" + dyn-enable-structural-tag: true reasoning-parser: "kimi_k3" dyn-reasoning-parser: "kimi_k3" no-enable-flashinfer-autotune: true diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index bfce571816..21dcd7e654 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -242,6 +242,24 @@ def test_gb200_kimi_compilation_config_preserves_all_settings() -> None: assert compilation_config["pass_config"]["fuse_allreduce_rms"] is False + +def test_gb200_dynamo_kimi_recipes_enable_structural_tool_constraints() -> None: + recipe_dir = ( + REPO_ROOT / "benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" + ) + recipes = [ + yaml.safe_load(path.read_text()) + for path in recipe_dir.glob("agg-gb200-*-agentic.yaml") + ] + + assert recipes + for recipe in recipes: + frontend = recipe["frontend"] + assert frontend["type"] == "dynamo" + config = recipe["backend"]["vllm_config"]["aggregated"] + assert config["dyn-tool-call-parser"] == "kimi_k3" + assert config["dyn-enable-structural-tag"] is True + def test_b200_kimi_recipe_uses_available_roce_devices() -> None: recipe_path = ( REPO_ROOT From 6d08f1e70d8ceab1d387958ede1ba244dddd3fee Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Wed, 12 Aug 2026 01:26:17 -0500 Subject: [PATCH 22/24] chore: format Kimi recipe regression MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：格式化 Kimi 配方回归测试。 --- runners/test_slurm_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index 21dcd7e654..a89a3354bb 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -2,8 +2,8 @@ import os import subprocess from pathlib import Path -import yaml +import yaml REPO_ROOT = Path(__file__).resolve().parents[1] SLURM_UTILS = REPO_ROOT / "runners" / "slurm_utils.sh" @@ -242,7 +242,6 @@ def test_gb200_kimi_compilation_config_preserves_all_settings() -> None: assert compilation_config["pass_config"]["fuse_allreduce_rms"] is False - def test_gb200_dynamo_kimi_recipes_enable_structural_tool_constraints() -> None: recipe_dir = ( REPO_ROOT / "benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" @@ -260,6 +259,7 @@ def test_gb200_dynamo_kimi_recipes_enable_structural_tool_constraints() -> None: assert config["dyn-tool-call-parser"] == "kimi_k3" assert config["dyn-enable-structural-tag"] is True + def test_b200_kimi_recipe_uses_available_roce_devices() -> None: recipe_path = ( REPO_ROOT From f1fb29da37dfe8883b721de555033ea035a2700c Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Wed, 12 Aug 2026 02:11:22 -0500 Subject: [PATCH 23/24] fix: retry transient Kimi verifier downloads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：重试 Kimi 验证器的临时下载失败。 --- benchmarks/benchmark_lib.sh | 75 +++++++++++++++++++-------- utils/evals/test_run_eval_dispatch.py | 35 ++++++++++++- 2 files changed, 86 insertions(+), 24 deletions(-) diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index e36f840c51..dd89c18fb5 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -871,6 +871,7 @@ import sys import tarfile import tempfile import time +from urllib.error import HTTPError, URLError from urllib.parse import quote, urlsplit, urlunsplit from urllib.request import Request, urlopen @@ -916,28 +917,58 @@ try: ) with tempfile.TemporaryFile() as archive_file: downloaded = 0 - deadline = time.monotonic() + 60 - with urlopen(request, timeout=60) as response: - while True: - remaining = deadline - time.monotonic() - if remaining <= 0: - raise TimeoutError("archive download exceeded the 60-second deadline") - sock = getattr(getattr(response, "fp", None), "raw", None) - sock = getattr(sock, "_sock", None) - if sock is not None: - sock.settimeout(max(0.001, remaining)) - try: - chunk = response.read(1024 * 1024) - except socket.timeout as error: - raise TimeoutError( - "archive download exceeded the 60-second deadline" - ) from error - if not chunk: - break - downloaded += len(chunk) - if downloaded > 128 * 1024 * 1024: - raise ValueError("archive download exceeds the 128 MiB safety limit") - archive_file.write(chunk) + for attempt in range(1, 4): + archive_file.seek(0) + archive_file.truncate() + downloaded = 0 + deadline = time.monotonic() + 60 + try: + with urlopen(request, timeout=60) as response: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError( + "archive download exceeded the 60-second deadline" + ) + sock = getattr(getattr(response, "fp", None), "raw", None) + sock = getattr(sock, "_sock", None) + if sock is not None: + sock.settimeout(max(0.001, remaining)) + try: + chunk = response.read(1024 * 1024) + except socket.timeout as error: + raise TimeoutError( + "archive download exceeded the 60-second deadline" + ) from error + if not chunk: + break + downloaded += len(chunk) + if downloaded > 128 * 1024 * 1024: + raise ValueError( + "archive download exceeds the 128 MiB safety limit" + ) + archive_file.write(chunk) + break + except HTTPError as error: + if error.code not in (408, 429) and not 500 <= error.code < 600: + raise + if attempt == 3: + raise + print( + f"WARN: Kimi-Vendor-Verifier archive download attempt " + f"{attempt}/3 failed: {error}; retrying", + file=sys.stderr, + ) + time.sleep(attempt) + except (TimeoutError, URLError, ConnectionError) as error: + if attempt == 3: + raise + print( + f"WARN: Kimi-Vendor-Verifier archive download attempt " + f"{attempt}/3 failed: {error}; retrying", + file=sys.stderr, + ) + time.sleep(attempt) if downloaded == 0: raise ValueError("downloaded archive is empty") archive_file.seek(0) diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index 6344ee9247..1dfafca2db 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -380,11 +380,19 @@ def _kimi_verifier_archive( @contextmanager -def _serve_archive(payload: bytes): +def _serve_archive(payload: bytes, *, transient_failures: int = 0): request_paths = [] + request_count = 0 + class ArchiveHandler(BaseHTTPRequestHandler): def do_GET(self): + nonlocal request_count request_paths.append(self.path) + request_count += 1 + if request_count <= transient_failures: + self.send_response(503) + self.end_headers() + return self.send_response(200) self.send_header("Content-Length", str(len(payload))) self.end_headers() @@ -411,6 +419,7 @@ def _prepare_local_kimi_verifier( tmp_path: Path, payload: bytes, verifier_ref: str = "1" * 40, + transient_failures: int = 0, ) -> tuple[subprocess.CompletedProcess[str], Path, list[str]]: checkout = tmp_path / "checkout" script = r''' @@ -419,7 +428,10 @@ def _prepare_local_kimi_verifier( mktemp() { mkdir "$CHECKOUT"; printf '%s\n' "$CHECKOUT"; } _prepare_kimi_vendor_verifier "$REPO_URL" "$VERIFIER_REF" ''' - with _serve_archive(payload) as (repo_url, request_paths): + with _serve_archive( + payload, + transient_failures=transient_failures, + ) as (repo_url, request_paths): result = subprocess.run( ["bash", "-c", script], env={ @@ -455,6 +467,25 @@ def test_kimi_vendor_verifier_fetches_expected_subset_without_git(tmp_path: Path assert "git must not be invoked" not in result.stderr +def test_kimi_vendor_verifier_retries_transient_archive_failure( + tmp_path: Path, +) -> None: + result, checkout, request_paths = _prepare_local_kimi_verifier( + tmp_path, + _kimi_verifier_archive(), + transient_failures=1, + ) + verifier_ref = "1" * 40 + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == str(checkout) + assert request_paths == [ + f"/owner/verifier/archive/{verifier_ref}.tar.gz", + f"/owner/verifier/archive/{verifier_ref}.tar.gz", + ] + assert "archive download attempt 1/3 failed" in result.stderr + + def test_kimi_vendor_verifier_removes_partial_checkout_when_member_missing( tmp_path: Path, ) -> None: From 0e679c50077d1b011702a732ddc688852fcf5928 Mon Sep 17 00:00:00 2001 From: adibarra <93070681+adibarra@users.noreply.github.com> Date: Wed, 12 Aug 2026 10:34:21 -0500 Subject: [PATCH 24/24] fix: stabilize and clean Kimi verifier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 中文：稳定并清理 Kimi 验证器集成。 --- .../workflows/benchmark-multinode-tmpl.yml | 26 -- .github/workflows/benchmark-tmpl.yml | 26 -- .github/workflows/e2e-tests.yml | 56 --- benchmarks/benchmark_lib.sh | 20 +- utils/evals/EVALS.md | 27 +- utils/evals/kimi_vendor_eval.py | 353 +----------------- utils/evals/test_kimi_vendor_eval.py | 198 +--------- utils/evals/test_run_eval_dispatch.py | 33 +- utils/test_collect_eval_results.py | 28 +- 9 files changed, 80 insertions(+), 687 deletions(-) diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 06a58745d9..7004faed99 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -143,26 +143,6 @@ on: type: string required: false default: "" - kimi-tool-call-diagnostic: - description: "Run temporary ordered Kimi tool-call diagnostics" - type: boolean - required: false - default: false - kimi-tool-call-diagnostic-temperature: - description: "Temperature for temporary Kimi diagnostics" - type: string - required: false - default: "0" - kimi-tool-call-diagnostic-seed: - description: "Seed for temporary Kimi diagnostics" - type: string - required: false - default: "1" - kimi-tool-call-diagnostic-sequence: - description: "Comma-separated unary/stream diagnostic order" - type: string - required: false - default: "unary,unary,stream,stream,stream,unary" eval-conc: description: "Concurrency value or space-separated list for eval requests (overrides default max-of-conc-list)" type: string @@ -264,10 +244,6 @@ env: EVAL_ONLY: ${{ inputs.eval-only }} EVAL_FRAMEWORK: ${{ inputs.eval-framework }} EVAL_SUITE: ${{ inputs.eval-suite }} - KIMI_TOOL_CALL_DIAGNOSTIC: ${{ inputs.kimi-tool-call-diagnostic }} - KIMI_TOOL_CALL_DIAGNOSTIC_TEMPERATURE: ${{ inputs.kimi-tool-call-diagnostic-temperature }} - KIMI_TOOL_CALL_DIAGNOSTIC_SEED: ${{ inputs.kimi-tool-call-diagnostic-seed }} - KIMI_TOOL_CALL_DIAGNOSTIC_SEQUENCE: ${{ inputs.kimi-tool-call-diagnostic-sequence }} EVAL_CONC: ${{ inputs.eval-conc }} EVAL_LIMIT: ${{ inputs.eval-limit }} SWEBENCH_GEN_MODE: ${{ inputs.swebench-gen-mode }} @@ -508,7 +484,6 @@ jobs: meta_env.json results*.json *_vendor_report.json - eval_kimi_tool_call_diagnostic.json sample*.jsonl agent_preds.json predictions.jsonl @@ -531,7 +506,6 @@ jobs: rm -f meta_env.json || true rm -f results*.json || true rm -f *_vendor_report.json || true - rm -f eval_kimi_tool_call_diagnostic.json || true rm -f sample*.jsonl || true rm -f agent_preds.json predictions.jsonl swebench_report_*.json *.traj* || true diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 612bb005f2..27f933dec5 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -95,26 +95,6 @@ on: type: string required: false default: "" - kimi-tool-call-diagnostic: - description: "Run temporary ordered Kimi tool-call diagnostics" - type: boolean - required: false - default: false - kimi-tool-call-diagnostic-temperature: - description: "Temperature for temporary Kimi diagnostics" - type: string - required: false - default: "0" - kimi-tool-call-diagnostic-seed: - description: "Seed for temporary Kimi diagnostics" - type: string - required: false - default: "1" - kimi-tool-call-diagnostic-sequence: - description: "Comma-separated unary/stream diagnostic order" - type: string - required: false - default: "unary,unary,stream,stream,stream,unary" random-range-ratio: required: false type: string @@ -205,10 +185,6 @@ env: EVAL_ONLY: ${{ inputs.eval-only }} EVAL_FRAMEWORK: ${{ inputs.eval-framework }} EVAL_SUITE: ${{ inputs.eval-suite }} - KIMI_TOOL_CALL_DIAGNOSTIC: ${{ inputs.kimi-tool-call-diagnostic }} - KIMI_TOOL_CALL_DIAGNOSTIC_TEMPERATURE: ${{ inputs.kimi-tool-call-diagnostic-temperature }} - KIMI_TOOL_CALL_DIAGNOSTIC_SEED: ${{ inputs.kimi-tool-call-diagnostic-seed }} - KIMI_TOOL_CALL_DIAGNOSTIC_SEQUENCE: ${{ inputs.kimi-tool-call-diagnostic-sequence }} # Agentic-coding env. Fixed-seq-len jobs leave these empty. SCENARIO_TYPE: ${{ inputs.scenario-type }} SCENARIO_SUBDIR: ${{ inputs.scenario-type == 'agentic-coding' && 'agentic/' || 'fixed_seq_len/' }} @@ -429,7 +405,6 @@ jobs: meta_env.json results*.json *_vendor_report.json - eval_kimi_tool_call_diagnostic.json sample*.jsonl agent_preds.json predictions.jsonl @@ -448,7 +423,6 @@ jobs: # Remove any eval results JSONs that were moved into workspace rm -f results*.json || true rm -f -- ./*_vendor_report.json || true - rm -f eval_kimi_tool_call_diagnostic.json || true rm -f sample*.jsonl || true rm -f agent_preds.json predictions.jsonl swebench_report_*.json *.traj* || true diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 2e52a9289e..dc4bbb665a 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -55,26 +55,6 @@ on: required: false type: string default: "" - kimi-tool-call-diagnostic: - description: "Run temporary ordered Kimi tool-call diagnostics" - required: false - type: boolean - default: false - kimi-tool-call-diagnostic-temperature: - description: "Temperature for temporary Kimi diagnostics" - required: false - type: string - default: "0" - kimi-tool-call-diagnostic-seed: - description: "Seed for temporary Kimi diagnostics" - required: false - type: string - default: "1" - kimi-tool-call-diagnostic-sequence: - description: "Comma-separated unary/stream diagnostic order" - required: false - type: string - default: "unary,unary,stream,stream,stream,unary" swebench-gen-mode: description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)." required: false @@ -165,26 +145,6 @@ on: required: false type: string default: "" - kimi-tool-call-diagnostic: - description: "Run temporary ordered Kimi tool-call diagnostics" - required: false - type: boolean - default: false - kimi-tool-call-diagnostic-temperature: - description: "Temperature for temporary Kimi diagnostics" - required: false - type: string - default: "0" - kimi-tool-call-diagnostic-seed: - description: "Seed for temporary Kimi diagnostics" - required: false - type: string - default: "1" - kimi-tool-call-diagnostic-sequence: - description: "Comma-separated unary/stream diagnostic order" - required: false - type: string - default: "unary,unary,stream,stream,stream,unary" swebench-gen-mode: description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)." required: false @@ -440,10 +400,6 @@ jobs: eval-conc: ${{ matrix.config['eval-all-concs'] && join(matrix.config.conc, ' ') || matrix.config['eval-conc'] }} eval-framework: ${{ inputs.eval-framework }} eval-suite: ${{ inputs.eval-suite }} - kimi-tool-call-diagnostic: ${{ inputs.kimi-tool-call-diagnostic }} - kimi-tool-call-diagnostic-temperature: ${{ inputs.kimi-tool-call-diagnostic-temperature }} - kimi-tool-call-diagnostic-seed: ${{ inputs.kimi-tool-call-diagnostic-seed }} - kimi-tool-call-diagnostic-sequence: ${{ inputs.kimi-tool-call-diagnostic-sequence }} ref: ${{ inputs.ref }} test-sweep-agentic: @@ -530,10 +486,6 @@ jobs: swebench-gen-mode: ${{ inputs.swebench-gen-mode }} eval-framework: ${{ inputs.eval-framework }} eval-suite: ${{ inputs.eval-suite }} - kimi-tool-call-diagnostic: ${{ inputs.kimi-tool-call-diagnostic }} - kimi-tool-call-diagnostic-temperature: ${{ inputs.kimi-tool-call-diagnostic-temperature }} - kimi-tool-call-diagnostic-seed: ${{ inputs.kimi-tool-call-diagnostic-seed }} - kimi-tool-call-diagnostic-sequence: ${{ inputs.kimi-tool-call-diagnostic-sequence }} scenario-type: agentic-coding ref: ${{ inputs.ref }} @@ -651,10 +603,6 @@ jobs: swebench-gen-mode: ${{ inputs.swebench-gen-mode }} eval-framework: ${{ inputs.eval-framework }} eval-suite: ${{ inputs.eval-suite }} - kimi-tool-call-diagnostic: ${{ inputs.kimi-tool-call-diagnostic }} - kimi-tool-call-diagnostic-temperature: ${{ inputs.kimi-tool-call-diagnostic-temperature }} - kimi-tool-call-diagnostic-seed: ${{ inputs.kimi-tool-call-diagnostic-seed }} - kimi-tool-call-diagnostic-sequence: ${{ inputs.kimi-tool-call-diagnostic-sequence }} scenario-type: agentic-coding ref: ${{ inputs.ref }} @@ -735,10 +683,6 @@ jobs: eval-limit: ${{ inputs.eval-limit }} eval-framework: ${{ inputs.eval-framework }} eval-suite: ${{ inputs.eval-suite }} - kimi-tool-call-diagnostic: ${{ inputs.kimi-tool-call-diagnostic }} - kimi-tool-call-diagnostic-temperature: ${{ inputs.kimi-tool-call-diagnostic-temperature }} - kimi-tool-call-diagnostic-seed: ${{ inputs.kimi-tool-call-diagnostic-seed }} - kimi-tool-call-diagnostic-sequence: ${{ inputs.kimi-tool-call-diagnostic-sequence }} ref: ${{ inputs.ref }} collect-results: diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index dd89c18fb5..ecad29da71 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -839,7 +839,8 @@ _install_kimi_vendor_eval_deps() { "httpx[http2]==0.28.1" \ "openai==2.14.0" \ "jsonschema==4.25.1" \ - "pytest==8.4.2" + "pytest==8.4.2" \ + "pytest-rerunfailures==16.4" } _prepare_kimi_vendor_runtime() { @@ -1144,15 +1145,6 @@ _run_kimi_tool_call_schema_eval() { local adapter_path="${INFERENCEX_REPO_ROOT}/utils/evals/kimi_vendor_eval.py" local runtime_dir="" local checkout_dir="" - local diagnostic_args=() - if [[ "${KIMI_TOOL_CALL_DIAGNOSTIC:-false}" == "true" ]]; then - diagnostic_args+=(--diagnostic) - diagnostic_args+=( - --diagnostic-temperature "${KIMI_TOOL_CALL_DIAGNOSTIC_TEMPERATURE:-0}" - --diagnostic-seed "${KIMI_TOOL_CALL_DIAGNOSTIC_SEED:-1}" - --diagnostic-sequence "${KIMI_TOOL_CALL_DIAGNOSTIC_SEQUENCE:-unary,unary,stream,stream,stream,unary}" - ) - fi mkdir -p "$results_dir" || return $? export EVAL_RESULT_DIR="$results_dir" @@ -1197,7 +1189,6 @@ _run_kimi_tool_call_schema_eval() { --api-key EMPTY \ --model "$model_name" \ --output-dir "$results_dir" \ - "${diagnostic_args[@]}" \ || eval_rc=$? _cleanup_kimi_vendor_eval "$runtime_dir" "$checkout_dir" return "$eval_rc" @@ -1547,7 +1538,7 @@ _write_lm_eval_meta_json() { fi fi fi - local eval_suite="${EVAL_SUITE:-}" + local eval_suite="${EVAL_COMPLETED_SUITE:-${EVAL_SUITE:-}}" if [ -z "$eval_suite" ] && [ -n "${EVAL_TASKS_DIR:-}" ]; then eval_suite="$(basename "${EVAL_TASKS_DIR}")" eval_suite="${eval_suite%.yaml}" @@ -2004,6 +1995,7 @@ run_eval() { local forwarded=() # Keep runner-selected suite identity scoped to this invocation. local EVAL_SUITE="${EVAL_SUITE:-}" + unset EVAL_COMPLETED_SUITE while [[ $# -gt 0 ]]; do case "$1" in @@ -2123,6 +2115,10 @@ run_eval() { *) echo "Unknown framework '${framework}'"; eval_rc=1 ;; esac + if [ "$framework" = "kimi-vendor" ]; then + export EVAL_COMPLETED_SUITE="$EVAL_SUITE" + fi + # Agentic eval-only recipes have no separate staging step. if [ "${EVAL_ONLY:-false}" = "true" ] && [ "$scenario_is_agentic" = "1" ]; then append_lm_eval_summary || true diff --git a/utils/evals/EVALS.md b/utils/evals/EVALS.md index 74c4974c1c..9c1375e7c0 100644 --- a/utils/evals/EVALS.md +++ b/utils/evals/EVALS.md @@ -78,25 +78,32 @@ The smoke runs the unmodified at commit `b9ed3a6665bdff2c943246f7d2903cd003d6ddd6`. Each run downloads the fresh pinned GitHub source archive and safely extracts only the upstream pytest configuration, tool-call schema tests, and bundled Walle cases. InferenceX does -not install the verifier package or reimplement its request, streaming, retry, -or validation logic. +not install the verifier package or reimplement its request, streaming, or +validation logic. Python 3.12 or newer is required. The runner installs the minimal pinned runtime -(`httpx[http2]`, `openai`, `jsonschema`, and `pytest`) into a temporary isolated -package directory, then runs upstream +(`httpx[http2]`, `openai`, `jsonschema`, `pytest`, and +`pytest-rerunfailures`) into a temporary isolated package directory, then runs +upstream `tests/tool_call_json_schema/test_tool_call_json_schema.py` with: - the local OpenAI-compatible endpoint, `EMPTY` API key, and served model name; - `--think-mode none --selection object --max-cases 1 --max-tokens 2048`; +- the upstream-recommended `--reruns 3 --reruns-delay 2`; - the bundled Walle case directory and `--tool-json-report`. The selection is `TestAdditionalProperties:1`, parametrized upstream in -non-streaming and streaming modes. The unchanged native report is uploaded as -`kimi_vendor_report.json`. `utils/evals/kimi_vendor_eval.py` only projects its -two outcomes into the existing eval result shape. Both must pass, so the -`kimi_tool_call_schema` threshold is `1.0`. Setup, timeout, and collection -failures emit a zero-score result with error metadata. The adapter bounds the -upstream pytest process to 900 seconds. +non-streaming and streaming modes. Pytest makes one initial attempt and up to +three reruns of each failing mode, with a two-second delay before each rerun. +These retries reduce transient transport and model-sampling flakes; they do not +make the smoke deterministic. The unchanged native report remains one final +outcome per mode because the upstream report deduplicates rerun records by case +and mode. It is uploaded as `kimi_vendor_report.json`, and +`utils/evals/kimi_vendor_eval.py` projects those two outcomes into the existing +eval result shape. Both must pass, so the `kimi_tool_call_schema` threshold is +`1.0`. Setup, timeout, and collection failures emit a zero-score result with +error metadata. The adapter's 900-second global timeout bounds the entire +upstream pytest process, including all attempts and rerun delays. This smoke validates one object-schema tool call. It does not cover tool choice, parallel calls, multi-turn execution, or general agent quality. Multi-value diff --git a/utils/evals/kimi_vendor_eval.py b/utils/evals/kimi_vendor_eval.py index 8da0eb9884..8851f2be66 100755 --- a/utils/evals/kimi_vendor_eval.py +++ b/utils/evals/kimi_vendor_eval.py @@ -4,8 +4,6 @@ from __future__ import annotations import argparse -import importlib -import importlib.metadata import json import subprocess import sys @@ -21,301 +19,6 @@ DEFAULT_TIMEOUT_SECONDS = 900 RESULT_FORMAT = "inferencex-eval-v1" ADAPTER_NAME = "kimi-vendor-verifier" -DIAGNOSTIC_REPORT_FILENAME = "eval_kimi_tool_call_diagnostic.json" -DEFAULT_DIAGNOSTIC_TEMPERATURE = 0.0 -DEFAULT_DIAGNOSTIC_SEED = 1 -DEFAULT_DIAGNOSTIC_SEQUENCE = ( - "unary", - "unary", - "stream", - "stream", - "stream", - "unary", -) - - -def _utc_timestamp() -> str: - return datetime.now(timezone.utc).isoformat() - - -def _jsonable(value: Any) -> Any: - """Convert OpenAI response models into JSON-compatible evidence.""" - if hasattr(value, "model_dump"): - return value.model_dump(mode="json") - if isinstance(value, Mapping): - return {str(key): _jsonable(item) for key, item in value.items()} - if isinstance(value, (list, tuple)): - return [_jsonable(item) for item in value] - if value is None or isinstance(value, (bool, int, float, str)): - return value - return repr(value) - - -def _response_ids(value: Any) -> list[str]: - ids: list[str] = [] - if isinstance(value, Mapping): - response_id = value.get("id") - if isinstance(response_id, str) and response_id not in ids: - ids.append(response_id) - for item in value.values(): - for nested_id in _response_ids(item): - if nested_id not in ids: - ids.append(nested_id) - elif isinstance(value, list): - for item in value: - for nested_id in _response_ids(item): - if nested_id not in ids: - ids.append(nested_id) - return ids - - -class _CapturingStream: - """Record stream chunks while preserving the upstream iterator contract.""" - - def __init__(self, stream: Any, record: dict[str, Any]) -> None: - self._stream = stream - self._record = record - - def __iter__(self): - try: - for chunk in self._stream: - raw_chunk = _jsonable(chunk) - self._record["raw_chunks"].append(raw_chunk) - for response_id in _response_ids(raw_chunk): - if response_id not in self._record["response_ids"]: - self._record["response_ids"].append(response_id) - yield chunk - except Exception as exc: - self._record["transport_error"] = { - "type": type(exc).__name__, - "message": str(exc), - } - raise - - -class _CapturingCompletions: - """Delegate OpenAI requests while retaining exact payloads and raw replies.""" - - def __init__( - self, - completions: Any, - record: dict[str, Any], - request_overrides: Mapping[str, Any], - ) -> None: - self._completions = completions - self._record = record - self._request_overrides = request_overrides - - def create(self, **request: Any) -> Any: - request.update(self._request_overrides) - self._record["request_payload"] = _jsonable(request) - self._record["request_started_at"] = _utc_timestamp() - try: - response = self._completions.create(**request) - except Exception as exc: - self._record["response_received_at"] = _utc_timestamp() - self._record["transport_error"] = { - "type": type(exc).__name__, - "message": str(exc), - } - raise - self._record["response_received_at"] = _utc_timestamp() - if request.get("stream"): - self._record["raw_chunks"] = [] - return _CapturingStream(response, self._record) - raw_response = _jsonable(response) - self._record["raw_response"] = raw_response - self._record["response_ids"] = _response_ids(raw_response) - return response - - -class _CapturingClient: - """Expose the OpenAI chat interface expected by the upstream helper.""" - - def __init__( - self, - client: Any, - record: dict[str, Any], - request_overrides: Mapping[str, Any], - ) -> None: - chat_type = type("_CapturingChat", (), {}) - self.chat = chat_type() - self.chat.completions = _CapturingCompletions( - client.chat.completions, record, request_overrides - ) - - -def _runtime_versions() -> dict[str, str]: - versions: dict[str, str] = {"python": sys.version} - for distribution in ("openai", "httpx", "jsonschema"): - try: - versions[distribution] = importlib.metadata.version(distribution) - except importlib.metadata.PackageNotFoundError: - versions[distribution] = "not-installed" - return versions - -def _capture_version_endpoint(base_url: str) -> dict[str, Any]: - version_url = f"{base_url.removesuffix('/v1').rstrip('/')}/version" - started_at = _utc_timestamp() - try: - httpx = importlib.import_module("httpx") - response = httpx.get(version_url, timeout=5.0) - safe_headers = { - key: value - for key, value in response.headers.items() - if key.lower() - in { - "content-type", - "date", - "server", - "x-request-id", - "x-sglang-version", - } - } - return { - "url": version_url, - "started_at": started_at, - "completed_at": _utc_timestamp(), - "status_code": response.status_code, - "headers": safe_headers, - "body": response.text, - } - except Exception as exc: - return { - "url": version_url, - "started_at": started_at, - "completed_at": _utc_timestamp(), - "error": {"type": type(exc).__name__, "message": str(exc)}, - } - - -def run_diagnostic_sequence( - *, - verifier_dir: Path, - base_url: str, - api_key: str, - model: str, - output_dir: Path, - temperature: float = DEFAULT_DIAGNOSTIC_TEMPERATURE, - seed: int = DEFAULT_DIAGNOSTIC_SEED, - sequence: Sequence[str] = DEFAULT_DIAGNOSTIC_SEQUENCE, -) -> None: - """Run ordered stock-helper requests and preserve client-visible evidence.""" - report_path = output_dir / DIAGNOSTIC_REPORT_FILENAME - records: list[dict[str, Any]] = [] - report: dict[str, Any] = { - "model": model, - "base_url": base_url, - "started_at": _utc_timestamp(), - "controls": { - "temperature": temperature, - "seed": seed, - "sequence": list(sequence), - }, - "runtime_versions": _runtime_versions(), - "version_endpoint": _capture_version_endpoint(base_url), - "results": records, - } - sys.path.insert(0, str(verifier_dir)) - client: Any = None - try: - validator = importlib.import_module("tests.tool_call_json_schema.validator") - report["parser"] = { - "module": validator.__name__, - "file": str(Path(validator.__file__).resolve()), - } - cases = validator.load_cases( - verifier_dir / "testdata" / "walle_validator_cases" / "validator_cases" - ) - selected = validator.select_cases( - cases, - selection="object", - requested_cases=set(), - max_cases=1, - ) - if len(selected) != 1: - raise ValueError( - f"diagnostic expected one selected case, found {len(selected)}" - ) - case, schema, selection_reason = selected[0] - report["case"] = { - "suite": case.suite, - "line": case.line, - "selection_reason": selection_reason, - "schema": schema, - } - client = validator.make_client(base_url, api_key, 120) - mode_occurrences = {"unary": 0, "stream": 0} - for index, mode in enumerate(sequence, start=1): - mode_occurrences[mode] += 1 - record: dict[str, Any] = { - "index": index, - "mode": mode, - "mode_occurrence": mode_occurrences[mode], - "temperature_state": ( - "cold" if mode_occurrences[mode] == 1 else "warm" - ), - "sampling_controls": { - "temperature": temperature, - "seed": seed, - }, - "started_at": _utc_timestamp(), - "response_ids": [], - } - records.append(record) - try: - response = validator.send_tool_schema( - _CapturingClient( - client, - record, - {"temperature": temperature, "seed": seed}, - ), - model, - schema, - 2048, - False, - "none", - stream=mode == "stream", - ) - valid, validation_message = validator.validate_arguments( - schema, response.arguments - ) - record["parser_output"] = { - "accepted": response.accepted, - "message": response.message, - "arguments": response.arguments, - "arguments_valid": valid, - "validation_message": validation_message, - "http_status": response.http_status, - "error_type": response.error_type, - } - except Exception as exc: - record["diagnostic_error"] = { - "type": type(exc).__name__, - "message": str(exc), - } - finally: - record["completed_at"] = _utc_timestamp() - except Exception as exc: - report["setup_error"] = { - "type": type(exc).__name__, - "message": str(exc), - } - finally: - if client is not None: - try: - client.close() - except Exception as exc: - report["client_close_error"] = { - "type": type(exc).__name__, - "message": str(exc), - } - report["completed_at"] = _utc_timestamp() - report_path.write_text( - json.dumps(report, indent=2) + "\n", - encoding="utf-8", - ) - sys.path.pop(0) def prepare_compatibility_path(output_dir: Path) -> Path: @@ -335,6 +38,10 @@ def build_pytest_command( "-m", "pytest", "tests/tool_call_json_schema/test_tool_call_json_schema.py", + "--reruns", + "3", + "--reruns-delay", + "2", "--base-url", base_url, "--api-key", @@ -464,10 +171,6 @@ def run_evaluation( model: str, output_dir: Path, timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS, - diagnostic: bool = False, - diagnostic_temperature: float = DEFAULT_DIAGNOSTIC_TEMPERATURE, - diagnostic_seed: int = DEFAULT_DIAGNOSTIC_SEED, - diagnostic_sequence: Sequence[str] = DEFAULT_DIAGNOSTIC_SEQUENCE, ) -> bool: """Run upstream pytest and always attempt to publish a compatibility result.""" output_dir.mkdir(parents=True, exist_ok=True) @@ -479,23 +182,6 @@ def run_evaluation( complete_pass = False try: native_report.unlink(missing_ok=True) - if diagnostic: - try: - run_diagnostic_sequence( - verifier_dir=verifier_dir, - base_url=base_url, - api_key=api_key, - model=model, - output_dir=output_dir, - temperature=diagnostic_temperature, - seed=diagnostic_seed, - sequence=diagnostic_sequence, - ) - except Exception as exc: - print( - f"WARNING: Kimi diagnostic collection failed: {exc}", - file=sys.stderr, - ) completed = subprocess.run( build_pytest_command( base_url=base_url, @@ -540,13 +226,6 @@ def _positive_int(value: str) -> int: raise argparse.ArgumentTypeError("must be a positive integer") return parsed -def _diagnostic_sequence(value: str) -> tuple[str, ...]: - sequence = tuple(item.strip() for item in value.split(",") if item.strip()) - if not sequence or any(mode not in {"unary", "stream"} for mode in sequence): - raise argparse.ArgumentTypeError( - "must be a comma-separated sequence of unary and stream" - ) - return sequence def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: @@ -562,26 +241,6 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: "--timeout-seconds", type=_positive_int, default=DEFAULT_TIMEOUT_SECONDS ) parser.add_argument("--integration-error") - parser.add_argument( - "--diagnostic", - action="store_true", - help="Run ordered raw request diagnostics before the unchanged verifier", - ) - parser.add_argument( - "--diagnostic-temperature", - type=float, - default=DEFAULT_DIAGNOSTIC_TEMPERATURE, - ) - parser.add_argument( - "--diagnostic-seed", - type=int, - default=DEFAULT_DIAGNOSTIC_SEED, - ) - parser.add_argument( - "--diagnostic-sequence", - type=_diagnostic_sequence, - default=DEFAULT_DIAGNOSTIC_SEQUENCE, - ) args = parser.parse_args(argv) if args.integration_error is None: missing = [ @@ -621,10 +280,6 @@ def main(argv: Sequence[str] | None = None) -> int: model=args.model, output_dir=args.output_dir, timeout_seconds=args.timeout_seconds, - diagnostic=args.diagnostic, - diagnostic_temperature=args.diagnostic_temperature, - diagnostic_seed=args.diagnostic_seed, - diagnostic_sequence=args.diagnostic_sequence, ) return 0 if passed else 1 diff --git a/utils/evals/test_kimi_vendor_eval.py b/utils/evals/test_kimi_vendor_eval.py index 85aef1e9c6..a71d3d962f 100644 --- a/utils/evals/test_kimi_vendor_eval.py +++ b/utils/evals/test_kimi_vendor_eval.py @@ -62,6 +62,10 @@ def test_builds_fixed_upstream_pytest_command(tmp_path: Path) -> None: "-m", "pytest", "tests/tool_call_json_schema/test_tool_call_json_schema.py", + "--reruns", + "3", + "--reruns-delay", + "2", "--base-url", "http://127.0.0.1:8000/v1", "--api-key", @@ -236,197 +240,3 @@ def test_cli_setup_failure_writes_zero_score_artifact(tmp_path: Path) -> None: assert _score(output_dir) == 0.0 assert projected["integration_error"]["message"] == "checkout failed" assert _n_eff(output_dir) == 0 - - -def test_diagnostic_defaults_are_explicit_and_stock_command_is_unchanged( - tmp_path: Path, -) -> None: - args = kve.parse_args( - [ - "--verifier-dir", - str(tmp_path), - "--base-url", - "http://localhost/v1", - "--model", - "model-a", - "--output-dir", - str(tmp_path / "output"), - "--diagnostic", - ] - ) - assert args.diagnostic_temperature == 0 - assert args.diagnostic_seed == 1 - assert args.diagnostic_sequence == ( - "unary", - "unary", - "stream", - "stream", - "stream", - "unary", - ) - assert "--diagnostic" not in kve.build_pytest_command( - base_url="http://localhost/v1", - api_key="EMPTY", - model="model-a", - report_path=tmp_path / "report.json", - ) - - -def test_diagnostic_captures_payload_sequence_and_raw_modes( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - verifier_dir = tmp_path / "verifier" - (verifier_dir / "testdata/walle_validator_cases/validator_cases").mkdir( - parents=True - ) - output_dir = tmp_path / "output" - output_dir.mkdir() - requests: list[dict[str, Any]] = [] - - class Raw: - def __init__(self, value: dict[str, Any]) -> None: - self.value = value - - def model_dump(self, *, mode: str) -> dict[str, Any]: - assert mode == "json" - return self.value - - class Completions: - def create(self, **request: Any) -> Any: - requests.append(request) - raw = { - "id": f"response-{len(requests)}", - "choices": [ - { - "message": { - "reasoning_content": "reasoning", - "tool_calls": [], - } - } - ], - } - if request.get("stream"): - return iter([Raw(raw)]) - return Raw(raw) - - client = SimpleNamespace( - chat=SimpleNamespace(completions=Completions()), - close=lambda: None, - ) - - def send_tool_schema( - capturing_client: Any, - model: str, - schema: Any, - max_tokens: int, - thinking: bool, - think_mode: str, - *, - stream: bool, - ) -> SimpleNamespace: - response = capturing_client.chat.completions.create( - model=model, - messages=[{"role": "user", "content": "identical"}], - tools=[{"type": "function", "function": {"parameters": schema}}], - max_tokens=max_tokens, - stream=stream, - ) - if stream: - list(response) - return SimpleNamespace( - accepted=True, - message="parsed", - arguments="{}", - http_status=None, - error_type=None, - ) - - fake_validator = SimpleNamespace( - __name__="tests.tool_call_json_schema.validator", - __file__=str(verifier_dir / "validator.py"), - load_cases=lambda path: ["case"], - select_cases=lambda cases, **kwargs: [ - ( - SimpleNamespace(suite="suite", line=1), - {"type": "object"}, - "object_parameter_schema", - ) - ], - make_client=lambda *args: client, - send_tool_schema=send_tool_schema, - validate_arguments=lambda schema, arguments: (True, "valid"), - ) - monkeypatch.setattr(kve.importlib, "import_module", lambda name: fake_validator) - monkeypatch.setattr( - kve, - "_capture_version_endpoint", - lambda base_url: {"status_code": 200, "body": "v1"}, - ) - - kve.run_diagnostic_sequence( - verifier_dir=verifier_dir, - base_url="http://localhost/v1", - api_key="EMPTY", - model="model-a", - output_dir=output_dir, - temperature=0.25, - seed=7, - sequence=("unary", "stream", "unary"), - ) - - report = json.loads((output_dir / kve.DIAGNOSTIC_REPORT_FILENAME).read_text()) - assert report["controls"] == { - "temperature": 0.25, - "seed": 7, - "sequence": ["unary", "stream", "unary"], - } - assert [row["mode"] for row in report["results"]] == [ - "unary", - "stream", - "unary", - ] - assert [row["temperature_state"] for row in report["results"]] == [ - "cold", - "cold", - "warm", - ] - assert all( - request["temperature"] == 0.25 and request["seed"] == 7 - for request in requests - ) - assert report["results"][0]["request_payload"] == requests[0] - assert report["results"][0]["raw_response"]["choices"][0]["message"][ - "reasoning_content" - ] == "reasoning" - assert report["results"][1]["raw_chunks"][0]["id"] == "response-2" - assert report["results"][1]["response_ids"] == ["response-2"] - assert report["results"][2]["parser_output"]["arguments_valid"] is True - - -def test_diagnostic_failure_does_not_change_stock_score( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - output_dir = tmp_path / "output" - - def broken_diagnostic(**kwargs: Any) -> None: - raise RuntimeError("diagnostic failed") - - def fake_run( - command: list[str], *, cwd: Path, check: bool, timeout: int - ) -> SimpleNamespace: - Path(command[command.index("--tool-json-report") + 1]).write_text( - json.dumps(_report()) - ) - return SimpleNamespace(returncode=0) - - monkeypatch.setattr(kve, "run_diagnostic_sequence", broken_diagnostic) - monkeypatch.setattr(kve.subprocess, "run", fake_run) - assert kve.run_evaluation( - verifier_dir=tmp_path, - base_url="http://localhost/v1", - api_key="EMPTY", - model="model-a", - output_dir=output_dir, - diagnostic=True, - ) - assert _score(output_dir) == 1.0 \ No newline at end of file diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index 1dfafca2db..cbcf25b7e3 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -202,7 +202,12 @@ def test_run_eval_scopes_runner_selected_suite_to_one_call() -> None: export EVAL_SUITE=kimi_tool_call_schema echo "DISPATCH=kimi-vendor SUITE=$EVAL_SUITE" } -run_lm_eval() { echo "DISPATCH=lm-eval SUITE=${EVAL_SUITE:-unset}"; } +run_lm_eval() { + echo "DISPATCH=lm-eval SUITE=${EVAL_SUITE:-unset} COMPLETED=${EVAL_COMPLETED_SUITE:-unset}" +} +append_lm_eval_summary() { + echo "METADATA=${EVAL_COMPLETED_SUITE:-gsm8k}" +} export EVAL_MAX_MODEL_LEN=16384 export EVAL_CONCURRENT_REQUESTS="" export EVAL_ONLY=false @@ -210,8 +215,12 @@ def test_run_eval_scopes_runner_selected_suite_to_one_call() -> None: unset EVAL_SUITE export EVAL_FRAMEWORK=kimi-vendor run_eval --port 8888 +printf 'KIMI_COMPLETED=%s\n' "${EVAL_COMPLETED_SUITE:-unset}" +append_lm_eval_summary export EVAL_FRAMEWORK=lm-eval run_eval --port 8888 +printf 'LM_COMPLETED=%s\n' "${EVAL_COMPLETED_SUITE:-unset}" +append_lm_eval_summary printf 'FINAL_SUITE=%s\n' "${EVAL_SUITE-unset}" ''' result = subprocess.run( @@ -224,7 +233,11 @@ def test_run_eval_scopes_runner_selected_suite_to_one_call() -> None: assert result.returncode == 0, result.stderr assert "DISPATCH=kimi-vendor SUITE=kimi_tool_call_schema" in result.stdout - assert "DISPATCH=lm-eval SUITE=unset" in result.stdout + assert "KIMI_COMPLETED=kimi_tool_call_schema" in result.stdout + assert "METADATA=kimi_tool_call_schema" in result.stdout + assert "DISPATCH=lm-eval SUITE=unset COMPLETED=unset" in result.stdout + assert "LM_COMPLETED=unset" in result.stdout + assert "METADATA=gsm8k" in result.stdout assert "FINAL_SUITE=unset" in result.stdout @@ -232,7 +245,7 @@ def test_kimi_default_suite_reaches_eval_only_metadata() -> None: script = r''' source "$BENCHMARK_LIB" run_kimi_vendor_eval() { echo "DISPATCH=$EVAL_SUITE"; } -append_lm_eval_summary() { echo "METADATA=$EVAL_SUITE"; } +append_lm_eval_summary() { echo "METADATA=$EVAL_COMPLETED_SUITE"; } export EVAL_FRAMEWORK=kimi-vendor export EVAL_ONLY=true export IS_AGENTIC=1 @@ -535,6 +548,7 @@ def test_kimi_vendor_dependency_install_is_isolated(tmp_path: Path) -> None: assert "PYTHON_ARG=<--target>" in result.stdout assert f"PYTHON_ARG=<{runtime_dir}>" in result.stdout + assert "PYTHON_ARG=" in result.stdout assert "--break-system-packages" not in result.stdout @@ -740,7 +754,7 @@ def _summary_metadata(tmp_path: Path, **overrides: str) -> dict: "CONC": "7", "KV_OFFLOADING": "none", } - for key in ("EVAL_SUITE", "EVAL_TASKS_DIR"): + for key in ("EVAL_COMPLETED_SUITE", "EVAL_SUITE", "EVAL_TASKS_DIR"): env.pop(key, None) env.update(overrides) subprocess.run(["bash", "-c", script], env=env, check=True) @@ -800,6 +814,17 @@ def test_summary_metadata_prefers_explicit_suite_then_task_basename( assert explicit["eval_suite"] == "kimi_tool_call_schema" +def test_summary_metadata_prefers_completed_eval_identity(tmp_path: Path) -> None: + meta = _summary_metadata( + tmp_path, + EVAL_COMPLETED_SUITE="kimi_tool_call_schema", + EVAL_SUITE="stale_input_selector", + EVAL_TASKS_DIR="/tmp/ignored.yaml", + ) + + assert meta["eval_suite"] == "kimi_tool_call_schema" + + def test_env_is_true_is_case_insensitive_and_unset_safe() -> None: script = r''' set -u diff --git a/utils/test_collect_eval_results.py b/utils/test_collect_eval_results.py index 3cac4d6b48..13f1cefe2e 100644 --- a/utils/test_collect_eval_results.py +++ b/utils/test_collect_eval_results.py @@ -42,23 +42,27 @@ def test_build_row_preserves_explicit_eval_suite() -> None: assert row["eval_suite"] == "kimi_tool_call_schema" -def _write_lm_eval_result(path: Path, score: float) -> None: +def _write_lm_eval_result( + path: Path, + score: float, + task: str = "gsm8k", +) -> None: path.write_text(json.dumps({ "lm_eval_version": "0.4.0", "model_name": "test-model", "results": { - "gsm8k": { + task: { "exact_match,strict-match": score, "exact_match_stderr,strict-match": 0.01, }, }, "configs": { - "gsm8k": { + task: { "metric_list": [{"metric": "exact_match"}], "filter_list": [{"name": "strict-match"}], }, }, - "n-samples": {"gsm8k": {"effective": 10}}, + "n-samples": {task: {"effective": 10}}, })) @@ -208,19 +212,23 @@ def test_collect_eval_rows_does_not_resurrect_stale_valid_result( artifact_dir = tmp_path / "eval_retry" artifact_dir.mkdir() (artifact_dir / "meta_env.json").write_text(json.dumps({ - "eval_suite": "gsm8k", + "eval_suite": "kimi_tool_call_schema", })) - stale_path = artifact_dir / "results_older.json" - _write_lm_eval_result(stale_path, 1.0) - current_path = artifact_dir / "results_current.json" - _write_lm_eval_result(current_path, 0.0) + stale_path = ( + artifact_dir / "results_kimi_vendor_2026-08-12T01-00-00.000000.json" + ) + _write_lm_eval_result(stale_path, 1.0, task="kimi_tool_call_schema") + current_path = ( + artifact_dir / "results_kimi_vendor_2026-08-12T02-00-00.000000.json" + ) + _write_lm_eval_result(current_path, 0.0, task="kimi_tool_call_schema") result = json.loads(current_path.read_text()) result["integration_error"] = { "type": "RuntimeError", "message": "vendor verifier checkout failed", } current_path.write_text(json.dumps(result)) - stale_path.touch() current_path.touch() + stale_path.touch() assert collect_eval_rows(tmp_path) == [] \ No newline at end of file