diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 0a666a6e90..7004faed99 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -133,6 +133,16 @@ on: type: boolean required: false default: false + eval-framework: + description: "Eval runner (lm-eval, swebench, or kimi-vendor)" + type: string + required: false + default: "lm-eval" + eval-suite: + description: "Kimi Vendor Verifier suite; leave empty for other eval runners" + type: string + required: false + default: "" eval-conc: description: "Concurrency value or space-separated list for eval requests (overrides default max-of-conc-list)" type: string @@ -232,6 +242,8 @@ env: DECODE_HARDWARE: ${{ inputs.decode-hardware }} RUN_EVAL: ${{ inputs.run-eval }} EVAL_ONLY: ${{ inputs.eval-only }} + EVAL_FRAMEWORK: ${{ inputs.eval-framework }} + EVAL_SUITE: ${{ inputs.eval-suite }} EVAL_CONC: ${{ inputs.eval-conc }} EVAL_LIMIT: ${{ inputs.eval-limit }} SWEBENCH_GEN_MODE: ${{ inputs.swebench-gen-mode }} @@ -467,10 +479,11 @@ jobs: if: ${{ always() && (env.RUN_EVAL == 'true' || inputs.eval-only) }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: eval_${{ env.RESULT_FILENAME }} + name: eval_${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_p${{ env.PREFILL_NUM_WORKERS }}x${{ env.PREFILL_TP }}p${{ env.PREFILL_PP_SIZE }}c${{ env.PREFILL_DCP_SIZE }}k${{ env.PREFILL_PCP_SIZE }}e${{ env.PREFILL_EP }}d${{ env.PREFILL_DP_ATTN }}_d${{ env.DECODE_NUM_WORKERS }}x${{ env.DECODE_TP }}p${{ env.DECODE_PP_SIZE }}c${{ env.DECODE_DCP_SIZE }}k${{ env.DECODE_PCP_SIZE }}e${{ env.DECODE_EP }}d${{ env.DECODE_DP_ATTN }}_kv${{ env.KV_OFFLOADING }}-${{ env.KV_OFFLOAD_BACKEND }}_spec${{ env.SPEC_DECODING }}_c${{ join(fromJson(inputs.conc-list), 'x') }}_${{ runner.name }} path: | meta_env.json results*.json + *_vendor_report.json sample*.jsonl agent_preds.json predictions.jsonl @@ -492,6 +505,7 @@ jobs: run: | rm -f meta_env.json || true rm -f results*.json || true + rm -f *_vendor_report.json || true rm -f sample*.jsonl || true rm -f agent_preds.json predictions.jsonl swebench_report_*.json *.traj* || true diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 6c4fe50fe5..27f933dec5 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -85,6 +85,16 @@ on: type: boolean required: false default: false + eval-framework: + description: "Eval runner (lm-eval, swebench, or kimi-vendor)" + type: string + required: false + default: "lm-eval" + eval-suite: + description: "Kimi Vendor Verifier suite; leave empty for other eval runners" + type: string + required: false + default: "" random-range-ratio: required: false type: string @@ -173,6 +183,8 @@ env: DISAGG: ${{ inputs.disagg }} RUN_EVAL: ${{ inputs.run-eval }} EVAL_ONLY: ${{ inputs.eval-only }} + EVAL_FRAMEWORK: ${{ inputs.eval-framework }} + EVAL_SUITE: ${{ inputs.eval-suite }} # Agentic-coding env. Fixed-seq-len jobs leave these empty. SCENARIO_TYPE: ${{ inputs.scenario-type }} SCENARIO_SUBDIR: ${{ inputs.scenario-type == 'agentic-coding' && 'agentic/' || 'fixed_seq_len/' }} @@ -197,7 +209,7 @@ env: MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} # These b300 nodes are currently broken. - SALLOC_EXCLUDE: 'b300-005,b300-006' + SALLOC_EXCLUDE: 'b300-005,b300-006,b300-017' permissions: contents: read @@ -392,6 +404,7 @@ jobs: path: | meta_env.json results*.json + *_vendor_report.json sample*.jsonl agent_preds.json predictions.jsonl @@ -409,6 +422,7 @@ jobs: rm -f meta_env.json || true # Remove any eval results JSONs that were moved into workspace rm -f results*.json || true + rm -f -- ./*_vendor_report.json || true rm -f sample*.jsonl || true rm -f agent_preds.json predictions.jsonl swebench_report_*.json *.traj* || true diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index c973df94e6..dc4bbb665a 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -45,6 +45,16 @@ on: required: false type: string default: "" + eval-framework: + description: "Agentic eval runner (lm-eval, swebench, or kimi-vendor)" + required: false + type: string + default: "lm-eval" + eval-suite: + description: "Kimi Vendor Verifier suite; empty for other runners" + required: false + type: string + default: "" swebench-gen-mode: description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)." required: false @@ -125,6 +135,16 @@ on: required: false type: string default: "" + eval-framework: + description: "Agentic eval runner (lm-eval, swebench, or kimi-vendor)" + required: false + type: string + default: "lm-eval" + eval-suite: + description: "Kimi Vendor Verifier suite; empty for other runners" + required: false + type: string + default: "" swebench-gen-mode: description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)." required: false @@ -236,7 +256,7 @@ jobs: CMD+=(--evals-only) fi RAW_CONFIG_JSON=$("${CMD[@]}") - CONFIG_JSON=$(python3 -c 'import json,sys; data=json.load(sys.stdin); rows=[row for family in ("single_node","multi_node") for group in data.get(family,{}).values() for row in group]; rows.extend(row for family in ("evals","agentic_evals","multinode_evals") for row in data.get(family,[])); print(json.dumps(rows))' <<<"$RAW_CONFIG_JSON") + CONFIG_JSON=$(python3 -c 'import json,sys; data=json.load(sys.stdin); rows=[row for family in ("single_node","multi_node") for group in data.get(family,{}).values() for row in group]; rows.extend(row for family in ("evals","agentic_evals","multinode_evals","multinode_agentic_evals") for row in data.get(family,[])); print(json.dumps(rows))' <<<"$RAW_CONFIG_JSON") else GENERATE_COMMAND="${{ inputs.generate-cli-command || github.event.inputs.generate-cli-command }}" if [ -z "$GENERATE_COMMAND" ]; then @@ -260,6 +280,7 @@ jobs: MULTI_AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x and not x.get('run-eval', False)]))" | score_matrix multi-agentic) MULTI_AGENTIC_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x and x.get('run-eval', False)]))" | score_matrix multi-agentic-eval) SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix single) + EVALS=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix eval) MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix multi) MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix multi-eval) { @@ -377,6 +398,8 @@ jobs: run-eval: true eval-only: true eval-conc: ${{ matrix.config['eval-all-concs'] && join(matrix.config.conc, ' ') || matrix.config['eval-conc'] }} + eval-framework: ${{ inputs.eval-framework }} + eval-suite: ${{ inputs.eval-suite }} ref: ${{ inputs.ref }} test-sweep-agentic: @@ -455,12 +478,14 @@ jobs: isl: '0' osl: '0' max-model-len: '0' - spec-decoding: 'none' + spec-decoding: ${{ matrix.config.spec-decoding }} disagg: ${{ 'false' }} run-eval: true eval-only: true eval-limit: ${{ inputs.eval-limit }} swebench-gen-mode: ${{ inputs.swebench-gen-mode }} + eval-framework: ${{ inputs.eval-framework }} + eval-suite: ${{ inputs.eval-suite }} scenario-type: agentic-coding ref: ${{ inputs.ref }} @@ -576,6 +601,8 @@ jobs: eval-conc: ${{ matrix.config['eval-conc'] }} eval-limit: ${{ inputs.eval-limit }} swebench-gen-mode: ${{ inputs.swebench-gen-mode }} + eval-framework: ${{ inputs.eval-framework }} + eval-suite: ${{ inputs.eval-suite }} scenario-type: agentic-coding ref: ${{ inputs.ref }} @@ -654,6 +681,8 @@ jobs: run-eval: true eval-only: true eval-limit: ${{ inputs.eval-limit }} + eval-framework: ${{ inputs.eval-framework }} + eval-suite: ${{ inputs.eval-suite }} ref: ${{ inputs.ref }} collect-results: diff --git a/.github/workflows/run-sweep.yml b/.github/workflows/run-sweep.yml index b8e95f0806..62d013934b 100644 --- a/.github/workflows/run-sweep.yml +++ b/.github/workflows/run-sweep.yml @@ -853,12 +853,12 @@ jobs: eval-only: true eval-conc: ${{ matrix.config['eval-all-concs'] && join(matrix.config.conc, ' ') || matrix.config['eval-conc'] }} - # Multi-node agentic (SWE-bench) eval rows carry the agentic input shape, - # so they are dispatched with sweep-multi-node-agentic's inputs rather - # than sweep-multi-node-evals' fixed-seq-len inputs (isl/osl/max-model-len, - # which agentic rows don't have). SWE-bench doesn't support batched - # concurrencies (unlike lm-eval), so eval-conc is always a single value, - # never the joined-list form sweep-multi-node-evals uses. + # Multi-node agentic GSM8K eval rows carry the agentic input shape, so + # they are dispatched with sweep-multi-node-agentic's inputs rather than + # sweep-multi-node-evals' fixed-seq-len inputs (isl/osl/max-model-len, + # which agentic rows don't have). Agentic selection uses one highest + # eval-conc per topology; fixed-sequence --all-evals rows may instead pass + # a joined concurrency list to lm-eval. sweep-multi-node-agentic-evals: needs: [setup, canary-select, canary-sweep] if: >- diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 8cc894940f..fe81bccc4b 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -8,6 +8,12 @@ export PYTHONDONTWRITEBYTECODE=1 export PYTHONPYCACHEPREFIX="${PYTHONPYCACHEPREFIX:-/tmp/inferencex-pycache}" mkdir -p "$PYTHONPYCACHEPREFIX" 2>/dev/null || true +INFERENCEX_BENCHMARK_LIB_DIR="$( + cd "$(dirname "${BASH_SOURCE[0]}")" && pwd +)" +INFERENCEX_REPO_ROOT="$( + cd "$INFERENCEX_BENCHMARK_LIB_DIR/.." && pwd +)" # Inference server port shared by every benchmark recipe. Launchers that need # a non-default value (e.g. launch_mi355x-amds.sh derives PORT from RUNNER_NAME @@ -886,8 +892,396 @@ _install_lm_eval_deps() { fi } +_require_kimi_vendor_python() { + if python3 -c 'import sys; raise SystemExit(sys.version_info < (3, 12))'; then + return 0 + fi + + local python_version + python_version="$(python3 -c 'import platform; print(platform.python_version())' 2>/dev/null || printf 'unavailable')" + echo "ERROR: Kimi Vendor Verifier requires Python >=3.12 (python3 is ${python_version})" >&2 + return 2 +} + +_install_kimi_vendor_eval_deps() { + local target_dir="$1" + python3 -m pip install -q --no-cache-dir --target "$target_dir" \ + "httpx[http2]==0.28.1" \ + "openai==2.14.0" \ + "jsonschema==4.25.1" \ + "pytest==8.4.2" \ + "pytest-rerunfailures==16.4" +} + +_prepare_kimi_vendor_runtime() { + local runtime_dir install_rc=0 + runtime_dir="$(mktemp -d /tmp/kimi-vendor-runtime-XXXXXX)" || return $? + _install_kimi_vendor_eval_deps "$runtime_dir" >&2 || install_rc=$? + if [ "$install_rc" -ne 0 ]; then + rm -rf "$runtime_dir" + return "$install_rc" + fi + printf '%s\n' "$runtime_dir" +} + +_prepare_kimi_vendor_verifier() { + local repo_url="$1" + local verifier_ref="$2" + local checkout_dir prepare_rc=0 + + checkout_dir="$(mktemp -d /tmp/kimi-vendor-verifier-XXXXXX)" || { + echo "ERROR: could not create a temporary directory for Kimi-Vendor-Verifier" >&2 + return 1 + } + + python3 - "$repo_url" "$verifier_ref" "$checkout_dir" <<'PY' || prepare_rc=$? +from pathlib import Path +import re +import socket +import sys +import tarfile +import tempfile +import time +from urllib.error import HTTPError, URLError +from urllib.parse import quote, urlsplit, urlunsplit +from urllib.request import Request, urlopen + + +repo_url, verifier_ref, checkout_dir_arg = sys.argv[1:] +checkout_dir = Path(checkout_dir_arg) +stage = "derive the pinned archive URL" + + +def archive_member_parts(name): + if not name or "\x00" in name or "\\" in name or name.startswith("/"): + raise ValueError(f"unsafe archive member path: {name!r}") + normalized = name.rstrip("/") + parts = normalized.split("/") + if not normalized or any(part in ("", ".", "..") for part in parts): + raise ValueError(f"unsafe archive member path: {name!r}") + return tuple(parts) + + +try: + if not re.fullmatch(r"[0-9a-fA-F]{40}", verifier_ref): + raise ValueError(f"expected a 40-character commit SHA, got {verifier_ref!r}") + + parsed_repo_url = urlsplit(repo_url) + if parsed_repo_url.scheme not in ("http", "https") or not parsed_repo_url.netloc: + raise ValueError(f"unsupported repository URL: {repo_url!r}") + if parsed_repo_url.query or parsed_repo_url.fragment: + raise ValueError(f"repository URL must not contain a query or fragment: {repo_url!r}") + repo_path = parsed_repo_url.path.rstrip("/") + if repo_path.endswith(".git"): + repo_path = repo_path[:-4] + if not repo_path: + raise ValueError(f"repository URL has no repository path: {repo_url!r}") + archive_path = f"{repo_path}/archive/{quote(verifier_ref, safe='')}.tar.gz" + archive_url = urlunsplit( + (parsed_repo_url.scheme, parsed_repo_url.netloc, archive_path, "", "") + ) + + stage = f"download {archive_url}" + request = Request( + archive_url, + headers={"User-Agent": "InferenceX-Kimi-Vendor-Verifier"}, + ) + with tempfile.TemporaryFile() as archive_file: + downloaded = 0 + for attempt in range(1, 4): + archive_file.seek(0) + archive_file.truncate() + downloaded = 0 + deadline = time.monotonic() + 60 + try: + with urlopen(request, timeout=60) as response: + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError( + "archive download exceeded the 60-second deadline" + ) + sock = getattr(getattr(response, "fp", None), "raw", None) + sock = getattr(sock, "_sock", None) + if sock is not None: + sock.settimeout(max(0.001, remaining)) + try: + chunk = response.read(1024 * 1024) + except socket.timeout as error: + raise TimeoutError( + "archive download exceeded the 60-second deadline" + ) from error + if not chunk: + break + downloaded += len(chunk) + if downloaded > 128 * 1024 * 1024: + raise ValueError( + "archive download exceeds the 128 MiB safety limit" + ) + archive_file.write(chunk) + break + except HTTPError as error: + if error.code not in (408, 429) and not 500 <= error.code < 600: + raise + if attempt == 3: + raise + print( + f"WARN: Kimi-Vendor-Verifier archive download attempt " + f"{attempt}/3 failed: {error}; retrying", + file=sys.stderr, + ) + time.sleep(attempt) + except (TimeoutError, URLError, ConnectionError) as error: + if attempt == 3: + raise + print( + f"WARN: Kimi-Vendor-Verifier archive download attempt " + f"{attempt}/3 failed: {error}; retrying", + file=sys.stderr, + ) + time.sleep(attempt) + if downloaded == 0: + raise ValueError("downloaded archive is empty") + archive_file.seek(0) + + stage = "validate the downloaded archive" + required_files = { + "pyproject.toml", + "tests/conftest.py", + "tests/__init__.py", + "tests/tool_call_json_schema/conftest.py", + "tests/tool_call_json_schema/__init__.py", + "tests/tool_call_json_schema/test_tool_call_json_schema.py", + "tests/tool_call_json_schema/validator.py", + "testdata/walle_validator_cases/validator_cases/TestAdditionalProperties/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestAnyOf/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestBasicTypes/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestDefs/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestDescription/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestEnforcerCases/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestID/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestKeywordsValidation/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestNestedDefsDepth/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestNumberFormat/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestRangeConstraints/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestRefInProperties/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestReferences/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestRequired/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestSingleTypeInArray/valid.jsonl", + "testdata/walle_validator_cases/validator_cases/TestTypeLocation/valid.jsonl", + } + selected_files = {} + archive_roots = set() + member_count = 0 + archive_size = 0 + selected_size = 0 + + with tarfile.open(fileobj=archive_file, mode="r|gz") as archive: + for member in archive: + member_count += 1 + if member_count > 100_000: + raise ValueError("archive contains more than 100000 members") + if member.size < 0: + raise ValueError( + f"archive member has a negative size: {member.name!r}" + ) + archive_size += member.size + if archive_size > 512 * 1024 * 1024: + raise ValueError("expanded archive exceeds the 512 MiB safety limit") + + parts = archive_member_parts(member.name) + archive_roots.add(parts[0]) + if len(archive_roots) > 1: + roots = ", ".join(sorted(archive_roots)) + raise ValueError(f"archive has multiple roots: {roots}") + if not (member.isdir() or member.isfile()): + raise ValueError( + f"archive member has unsafe type: {member.name!r}" + ) + if len(parts) == 1: + continue + + relative_path = "/".join(parts[1:]) + if relative_path not in required_files: + continue + if relative_path in selected_files: + raise ValueError( + f"archive contains duplicate selected path: {relative_path!r}" + ) + if not member.isfile(): + raise ValueError( + f"required path is not a regular file: {relative_path}" + ) + selected_size += member.size + if selected_size > 256 * 1024 * 1024: + raise ValueError( + "selected archive subset exceeds the 256 MiB safety limit" + ) + source = archive.extractfile(member) + if source is None: + raise ValueError(f"could not read archive member: {member.name!r}") + with source: + content = source.read(member.size + 1) + if len(content) != member.size: + raise ValueError( + f"archive member size mismatch: {member.name!r}" + ) + selected_files[relative_path] = content + + if member_count == 0: + raise ValueError("archive contains no members") + if len(archive_roots) != 1: + raise ValueError("archive does not have exactly one root") + missing_files = sorted(required_files - selected_files.keys()) + if missing_files: + raise ValueError( + "archive is missing required files: " + ", ".join(missing_files) + ) + + stage = "extract the verified archive subset" + if any(checkout_dir.iterdir()): + raise ValueError(f"checkout directory is not empty: {checkout_dir}") + for relative_path, content in selected_files.items(): + destination = checkout_dir.joinpath(*relative_path.split("/")) + destination.parent.mkdir(parents=True, exist_ok=True) + with destination.open("xb") as output: + output.write(content) +except Exception as error: + print( + f"ERROR: failed to {stage} for Kimi-Vendor-Verifier " + f"at {verifier_ref}: {error}", + file=sys.stderr, + ) + raise SystemExit(1) +PY + + if [ "$prepare_rc" -ne 0 ]; then + if ! rm -rf "$checkout_dir"; then + echo "ERROR: failed to remove partial Kimi-Vendor-Verifier directory ${checkout_dir}" >&2 + fi + return "$prepare_rc" + fi + + printf '%s\n' "$checkout_dir" +} + +_cleanup_kimi_vendor_eval() { + local path + for path in "$@"; do + [ -z "$path" ] || rm -rf "$path" || true + done +} + +_write_kimi_vendor_integration_error() { + local adapter_path="$1" + local model_name="$2" + local results_dir="$3" + local message="$4" + + python3 "$adapter_path" \ + --model "$model_name" \ + --output-dir "$results_dir" \ + --integration-error "$message" +} + +_run_kimi_tool_call_schema_eval() { + local port="${PORT:-8888}" + local results_dir="${EVAL_RESULT_DIR:-$(mktemp -d /tmp/eval_out-XXXXXX)}" + local verifier_repo="https://github.com/MoonshotAI/Kimi-Vendor-Verifier.git" + local verifier_ref="b9ed3a6665bdff2c943246f7d2903cd003d6ddd6" + + while [[ $# -gt 0 ]]; do + case "$1" in + --port|--results-dir) + if [[ $# -lt 2 || -z "${2:-}" || "${2:-}" == --* ]]; then + echo "ERROR: $1 requires a value" >&2 + return 2 + fi + case "$1" in + --port) port="$2" ;; + --results-dir) results_dir="$2" ;; + esac + shift 2 + ;; + *) + echo "Unknown parameter: $1" >&2 + return 2 + ;; + esac + done + + + local model_name="${MODEL_NAME:-${MODEL:-}}" + local adapter_path="${INFERENCEX_REPO_ROOT}/utils/evals/kimi_vendor_eval.py" + local runtime_dir="" + local checkout_dir="" + + mkdir -p "$results_dir" || return $? + export EVAL_RESULT_DIR="$results_dir" + + local setup_rc=0 integration_error="" + _require_kimi_vendor_python || { + setup_rc=$? + integration_error="Kimi Vendor Verifier Python version check failed with exit code ${setup_rc}" + } + if [ "$setup_rc" -eq 0 ]; then + runtime_dir=$(_prepare_kimi_vendor_runtime) || { + setup_rc=$? + integration_error="Kimi Vendor Verifier dependency installation failed with exit code ${setup_rc}" + } + fi + if [ "$setup_rc" -eq 0 ]; then + checkout_dir=$( + _prepare_kimi_vendor_verifier "$verifier_repo" "$verifier_ref" + ) || { + setup_rc=$? + integration_error="Kimi Vendor Verifier checkout failed with exit code ${setup_rc}" + } + fi + if [ "$setup_rc" -ne 0 ]; then + _cleanup_kimi_vendor_eval "$runtime_dir" "$checkout_dir" + echo "ERROR: ${integration_error}" >&2 + local artifact_rc=0 + _write_kimi_vendor_integration_error \ + "$adapter_path" "$model_name" "$results_dir" "$integration_error" \ + || artifact_rc=$? + if [ "$artifact_rc" -ne 0 ]; then + echo "ERROR: failed to write Kimi verifier failure artifact (exit code ${artifact_rc})" >&2 + fi + return "$setup_rc" + fi + + local eval_rc=0 + PYTHONPATH="${runtime_dir}${PYTHONPATH:+:${PYTHONPATH}}" \ + python3 "$adapter_path" \ + --verifier-dir "$checkout_dir" \ + --base-url "http://127.0.0.1:${port}/v1" \ + --api-key EMPTY \ + --model "$model_name" \ + --output-dir "$results_dir" \ + || eval_rc=$? + _cleanup_kimi_vendor_eval "$runtime_dir" "$checkout_dir" + return "$eval_rc" +} + +run_kimi_vendor_eval() { + local eval_suite="${EVAL_SUITE:-kimi_tool_call_schema}" + export EVAL_SUITE="$eval_suite" + + case "$eval_suite" in + kimi_tool_call_schema) + _run_kimi_tool_call_schema_eval "$@" + ;; + *) + echo "ERROR: unsupported Kimi Vendor Verifier suite '${eval_suite}'" >&2 + export EVAL_RESULT_DIR="" + return 2 + ;; + esac +} + _eval_patches_dir() { - cd "$(dirname "${BASH_SOURCE[0]}")/../utils/evals/patches" && pwd + printf '%s\n' "${INFERENCEX_REPO_ROOT}/utils/evals/patches" } _patch_lm_eval() { @@ -996,14 +1390,15 @@ run_lm_eval() { done # Serving images may use a different WORKDIR. - local _repo_root - _repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + local _repo_root="$INFERENCEX_REPO_ROOT" if [[ "$tasks_dir" == *.yaml && "$tasks_dir" != /* \ && ! -f "$tasks_dir" && -f "$_repo_root/$tasks_dir" ]]; then echo "run_lm_eval: anchoring relative task '$tasks_dir' to repo root -> $_repo_root/$tasks_dir" tasks_dir="$_repo_root/$tasks_dir" fi + export EVAL_TASKS_DIR="$tasks_dir" + if [ "${INFERENCEX_LM_EVAL_RUNTIME_READY:-false}" != "true" ]; then _install_lm_eval_deps _patch_lm_eval @@ -1119,8 +1514,8 @@ _eval_concs_to_json() { } _env_is_true() { - case "${1,,}" in - 1|true|yes|on) return 0 ;; + case "${1:-}" in + 1|[Tt][Rr][Uu][Ee]|[Yy][Ee][Ss]|[Oo][Nn]) return 0 ;; *) return 1 ;; esac } @@ -1213,6 +1608,13 @@ _write_lm_eval_meta_json() { fi fi fi + local eval_suite="${EVAL_COMPLETED_SUITE:-${EVAL_SUITE:-}}" + if [ -z "$eval_suite" ] && [ -n "${EVAL_TASKS_DIR:-}" ]; then + eval_suite="$(basename "${EVAL_TASKS_DIR}")" + eval_suite="${eval_suite%.yaml}" + eval_suite="${eval_suite%.yml}" + fi + eval_suite="${eval_suite:-gsm8k}" cat > "${meta_json}" <&2 + return 2 + ;; + esac + + if [ -n "${EVAL_SUITE:-}" ] && [ "$framework" != "kimi-vendor" ]; then + echo "ERROR: EVAL_SUITE is only supported with EVAL_FRAMEWORK=kimi-vendor" >&2 + return 2 + fi + + # Kimi Vendor Verifier uses a fixed request budget and does not consume + # EVAL_MAX_MODEL_LEN, so avoid loading model configuration for that path. + if [ "$framework" != "kimi-vendor" ] && [ -z "${EVAL_MAX_MODEL_LEN:-}" ]; then compute_eval_context_length "$MODEL" "${MAX_MODEL_LEN:-0}" > /dev/null fi @@ -1757,9 +2181,14 @@ run_eval() { case "$framework" in lm-eval|lm_eval) run_lm_eval "${forwarded[@]}" || eval_rc=$? ;; swebench) run_swebench_eval "${forwarded[@]}" || eval_rc=$? ;; + kimi-vendor) run_kimi_vendor_eval "${forwarded[@]}" || eval_rc=$? ;; *) echo "Unknown framework '${framework}'"; eval_rc=1 ;; esac + if [ "$framework" = "kimi-vendor" ]; then + export EVAL_COMPLETED_SUITE="$EVAL_SUITE" + fi + # Agentic eval-only recipes have no separate staging step. if [ "${EVAL_ONLY:-false}" = "true" ] && [ "$scenario_is_agentic" = "1" ]; then append_lm_eval_summary || true @@ -1767,7 +2196,7 @@ run_eval() { if [ "$eval_rc" -ne 0 ]; then echo "ERROR: run_eval failed with exit code $eval_rc" >&2 - if [ "${EVAL_ONLY}" = "true" ]; then + if [ "${EVAL_ONLY:-false}" = "true" ]; then echo "Eval-only mode: failing after artifact collection" >&2 return "$eval_rc" fi diff --git a/benchmarks/multi_node/agentic_srt.sh b/benchmarks/multi_node/agentic_srt.sh index 79a36da524..dea0881327 100644 --- a/benchmarks/multi_node/agentic_srt.sh +++ b/benchmarks/multi_node/agentic_srt.sh @@ -26,8 +26,6 @@ for concurrency in "${CONCURRENCIES[@]}"; do fi done -resolve_trace_source -install_agentic_deps wait_for_agentic_servers_idle() { local timeout_seconds="${AIPERF_DRAIN_TIMEOUT_SECONDS:-1800}" @@ -98,6 +96,9 @@ raise SystemExit(f"Agentic servers did not drain within {timeout_seconds} second PY } +resolve_trace_source +install_agentic_deps + # The AgentX scenario's first-turn cache-bust marker includes AIPerf's unique # per-invocation benchmark ID. Each point therefore gets a disjoint KV keyspace # while its own warmup and profile phases share markers. This makes sequential diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml index a0207d65f2..e6ed84715d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml @@ -80,7 +80,7 @@ backend: TILELANG_CLEANUP_TEMP_FILES: "1" UCX_MEMTYPE_CACHE: "n" UCX_MEMTYPE_REG_WHOLE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_10:1,mlx5_11:1" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1,mlx5_4:1,mlx5_5:1,mlx5_6:1,mlx5_7:1" HF_HUB_CACHE: "/hf_hub_cache" HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" vllm_config: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-agentic.yaml index e7de6389a1..3ab5688e08 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-agentic.yaml @@ -116,6 +116,7 @@ backend: enable-prefix-caching: true scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" dyn-tool-call-parser: "kimi_k3" + dyn-enable-structural-tag: true reasoning-parser: "kimi_k3" dyn-reasoning-parser: "kimi_k3" no-enable-flashinfer-autotune: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-vllm-simple-offload-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-vllm-simple-offload-agentic.yaml index 703262fc3b..d80a3d1e49 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-vllm-simple-offload-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dep16-throughput-vllm-simple-offload-agentic.yaml @@ -119,6 +119,7 @@ backend: kv-transfer-config: '{"kv_connector":"SimpleCPUOffloadConnector","kv_role":"kv_both","kv_connector_extra_config":{"cpu_bytes_to_use":549755813888,"cpu_bytes_to_use_per_rank":137438953472,"lazy_offload":false}}' scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" dyn-tool-call-parser: "kimi_k3" + dyn-enable-structural-tag: true reasoning-parser: "kimi_k3" dyn-reasoning-parser: "kimi_k3" no-enable-flashinfer-autotune: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml index bf135c0e33..6c73e65705 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tep16-balanced-agentic.yaml @@ -101,17 +101,17 @@ backend: max-num-seqs: 32 max-num-batched-tokens: 8192 speculative-config: '{"method":"dspark","model":"/tmp/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' - compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96]}' + compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96],"pass_config":{"fuse_allreduce_rms":false}}' block-size: 64 language-model-only: true disable-custom-all-reduce: true enable-prefix-caching: true scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" dyn-tool-call-parser: "kimi_k3" + dyn-enable-structural-tag: true reasoning-parser: "kimi_k3" dyn-reasoning-parser: "kimi_k3" no-enable-flashinfer-autotune: true - compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' sbatch_directives: cpus-per-task: "144" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml index baed3f19e7..bb1fc550e5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp16-latency-agentic.yaml @@ -98,17 +98,17 @@ backend: max-num-seqs: 8 max-num-batched-tokens: 8192 speculative-config: '{"method":"dspark","model":"/tmp/Kimi-K3-DSpark","num_speculative_tokens":2,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":2.51}' - compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24]}' + compilation-config: '{"cudagraph_capture_sizes":[3,6,9,12,15,18,21,24],"pass_config":{"fuse_allreduce_rms":false}}' block-size: 64 language-model-only: true disable-custom-all-reduce: true enable-prefix-caching: true scheduler-cls: "vllm.v1.core.sched.async_scheduler.AsyncScheduler" dyn-tool-call-parser: "kimi_k3" + dyn-enable-structural-tag: true reasoning-parser: "kimi_k3" dyn-reasoning-parser: "kimi_k3" no-enable-flashinfer-autotune: true - compilation-config: '{"pass_config":{"fuse_allreduce_rms":false}}' sbatch_directives: cpus-per-task: "144" diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh index d7ed7a359d..62ec5db23a 100644 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh @@ -4,7 +4,6 @@ set -x source "$(dirname "$0")/../../benchmark_lib.sh" - export EVAL_FRAMEWORK="lm-eval" check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh index be705c0c21..58566be51f 100755 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_b200_mtp.sh @@ -21,8 +21,6 @@ set -x source "$(dirname "$0")/../../benchmark_lib.sh" -export EVAL_FRAMEWORK="lm-eval" - check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh index 3593cac4ce..1b65687c18 100755 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_b300_mtp.sh @@ -7,8 +7,6 @@ set -x source "$(dirname "$0")/../../benchmark_lib.sh" -export EVAL_FRAMEWORK="lm-eval" - check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh index 7faaf69a51..a82924b167 100644 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh @@ -10,13 +10,6 @@ set -x source "$(dirname "$0")/../../benchmark_lib.sh" -# Force the eval framework to lm-eval for this recipe. run_eval derives its -# default as swebench for agentic scenarios (scenario_default=swebench when -# IS_AGENTIC/SCENARIO_TYPE=agentic-coding), but EVAL_FRAMEWORK takes precedence -# over that default (benchmark_lib.sh: framework=${EVAL_FRAMEWORK:-...}), so -# setting it here makes the effective framework always lm-eval, never swebench. -export EVAL_FRAMEWORK="lm-eval" - check_env_vars MODEL TP CONC KV_OFFLOADING KV_OFFLOAD_BACKEND TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION echo "MODEL=$MODEL TP=$TP CONC=$CONC KV_OFFLOADING=$KV_OFFLOADING TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB RESULT_DIR=$RESULT_DIR DURATION=$DURATION EP_SIZE=$EP_SIZE DP_ATTENTION=$DP_ATTENTION" diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh index f0fc1d09da..fc956e14a0 100755 --- a/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp4_b200_sglang_mtp.sh @@ -8,9 +8,6 @@ set -x source "$(dirname "$0")/../../benchmark_lib.sh" -# Use the lightweight GSM8K eval instead of the AgentX SWE-bench default. -export EVAL_FRAMEWORK="lm-eval" - check_env_vars \ MODEL TP CONC EP_SIZE KV_OFFLOADING \ TOTAL_CPU_DRAM_GB RESULT_DIR DURATION diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_b300_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_b300_sglang_mtp.sh index a6a0ccf3bd..b43eb06454 100755 --- a/benchmarks/single_node/agentic/qwen3.5_fp4_b300_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp4_b300_sglang_mtp.sh @@ -8,9 +8,6 @@ set -x source "$(dirname "$0")/../../benchmark_lib.sh" -# Use the lightweight GSM8K eval instead of the AgentX SWE-bench default. -export EVAL_FRAMEWORK="lm-eval" - check_env_vars \ MODEL TP CONC EP_SIZE KV_OFFLOADING \ TOTAL_CPU_DRAM_GB RESULT_DIR DURATION diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh index 472b9e4b12..536602459b 100755 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_b200_sglang_mtp.sh @@ -8,9 +8,6 @@ set -x source "$(dirname "$0")/../../benchmark_lib.sh" -# Use the lightweight GSM8K eval instead of the AgentX SWE-bench default. -export EVAL_FRAMEWORK="lm-eval" - check_env_vars \ MODEL TP CONC EP_SIZE KV_OFFLOADING \ TOTAL_CPU_DRAM_GB RESULT_DIR DURATION diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang_mtp.sh index 2d0a390250..96583ebe82 100755 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang_mtp.sh @@ -8,9 +8,6 @@ set -x source "$(dirname "$0")/../../benchmark_lib.sh" -# Use the lightweight GSM8K eval instead of the AgentX SWE-bench default. -export EVAL_FRAMEWORK="lm-eval" - check_env_vars \ MODEL TP CONC EP_SIZE KV_OFFLOADING \ TOTAL_CPU_DRAM_GB RESULT_DIR DURATION diff --git a/docs/eval-agentx-procedures.md b/docs/eval-agentx-procedures.md index a3afb3076a..263927d908 100644 --- a/docs/eval-agentx-procedures.md +++ b/docs/eval-agentx-procedures.md @@ -18,11 +18,11 @@ There are two distinct layers: the matrix generator decides **which jobs exist** | Normal sweep | no eval option | Throughput jobs plus the selected 8k/1k eval subset | | Throughput only | `--no-evals` | No eval jobs | | Selected eval subset only | `--evals-only` | Jobs have `RUN_EVAL=true`, `EVAL_ONLY=true` | -| Every eligible eval only | `--all-evals` | Equivalent to `--evals-only --all-evals` and includes all fixed-sequence 8k/1k rows, plus single-node agentic SWE-bench rows | +| Every eligible eval only | `--all-evals` | Equivalent to `--evals-only --all-evals` and includes all fixed-sequence 8k/1k rows plus single-node and multi-node agentic GSM8K rows | | Throughput then eval in one recipe | `RUN_EVAL=true`, `EVAL_ONLY=false` | Server starts, throughput runs, then `run_eval` runs | | Eval against a freshly started server | `RUN_EVAL=true`, `EVAL_ONLY=true` | Launcher expands eval context, skips throughput, and runs the eval | -Default selection is scenario-aware. Single-node fixed-sequence evals use the median and highest eligible concurrency for each 8k/1k model/runner/framework/precision/parallelism group. Multi-node evals use the highest eligible concurrency per topology. Concurrency below 16 is not selected. Agentic evals are opt-in, and multi-node agentic eval is unsupported. See [`mark_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L238-L339) and [`mark_all_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L342-L398). +Default selection is scenario-aware. Single-node fixed-sequence evals use the median and highest eligible concurrency for each 8k/1k model/runner/framework/precision/parallelism group. Multi-node evals use the highest eligible concurrency per topology. Concurrency below 16 is not selected. Agentic evals are opt-in; single-node and multi-node agentic rows select their highest eligible concurrency. See [`mark_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L276-L396) and [`mark_all_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L399-L482). On a PR, combine one primary sweep label (normally `full-sweep-fail-fast`) with eval modifiers. `all-evals` expands coverage without suppressing throughput. `evals-only` suppresses throughput. Together they run all eligible evals only. Runs with `evals-only` are not reusable, while normal full sweeps and `all-evals` full sweeps are reusable. Adding or removing a modifier restarts the active sweep ([label policy](../.github/workflows/README.md#pr-eval-modifiers)). @@ -48,14 +48,14 @@ uv run --no-project --with pydantic --with pyyaml --python 3.12 \ --config-files configs/nvidia-master.yaml | jq . ``` -A correct AgentX eval row contains `"scenario-type": "agentic-coding"`, `"run-eval": true`, and `"eval-only": true`. The workflow splits generated rows into throughput, fixed-sequence eval, and agentic eval jobs in [`.github/workflows/e2e-tests.yml`](../.github/workflows/e2e-tests.yml#L257-L271). +A correct AgentX eval row contains `"scenario-type": "agentic-coding"`, `"run-eval": true`, and `"eval-only": true`. The workflow splits generated rows into throughput, fixed-sequence eval, and agentic eval jobs in [`.github/workflows/e2e-tests.yml`](../.github/workflows/e2e-tests.yml#L278-L293). ## 2. Add a graded eval 1. Add `utils/evals/.yaml` using the lm-evaluation-harness task format. Pin the dataset/split, deterministic generation settings, prompt contract, filters, and primary metric. Use [`gsm8k.yaml`](../utils/evals/gsm8k.yaml) or [`gpqa_diamond.yaml`](../utils/evals/gpqa_diamond.yaml) as an in-tree pattern. 2. Give `task:` a stable name. That exact name is the key used by score thresholds and appears in collected rows. 3. Add the minimum accepted score to [`utils/evals/thresholds.yaml`](../utils/evals/thresholds.yaml). Put a general floor under `default`. Add `models..` only when a justified model-specific floor is required. -4. If the task's primary result is not compatible with the collector's strict/extract/accuracy rules, extend [`extract_lm_metrics()`](../utils/collect_eval_results.py#L114-L181). Do not publish a row whose `score` is null. +4. If the task's primary result is not compatible with the collector's strict/extract/accuracy rules, extend [`extract_lm_metrics()`](../utils/collect_eval_results.py#L115-L197). Do not publish a row whose `score` is null. 5. Run a small explicit slice, inspect samples, then run the full split. `EVAL_LIMIT` is a smoke-test control, not a publishable score setting. Against an already healthy OpenAI-compatible server: @@ -85,7 +85,7 @@ append_lm_eval_summary python3 utils/evals/validate_scores.py --model-prefix "$MODEL_PREFIX" ``` -`run_lm_eval` passes concurrency through `num_concurrent` in `--model_args`. It is deliberately an environment variable, not a `run_eval` CLI option. The exact invocation is in [`run_lm_eval()`](../benchmarks/benchmark_lib.sh#L890-L970). +`run_lm_eval` passes concurrency through `num_concurrent` in `--model_args`. It is deliberately an environment variable, not a `run_eval` CLI option. The exact invocation is in [`run_lm_eval()`](../benchmarks/benchmark_lib.sh#L1080-L1162). ## 3. `EVAL_ONLY` is a launcher contract @@ -97,9 +97,9 @@ Set `EVAL_ONLY=true` **before server launch**. It is not merely a switch inside 4. Throughput returns immediately or is skipped. 5. `run_eval` and artifact staging run. -Relevant implementation: [context setup](../benchmarks/benchmark_lib.sh#L853-L888), [eval dispatch and failure policy](../benchmarks/benchmark_lib.sh#L1537-L1654), and [workflow inputs](../.github/workflows/benchmark-tmpl.yml#L162-L185). +Relevant implementation: [context setup](../benchmarks/benchmark_lib.sh#L1049-L1078), [eval dispatch and failure policy](../benchmarks/benchmark_lib.sh#L1789-L1923), and [workflow inputs](../.github/workflows/benchmark-tmpl.yml#L79-L97). -Do not toggle `EVAL_ONLY` after a throughput-sized server is already running and assume the context changed. Restart through the recipe. In eval-only mode an eval failure is returned after available artifacts are staged. In a workflow, upload happens with `always()` before score validation so failed evidence survives ([single-node upload and gate](../.github/workflows/benchmark-tmpl.yml#L387-L404), [multi-node upload and gate](../.github/workflows/benchmark-multinode-tmpl.yml#L450-L468)). +Do not toggle `EVAL_ONLY` after a throughput-sized server is already running and assume the context changed. Restart through the recipe. In eval-only mode an eval failure is returned after available artifacts are staged. In a workflow, upload happens with `always()` before score validation so failed evidence survives ([single-node upload and gate](../.github/workflows/benchmark-tmpl.yml#L399-L417), [multi-node upload and gate](../.github/workflows/benchmark-multinode-tmpl.yml#L466-L488)). ## 4. Batched eval concurrency @@ -121,9 +121,9 @@ The batch runner creates a fresh temporary output directory per point, stages fi - `completed_eval_concs`: eval and staging both succeeded. - `failed_eval_concs`: either eval or staging failed. -A failed point is deferred so artifacts from every attempted point can upload. The post-upload validator then fails the job. Batched mode accepts positive integers and supports only `lm-eval`. See [`run_eval` batching](../benchmarks/benchmark_lib.sh#L1537-L1631), [artifact suffixing](../benchmarks/benchmark_lib.sh#L972-L1030), and [manifest validation](../utils/evals/validate_scores.py#L72-L171). +A failed point is deferred so artifacts from every attempted point can upload. The post-upload validator then fails the job. Batched mode accepts positive integers and supports only `lm-eval`. See [`run_eval` batching](../benchmarks/benchmark_lib.sh#L1839-L1900), [artifact suffixing](../benchmarks/benchmark_lib.sh#L1163-L1222), and [manifest validation](../utils/evals/validate_scores.py#L72-L171). -For multi-node `all-evals`, the workflow constructs `EVAL_CONC` by joining the topology's concurrency list ([dispatch](../.github/workflows/e2e-tests.yml#L375-L378)). Never compare a point if its `_conc` result or completed-manifest entry is missing. +For multi-node `all-evals`, the workflow constructs `EVAL_CONC` by joining the topology's concurrency list ([dispatch](../.github/workflows/e2e-tests.yml#L397-L400)). Never compare a point if its `_conc` result or completed-manifest entry is missing. ## 5. Validate scores, not file existence @@ -173,7 +173,7 @@ Retain `meta_env.json`, `results*.json`, and `sample*.jsonl`. Agentic SWE-bench ## 7. Run AgentX: fast feedback versus canonical evidence -AgentX is AIPerf `inferencex-agentx-mvp` trace replay, not a fixed-token synthetic benchmark. The checked-in default uses ten additional warmup requests per trajectory lane and the recipe's configured profile duration. `agentx-fast` forces one warmup request per lane and a 1,200-second profile. It affects single- and multi-node AgentX throughput only. Fixed-sequence throughput and evals remain canonical. Fast runs are not eligible for artifact reuse ([workflow policy](../.github/workflows/README.md#agentx-fast-mode), [fast replay settings](../benchmarks/benchmark_lib.sh#L1824-L1848)). +AgentX is AIPerf `inferencex-agentx-mvp` trace replay, not a fixed-token synthetic benchmark. The checked-in default uses ten additional warmup requests per trajectory lane and the recipe's configured profile duration. `agentx-fast` forces one warmup request per lane and a 1,200-second profile. It affects single- and multi-node AgentX throughput only. Fixed-sequence throughput and evals remain canonical. Fast runs are not eligible for artifact reuse ([workflow policy](../.github/workflows/README.md#agentx-fast-mode), [fast replay settings](../benchmarks/benchmark_lib.sh#L2104-L2128)). Targeted canonical run (configured duration and warmup, with fast and duration overrides omitted): @@ -199,17 +199,18 @@ Targeted AgentX SWE-bench smoke eval (first ten instances, real agentic generati gh workflow run e2e-tests.yml --repo SemiAnalysisAI/InferenceX --ref "$REF" \ -f generate-cli-command='test-config --config-keys qwen3.5-fp8-b200-sglang-agentic --conc 1 --evals-only --config-files configs/nvidia-master.yaml' \ -f test-name='swebench-smoke-qwen35-c1' \ + -f eval-framework=swebench \ -f eval-limit='10' \ -f swebench-gen-mode='agentic' ``` For a publishable SWE-bench score, omit `eval-limit`. Do not use `single-shot`, which is only a debugging escape hatch. SWE-bench generation/scoring controls and its `0.50` full-split threshold are documented next to the implementation in [`utils/evals/EVALS.md`](../utils/evals/EVALS.md#swe-bench-lite---framework-swebench). -Treat fast results as bring-up evidence, never as a replacement for the canonical candidate. A duration below 900 seconds or `AIPERF_UNSAFE_OVERRIDE=true` adds AIPerf's `--unsafe-override` and flags the submission invalid. Use it only for smoke diagnosis ([source](../benchmarks/benchmark_lib.sh#L1982-L1989)). After a fast run is healthy, run the exact candidate canonically before claiming benchmark success. +Treat fast results as bring-up evidence, never as a replacement for the canonical candidate. A duration below 900 seconds or `AIPERF_UNSAFE_OVERRIDE=true` adds AIPerf's `--unsafe-override` and flags the submission invalid. Use it only for smoke diagnosis ([source](../benchmarks/benchmark_lib.sh#L2266-L2268)). After a fast run is healthy, run the exact candidate canonically before claiming benchmark success. ## 8. Preserve trace and run provenance -AgentX defaults to recorded assistant-response replay. Live server outputs are measured but discarded when constructing later turns. Set `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1` only for an explicitly different live-assistant experiment. The selected trace corpus is model-family dependent unless `WEKA_LOADER_OVERRIDE` pins it. The resolver logs both loader and Hugging Face dataset ([trace resolution](../benchmarks/benchmark_lib.sh#L1743-L1822), [replay semantics](../benchmarks/benchmark_lib.sh#L1824-L1850)). +AgentX defaults to recorded assistant-response replay. Live server outputs are measured but discarded when constructing later turns. Set `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1` only for an explicitly different live-assistant experiment. The selected trace corpus is model-family dependent unless `WEKA_LOADER_OVERRIDE` pins it. The resolver logs both loader and Hugging Face dataset ([trace resolution](../benchmarks/benchmark_lib.sh#L2023-L2102), [replay semantics](../benchmarks/benchmark_lib.sh#L2104-L2270)). Capture orchestration provenance immediately: @@ -241,7 +242,7 @@ For each concurrency retain: - server/frontend logs and every metrics endpoint represented. - run URL/ID, attempt, head SHA, recipe/config identity, image, topology, fast flag, and any override. -The runner writes the command before replay and validates raw results after aggregation ([execution path](../benchmarks/benchmark_lib.sh#L2040-L2079)). Aggregation preserves dataset provenance and hardware/model/topology fields ([aggregate construction](../utils/agentic/aggregation/process_agentic_result.py#L194-L272)). Raw workflow uploads intentionally omit very large `inputs.json` and `profile_export_raw.jsonl`. If those are required for an investigation, preserve them from the live allocation before cleanup ([single-node artifact contract](../.github/workflows/benchmark-tmpl.yml#L337-L346), [multi-node contract](../.github/workflows/benchmark-multinode-tmpl.yml#L439-L448)). +The runner writes the command before replay and validates raw results after aggregation ([execution path](../benchmarks/benchmark_lib.sh#L2320-L2360)). Aggregation preserves dataset provenance and hardware/model/topology fields ([aggregate construction](../utils/agentic/aggregation/process_agentic_result.py#L194-L272)). Raw workflow uploads intentionally omit very large `inputs.json` and `profile_export_raw.jsonl`. If those are required for an investigation, preserve them from the live allocation before cleanup ([single-node artifact contract](../.github/workflows/benchmark-tmpl.yml#L349-L358), [multi-node contract](../.github/workflows/benchmark-multinode-tmpl.yml#L455-L464)). ## 9. Debug long AgentX runs from live evidence @@ -290,7 +291,7 @@ curl -fsS '' | \ rg -i 'request|queue|cache|token|prefill|decode|error|fail' ``` -Track trends over repeated samples: running/waiting requests, KV usage, prefix hits, input/output token rates, completed/cancelled/errored requests, frontend routing balance, and disaggregated KV transfer. AIPerf records endpoint identity for every server series ([metrics wiring](../benchmarks/benchmark_lib.sh#L1963-L1980)). +Track trends over repeated samples: running/waiting requests, KV usage, prefix hits, input/output token rates, completed/cancelled/errored requests, frontend routing balance, and disaggregated KV transfer. AIPerf records endpoint identity for every server series ([metrics wiring](../benchmarks/benchmark_lib.sh#L2236-L2260)). Use phase markers, not total Slurm age: diff --git a/docs/eval-agentx-procedures_zh.md b/docs/eval-agentx-procedures_zh.md index 669ed290c3..66c205782e 100644 --- a/docs/eval-agentx-procedures_zh.md +++ b/docs/eval-agentx-procedures_zh.md @@ -18,11 +18,11 @@ | 常规 sweep | 不加 eval 选项 | 吞吐量作业，加上选定的 8k/1k eval 子集 | | 仅吞吐量 | `--no-evals` | 不生成 eval 作业 | | 仅选定的 eval 子集 | `--evals-only` | 作业带有 `RUN_EVAL=true`、`EVAL_ONLY=true` | -| 仅运行所有符合条件的 eval | `--all-evals` | 等价于 `--evals-only --all-evals`；包含全部定长序列 8k/1k 行，以及单节点 agentic SWE-bench 行 | +| 仅运行所有符合条件的 eval | `--all-evals` | 等价于 `--evals-only --all-evals`；包含全部定长序列 8k/1k 行，以及单节点和多节点 agentic GSM8K 行 | | 在一个 recipe 中先跑吞吐量再跑 eval | `RUN_EVAL=true`、`EVAL_ONLY=false` | 启动服务，运行吞吐量，然后执行 `run_eval` | | 对新启动的服务仅运行 eval | `RUN_EVAL=true`、`EVAL_ONLY=true` | launcher 扩大 eval context，跳过吞吐量并运行 eval | -默认选择会区分场景。单节点定长序列 eval 对每个 8k/1k 的模型/runner/framework/precision/并行配置分组选取符合条件的中位和最高并发；多节点 eval 对每种拓扑选取符合条件的最高并发。低于 16 的并发不会被选中。Agentic eval 需要显式启用；不支持多节点 agentic eval。参见 [`mark_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L238-L339) 与 [`mark_all_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L342-L398)。 +默认选择会区分场景。单节点定长序列 eval 对每个 8k/1k 的模型/runner/framework/precision/并行配置分组选取符合条件的中位和最高并发；多节点 eval 对每种拓扑选取符合条件的最高并发。低于 16 的并发不会被选中。Agentic eval 需要显式启用；单节点和多节点 agentic 行均选择符合条件的最高并发。参见 [`mark_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L276-L396) 与 [`mark_all_eval_entries()`](../utils/matrix_logic/generate_sweep_configs.py#L399-L482)。 在 PR 上，应将一个主要 sweep label（通常为 `full-sweep-fail-fast`）与 eval modifier 组合使用。`all-evals` 在不抑制吞吐量的情况下扩大覆盖范围；`evals-only` 会抑制吞吐量；两者一起使用时只运行所有符合条件的 eval。带有 `evals-only` 的运行不可复用，而常规 full sweep 和 `all-evals` full sweep 可以复用。添加或移除 modifier 会重启当前 sweep（[label 策略](../.github/workflows/README.md#pr-eval-modifiers)）。 @@ -48,14 +48,14 @@ uv run --no-project --with pydantic --with pyyaml --python 3.12 \ --config-files configs/nvidia-master.yaml | jq . ``` -正确的 AgentX eval 行包含 `"scenario-type": "agentic-coding"`、`"run-eval": true` 和 `"eval-only": true`。工作流会在 [`.github/workflows/e2e-tests.yml`](../.github/workflows/e2e-tests.yml#L257-L271) 中将生成的行拆分到吞吐量、定长序列 eval 和 agentic eval 作业。 +正确的 AgentX eval 行包含 `"scenario-type": "agentic-coding"`、`"run-eval": true` 和 `"eval-only": true`。工作流会在 [`.github/workflows/e2e-tests.yml`](../.github/workflows/e2e-tests.yml#L278-L293) 中将生成的行拆分到吞吐量、定长序列 eval 和 agentic eval 作业。 ## 2. 添加评分 eval 1. 按照 lm-evaluation-harness task 格式添加 `utils/evals/.yaml`。固定 dataset/split、确定性生成设置、prompt 约定、filter 和主指标。可参考仓库内的 [`gsm8k.yaml`](../utils/evals/gsm8k.yaml) 或 [`gpqa_diamond.yaml`](../utils/evals/gpqa_diamond.yaml)。 2. 为 `task:` 指定稳定名称。分数阈值以该精确名称为键，收集后的行中也会出现该名称。 3. 在 [`utils/evals/thresholds.yaml`](../utils/evals/thresholds.yaml) 中添加最低可接受分数。通用下限放在 `default`；只有在确有依据需要模型专用下限时，才添加 `models..`。 -4. 如果 task 的主结果与 collector 的 strict/extract/accuracy 规则不兼容，请扩展 [`extract_lm_metrics()`](../utils/collect_eval_results.py#L114-L181)。不要发布 `score` 为 null 的行。 +4. 如果 task 的主结果与 collector 的 strict/extract/accuracy 规则不兼容，请扩展 [`extract_lm_metrics()`](../utils/collect_eval_results.py#L115-L197)。不要发布 `score` 为 null 的行。 5. 先运行一个显式的小切片并检查样本，再运行完整 split。`EVAL_LIMIT` 是 smoke test 控制项，不是可发布分数的运行设置。 对已经健康的 OpenAI-compatible 服务执行： @@ -85,7 +85,7 @@ append_lm_eval_summary python3 utils/evals/validate_scores.py --model-prefix "$MODEL_PREFIX" ``` -`run_lm_eval` 通过 `--model_args` 中的 `num_concurrent` 传递并发；它刻意采用环境变量，而不是 `run_eval` CLI 选项。准确调用见 [`run_lm_eval()`](../benchmarks/benchmark_lib.sh#L890-L970)。 +`run_lm_eval` 通过 `--model_args` 中的 `num_concurrent` 传递并发；它刻意采用环境变量，而不是 `run_eval` CLI 选项。准确调用见 [`run_lm_eval()`](../benchmarks/benchmark_lib.sh#L1080-L1162)。 ## 3. `EVAL_ONLY` 是 launcher 约定 @@ -97,9 +97,9 @@ python3 utils/evals/validate_scores.py --model-prefix "$MODEL_PREFIX" 4. 吞吐量路径立即返回或被跳过。 5. 运行 `run_eval` 和 artifact staging。 -相关实现：[context 设置](../benchmarks/benchmark_lib.sh#L853-L888)、[eval 分派与失败策略](../benchmarks/benchmark_lib.sh#L1537-L1654) 和[工作流输入](../.github/workflows/benchmark-tmpl.yml#L162-L185)。 +相关实现：[context 设置](../benchmarks/benchmark_lib.sh#L1049-L1078)、[eval 分派与失败策略](../benchmarks/benchmark_lib.sh#L1789-L1923) 和[工作流输入](../.github/workflows/benchmark-tmpl.yml#L79-L97)。 -不要在吞吐量规格的服务已经运行后才切换 `EVAL_ONLY`，并假定 context 会随之变化。应通过 recipe 重启。Eval-only 模式会在暂存已有 artifact 后返回 eval 失败；在工作流中，上传步骤使用 `always()`，并位于分数校验前，因此失败证据仍会保留（[单节点上传与 gate](../.github/workflows/benchmark-tmpl.yml#L387-L404)、[多节点上传与 gate](../.github/workflows/benchmark-multinode-tmpl.yml#L450-L468)）。 +不要在吞吐量规格的服务已经运行后才切换 `EVAL_ONLY`，并假定 context 会随之变化。应通过 recipe 重启。Eval-only 模式会在暂存已有 artifact 后返回 eval 失败；在工作流中，上传步骤使用 `always()`，并位于分数校验前，因此失败证据仍会保留（[单节点上传与 gate](../.github/workflows/benchmark-tmpl.yml#L399-L417)、[多节点上传与 gate](../.github/workflows/benchmark-multinode-tmpl.yml#L466-L488)）。 ## 4. 批量 eval 并发 @@ -121,9 +121,9 @@ python3 utils/evals/validate_scores.py --expected-concs '16 32 64' - `completed_eval_concs`：eval 与 staging 均成功的点； - `failed_eval_concs`：eval 或 staging 失败的点。 -失败点会延迟报错，使所有已尝试点的 artifact 都能上传；随后 post-upload validator 会使作业失败。批量模式只接受正整数，且仅支持 `lm-eval`。参见 [`run_eval` batching](../benchmarks/benchmark_lib.sh#L1537-L1631)、[artifact 后缀处理](../benchmarks/benchmark_lib.sh#L972-L1030) 和[manifest 校验](../utils/evals/validate_scores.py#L72-L171)。 +失败点会延迟报错，使所有已尝试点的 artifact 都能上传；随后 post-upload validator 会使作业失败。批量模式只接受正整数，且仅支持 `lm-eval`。参见 [`run_eval` batching](../benchmarks/benchmark_lib.sh#L1839-L1900)、[artifact 后缀处理](../benchmarks/benchmark_lib.sh#L1163-L1222) 和[manifest 校验](../utils/evals/validate_scores.py#L72-L171)。 -对于多节点 `all-evals`，工作流通过连接拓扑的并发列表构造 `EVAL_CONC`（[分派](../.github/workflows/e2e-tests.yml#L375-L378)）。如果缺少某点的 `_conc` 结果或 completed manifest 条目，绝不能比较该点。 +对于多节点 `all-evals`，工作流通过连接拓扑的并发列表构造 `EVAL_CONC`（[分派](../.github/workflows/e2e-tests.yml#L397-L400)）。如果缺少某点的 `_conc` 结果或 completed manifest 条目，绝不能比较该点。 ## 5. 校验分数，而不只是检查文件存在 @@ -173,7 +173,7 @@ gh run download "$RUN_ID" --repo SemiAnalysisAI/InferenceX \ ## 7. 运行 AgentX：快速反馈与 canonical 证据 -AgentX 是 AIPerf `inferencex-agentx-mvp` trace replay，不是固定 token 的合成 benchmark。仓库默认设置对每条 trajectory lane 额外执行十个 warmup 请求，并使用 recipe 配置的 profile 时长。`agentx-fast` 强制每条 lane 只运行一个 warmup 请求，并将 profile 设为 1,200 秒。它只影响单节点和多节点 AgentX 吞吐量；定长序列吞吐量与 eval 保持 canonical。Fast 运行不符合 artifact reuse 条件（[工作流策略](../.github/workflows/README.md#agentx-fast-mode)、[Fast replay 设置](../benchmarks/benchmark_lib.sh#L1824-L1848)）。 +AgentX 是 AIPerf `inferencex-agentx-mvp` trace replay，不是固定 token 的合成 benchmark。仓库默认设置对每条 trajectory lane 额外执行十个 warmup 请求，并使用 recipe 配置的 profile 时长。`agentx-fast` 强制每条 lane 只运行一个 warmup 请求，并将 profile 设为 1,200 秒。它只影响单节点和多节点 AgentX 吞吐量；定长序列吞吐量与 eval 保持 canonical。Fast 运行不符合 artifact reuse 条件（[工作流策略](../.github/workflows/README.md#agentx-fast-mode)、[fast replay 设置](../benchmarks/benchmark_lib.sh#L2104-L2128)）。 目标 canonical 运行（使用配置的 duration 和 warmup；不要加 fast 或 duration override）： @@ -199,17 +199,18 @@ gh workflow run e2e-tests.yml --repo SemiAnalysisAI/InferenceX --ref "$REF" \ gh workflow run e2e-tests.yml --repo SemiAnalysisAI/InferenceX --ref "$REF" \ -f generate-cli-command='test-config --config-keys qwen3.5-fp8-b200-sglang-agentic --conc 1 --evals-only --config-files configs/nvidia-master.yaml' \ -f test-name='swebench-smoke-qwen35-c1' \ + -f eval-framework=swebench \ -f eval-limit='10' \ -f swebench-gen-mode='agentic' ``` 要得到可发布的 SWE-bench 分数，省略 `eval-limit`；不要使用 `single-shot`，它只是诊断逃生选项。SWE-bench generation/scoring 控制项以及完整 split 的 `0.50` 阈值在实现旁的 [`utils/evals/EVALS.md`](../utils/evals/EVALS.md#swe-bench-lite---framework-swebench) 中说明。 -Fast 结果只能作为 bring-up 证据，绝不能替代 canonical candidate。小于 900 秒的 duration 或 `AIPERF_UNSAFE_OVERRIDE=true` 会添加 AIPerf 的 `--unsafe-override` 并将 submission 标记为无效；只能用于 smoke 诊断（[源码](../benchmarks/benchmark_lib.sh#L1982-L1989)）。Fast 运行健康后，必须对完全相同的 candidate 进行 canonical 运行，才能宣称 benchmark 成功。 +Fast 结果只能作为 bring-up 证据，绝不能替代 canonical candidate。小于 900 秒的 duration 或 `AIPERF_UNSAFE_OVERRIDE=true` 会添加 AIPerf 的 `--unsafe-override` 并将 submission 标记为无效；只能用于 smoke 诊断（[源码](../benchmarks/benchmark_lib.sh#L2266-L2268)）。Fast 运行健康后，必须对完全相同的 candidate 进行 canonical 运行，才能宣称 benchmark 成功。 ## 8. 保留 trace 与运行 provenance -AgentX 默认 replay 已记录的 assistant response。实时服务输出会被测量，但构造后续 turn 时会丢弃。只有在明确要进行不同的 live-assistant 实验时，才设置 `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1`。除非用 `WEKA_LOADER_OVERRIDE` 固定，否则所选 trace corpus 依赖模型 family；resolver 会同时记录 loader 与 Hugging Face dataset（[trace 解析](../benchmarks/benchmark_lib.sh#L1743-L1822)、[replay 语义](../benchmarks/benchmark_lib.sh#L1824-L1850)）。 +AgentX 默认 replay 已记录的 assistant response。实时服务输出会被测量，但构造后续 turn 时会丢弃。只有在明确要进行不同的 live-assistant 实验时，才设置 `AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES=1`。除非用 `WEKA_LOADER_OVERRIDE` 固定，否则所选 trace corpus 依赖模型 family；resolver 会同时记录 loader 与 Hugging Face dataset（[trace 解析](../benchmarks/benchmark_lib.sh#L2023-L2102)、[replay 语义](../benchmarks/benchmark_lib.sh#L2104-L2270)）。 立即记录 orchestration provenance： @@ -241,7 +242,7 @@ gh run download "$RUN_ID" --repo SemiAnalysisAI/InferenceX \ - server/frontend 日志以及所代表的每个 metrics endpoint； - run URL/ID、attempt、head SHA、recipe/config 标识、image、topology、fast 标志和所有 override。 -Runner 会在 replay 前写入命令，并在聚合后校验原始结果（[执行路径](../benchmarks/benchmark_lib.sh#L2040-L2079)）。聚合会保留 dataset provenance 以及硬件/模型/拓扑字段（[aggregate 构造](../utils/agentic/aggregation/process_agentic_result.py#L194-L272)）。工作流的 raw upload 会有意排除体积很大的 `inputs.json` 和 `profile_export_raw.jsonl`；如果调查需要这些文件，应在清理前从实时 allocation 保存（[单节点 artifact 约定](../.github/workflows/benchmark-tmpl.yml#L337-L346)、[多节点约定](../.github/workflows/benchmark-multinode-tmpl.yml#L439-L448)）。 +Runner 会在 replay 前写入命令，并在聚合后校验原始结果（[执行路径](../benchmarks/benchmark_lib.sh#L2320-L2360)）。聚合会保留 dataset provenance 以及硬件/模型/拓扑字段（[aggregate 构造](../utils/agentic/aggregation/process_agentic_result.py#L194-L272)）。工作流的 raw upload 会有意排除体积很大的 `inputs.json` 和 `profile_export_raw.jsonl`；如果调查需要这些文件，应在清理前从实时 allocation 保存（[单节点 artifact 约定](../.github/workflows/benchmark-tmpl.yml#L349-L358)、[多节点约定](../.github/workflows/benchmark-multinode-tmpl.yml#L455-L464)）。 ## 9. 用实时证据调试长时间 AgentX 运行 @@ -290,7 +291,7 @@ curl -fsS '' | \ rg -i 'request|queue|cache|token|prefill|decode|error|fail' ``` -通过重复 sample 跟踪趋势：running/waiting request、KV usage、prefix hit、input/output token rate、completed/cancelled/errored request、frontend routing balance，以及 disaggregated KV transfer。AIPerf 会为每条 server series 记录 endpoint identity（[metrics 接线](../benchmarks/benchmark_lib.sh#L1963-L1980)）。 +通过重复 sample 跟踪趋势：running/waiting request、KV usage、prefix hit、input/output token rate、completed/cancelled/errored request、frontend routing balance，以及 disaggregated KV transfer。AIPerf 会为每条 server series 记录 endpoint identity（[metrics 接线](../benchmarks/benchmark_lib.sh#L2236-L2260)）。 应使用 phase marker，而不是 Slurm 总运行时间： diff --git a/runners/inject_synthetic_acceptance.py b/runners/inject_synthetic_acceptance.py index 80d2740e79..1382454d13 100644 --- a/runners/inject_synthetic_acceptance.py +++ b/runners/inject_synthetic_acceptance.py @@ -1,18 +1,13 @@ #!/usr/bin/env python3 -"""Inject synthetic acceptance parameters into an srt-slurm recipe (generic driver). +"""Configure speculative acceptance in an srt-slurm recipe. -This is the framework-agnostic half of the synthetic-acceptance mechanism. It -decides *whether* to inject (the ``SYNTHETIC_ACCEPTANCE`` flag) and *what* mean -acceptance length to inject, then delegates the actual recipe rewrite to a -framework-specific backend (see ``runners/synthetic_injectors/``). - -The script is a no-op (exit 0, file untouched) when: - - SYNTHETIC_ACCEPTANCE is unset/false, -so existing callers that do not opt in get exactly the previous behavior. When -enabled it requires a backend registered for the given framework; the vLLM -backend is added in a follow-up framework-support change. +Eval-only runs remove synthetic acceptance so generated text is checked against +the target model. Throughput runs inject a configured synthetic acceptance +length only when ``SYNTHETIC_ACCEPTANCE=true``. Framework-specific rewriting +lives under ``runners/synthetic_injectors/``. Environment variables: + EVAL_ONLY "true" to restore real target verification SYNTHETIC_ACCEPTANCE "true" to enable (default: "false") SYNTHETIC_ACCEPTANCE_LENGTH target mean acceptance length; if unset, it is auto-resolved from the reference AL YAML using @@ -72,7 +67,9 @@ def _lookup_al(model_block, num_spec_tokens): if isinstance(model_block, dict): # Thinking matrix form: pick the requested mode, then index by level. if any(str(k).startswith("thinking") for k in model_block): - mode = os.environ.get("THINKING_MODE", "thinking_on").strip() or "thinking_on" + mode = ( + os.environ.get("THINKING_MODE", "thinking_on").strip() or "thinking_on" + ) mode_block = model_block.get(mode) if mode_block is None: sys.exit( @@ -115,7 +112,9 @@ def _resolve_al(config_text, injector, ref_yaml): al = _lookup_al(model_block, num_spec_tokens) if al is None: - sys.exit(f"ERROR: num_spec_tokens={num_spec_tokens} not found for {key} in {ref_yaml}") + sys.exit( + f"ERROR: num_spec_tokens={num_spec_tokens} not found for {key} in {ref_yaml}" + ) _log( f"Auto-resolved AL={al} from {ref_yaml} " @@ -125,14 +124,30 @@ def _resolve_al(config_text, injector, ref_yaml): def inject(config_file, framework): + injector = get_injector(framework) + if _enabled("EVAL_ONLY"): - print("[Synthetic AL] EVAL_ONLY=true: keeping real MTP recipe") + if injector is None or not hasattr(injector, "rewrite_real"): + print( + f"[Synthetic AL] EVAL_ONLY=true: no real-acceptance rewriter " + f"for FRAMEWORK='{framework}'" + ) + return 0 + + with open(config_file) as f: + content = f.read() + new_content, count = injector.rewrite_real(content, _log) + if count: + with open(config_file, "w") as f: + f.write(new_content) + _log(f"Restored real acceptance in {count} speculative-config entries") + else: + _log("EVAL_ONLY=true: recipe already uses real acceptance") return 0 if not _enabled("SYNTHETIC_ACCEPTANCE"): return 0 - injector = get_injector(framework) if injector is None: sys.exit( "ERROR: SYNTHETIC_ACCEPTANCE=true but no synthetic-acceptance " @@ -145,7 +160,12 @@ def inject(config_file, framework): al = _resolve_al( content, injector, - os.path.join(os.path.dirname(__file__), "..", "benchmarks", "speedbench-reference-al.yaml"), + os.path.join( + os.path.dirname(__file__), + "..", + "benchmarks", + "speedbench-reference-al.yaml", + ), ) _log(f"Injecting synthetic acceptance (length={al}) into {config_file}") diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 8c2b357b00..7758ad90ee 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -220,6 +220,9 @@ if [[ "$IS_MULTINODE" == "true" ]]; then cd "$SRT_REPO_DIR" || exit 1 git checkout sa-submission-q2-2026 fi + if [[ "${EVAL_FRAMEWORK:-lm-eval}" == "kimi-vendor" ]]; then + python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" || exit 1 + fi echo "Installing srtctl..." export UV_INSTALL_DIR="$GITHUB_WORKSPACE/.local/bin" @@ -368,6 +371,8 @@ EOF # so large-model loads (e.g. DSR1-FP8 ~680GB off shared FS) finish in time. # Uses ${CONFIG_FILE%%:*} because CONFIG_FILE may carry an :override[N] suffix. sed -i 's/^ max_attempts: [0-9]*/ max_attempts: 720/' "${CONFIG_FILE%%:*}" + python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \ + "${CONFIG_FILE%%:*}" "$FRAMEWORK" || exit 1 SRTCTL_PREFLIGHT_ARGS=() # Kimi K2.6 weights are staged on the Slurm compute nodes, not the login node. diff --git a/runners/launch_b300-nv.sh b/runners/launch_b300-nv.sh index cad9ba30aa..5fced02ffb 100644 --- a/runners/launch_b300-nv.sh +++ b/runners/launch_b300-nv.sh @@ -481,8 +481,21 @@ else export GPU_COUNT="${GPU_COUNT:-${TP:?TP must be set}}" - SALLOC_TIME_LIMIT="${SALLOC_TIME_LIMIT:-480}" - salloc --partition=$SLURM_PARTITION --account=$SLURM_ACCOUNT -N 1 --gres=gpu:$GPU_COUNT --exclusive --mem=0 --time="$SALLOC_TIME_LIMIT" --no-shell --job-name="$RUNNER_NAME" + SALLOC_ARGS=( + --partition="$SLURM_PARTITION" + --account="$SLURM_ACCOUNT" + -N 1 + --gres="gpu:$GPU_COUNT" + --exclusive + --mem=0 + --time="${SALLOC_TIME_LIMIT:-480}" + --no-shell + --job-name="$RUNNER_NAME" + ) + if [[ -n "${SALLOC_EXCLUDE:-}" ]]; then + SALLOC_ARGS+=(--exclude="$SALLOC_EXCLUDE") + fi + salloc "${SALLOC_ARGS[@]}" JOB_ID=$(squeue --name="$RUNNER_NAME" -u "$USER" -h -o %A | head -n1) srun --jobid=$JOB_ID \ diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 660ae078d5..c4c0e53847 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -503,6 +503,9 @@ else git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" fi +if [[ "${EVAL_FRAMEWORK:-lm-eval}" == "kimi-vendor" ]]; then + python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" || exit 1 +fi echo "Installing srtctl..." curl -LsSf https://astral.sh/uv/install.sh | sh @@ -650,10 +653,10 @@ if command -v squeue >/dev/null 2>&1; then fi sed -i "s/^name:.*/name: \"${SRT_SLURM_JOB_NAME}\"/" "$CONFIG_PATH" -# Optionally inject synthetic acceptance into the recipe's speculative-config -# when SYNTHETIC_ACCEPTANCE=true (no-op otherwise). Must run after the name -# override and before srtctl apply so the rendered job picks it up. -python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" "$CONFIG_PATH" "$FRAMEWORK" +# Restore real acceptance for eval-only jobs, or inject synthetic acceptance +# when a throughput run explicitly enables it. +python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \ + "$CONFIG_PATH" "$FRAMEWORK" || exit 1 # Don't leak the login-node venv to the compute-node orchestrator. sbatch's # default --export=ALL propagates VIRTUAL_ENV (set by `source diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index f904d8bc27..61c290146e 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -106,6 +106,9 @@ if [[ "$IS_MULTINODE" == "true" ]]; then cd "$SRT_REPO_DIR" git checkout sa-submission-q2-2026 fi + if [[ "${EVAL_FRAMEWORK:-lm-eval}" == "kimi-vendor" ]]; then + python3 "$GITHUB_WORKSPACE/runners/patch_srt_eval_dispatch.py" "$(pwd)" + fi echo "Installing srtctl..." curl -LsSf https://astral.sh/uv/install.sh | sh @@ -233,6 +236,8 @@ EOF sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" sed -i '/^health_check:/,/^[^ ]/{ /^health_check:/d; /^ /d; }' "$CONFIG_PATH" printf '\nhealth_check:\n max_attempts: 720\n interval_seconds: 10\n' >> "$CONFIG_PATH" + python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \ + "$CONFIG_PATH" "$FRAMEWORK" WORKLOAD_TAG="${ISL}x${OSL}" if [[ "$IS_AGENTIC" == "1" ]]; then WORKLOAD_TAG="agentic" diff --git a/runners/patch_srt_eval_dispatch.py b/runners/patch_srt_eval_dispatch.py new file mode 100755 index 0000000000..daa52d9ab0 --- /dev/null +++ b/runners/patch_srt_eval_dispatch.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Enable InferenceX-selected eval dispatch in an srt-slurm checkout.""" + +from __future__ import annotations + +import sys +from pathlib import Path + + +DO_SWEEP_ENV_BLOCK = """ "EVAL_ONLY", + "IS_MULTINODE",""" +DO_SWEEP_ENV_REPLACEMENT = """ "EVAL_ONLY", + "EVAL_FRAMEWORK", + "EVAL_SUITE", + "IS_MULTINODE",""" +LM_EVAL_COMMAND = 'run_eval --framework lm-eval --port "$PORT" || eval_rc=$?' +GENERIC_EVAL_COMMAND = 'run_eval --port "$PORT" || eval_rc=$?' +EVAL_ARTIFACT_COPY = """cp -v results*.json /logs/eval_results/ 2>/dev/null || true +cp -v sample*.jsonl /logs/eval_results/ 2>/dev/null || true""" +KIMI_ARTIFACT_COPY = """cp -v results*.json /logs/eval_results/ 2>/dev/null || true +cp -v *_vendor_report.json /logs/eval_results/ 2>/dev/null || true +cp -v sample*.jsonl /logs/eval_results/ 2>/dev/null || true""" + + +def prepare_replacements( + path: Path, + replacements: tuple[tuple[str, str], ...], +) -> tuple[str, str, bool]: + """Validate source replacements without mutating the checkout.""" + original = path.read_text() + content = original + changed = False + for old, new in replacements: + old_count = content.count(old) + new_count = content.count(new) + if old_count == 1 and new_count == 0: + content = content.replace(old, new, 1) + changed = True + elif old_count != 0 or new_count != 1: + raise RuntimeError( + f"invalid patch state in {path}: old anchor count={old_count}, " + f"replacement count={new_count}" + ) + return original, content, changed + + +def patch_checkout(root: Path) -> list[Path]: + """Patch both post-eval sources after validating the complete checkout.""" + patches = ( + ( + root / "src/srtctl/cli/do_sweep.py", + ((DO_SWEEP_ENV_BLOCK, DO_SWEEP_ENV_REPLACEMENT),), + ), + ( + root / "src/srtctl/benchmarks/scripts/lm-eval/bench.sh", + ( + (LM_EVAL_COMMAND, GENERIC_EVAL_COMMAND), + (EVAL_ARTIFACT_COPY, KIMI_ARTIFACT_COPY), + ), + ), + ) + staged = [ + (path, *prepare_replacements(path, replacements)) + for path, replacements in patches + ] + changed = [] + written = [] + try: + for path, original, replacement, needs_write in staged: + if needs_write: + path.write_text(replacement) + written.append((path, original)) + changed.append(path) + except OSError: + for path, original in reversed(written): + path.write_text(original) + raise + return changed + + +def main(argv: list[str]) -> int: + """Patch the checkout named on the command line.""" + if len(argv) != 2: + print(f"Usage: {argv[0]} SRT_SLURM_CHECKOUT", file=sys.stderr) + return 2 + root = Path(argv[1]).resolve() + try: + changed = patch_checkout(root) + except (OSError, RuntimeError) as error: + print( + f"ERROR: failed to patch srt-slurm eval dispatch: {error}", file=sys.stderr + ) + return 1 + if changed: + for path in changed: + print(f"Patched srt-slurm eval dispatch: {path}") + else: + print("srt-slurm eval dispatch is already patched") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/runners/synthetic_injectors/vllm.py b/runners/synthetic_injectors/vllm.py index e71a5df2b4..d36740195c 100644 --- a/runners/synthetic_injectors/vllm.py +++ b/runners/synthetic_injectors/vllm.py @@ -1,13 +1,8 @@ -"""vLLM synthetic-acceptance backend (FRAMEWORK=dynamo-vllm). +"""vLLM speculative-acceptance recipe rewriting. -Rewrites every ``speculative-config: ''`` entry in an srt-slurm recipe to -use synthetic rejection sampling: it adds ``rejection_sample_method=synthetic`` -and ``synthetic_acceptance_length=`` to the JSON so the engine emits a -controlled mean acceptance length instead of running the real draft model. - -Registered under the "dynamo-vllm" framework key at import time, so importing -the ``synthetic_injectors`` package is enough for the generic driver to resolve -this backend. +Throughput opt-ins can inject synthetic acceptance. Eval-only runs restore real +block verification so model outputs remain valid for accuracy checks. The +backend is registered for both direct vLLM and Dynamo-vLLM recipes. """ import json @@ -57,7 +52,9 @@ def _replace(match): new_content, count = _SPEC_CONFIG_RE.subn(_replace, content) if count: - after = [ln.strip() for ln in new_content.splitlines() if _SPEC_CONFIG_RE.search(ln)] + after = [ + ln.strip() for ln in new_content.splitlines() if _SPEC_CONFIG_RE.search(ln) + ] if after: log("After:") for ln in after: @@ -66,4 +63,28 @@ def _replace(match): return new_content, count +def rewrite_real(content, log): + """Restore real block verification in every synthetic config entry.""" + modified = 0 + + def _replace(match): + nonlocal modified + spec = json.loads(match.group(1)) + if ( + spec.get("rejection_sample_method") != "synthetic" + and "synthetic_acceptance_length" not in spec + ): + return match.group(0) + spec["rejection_sample_method"] = "block" + spec.pop("synthetic_acceptance_length", None) + modified += 1 + return "speculative-config: '" + json.dumps(spec, separators=(",", ":")) + "'" + + new_content = _SPEC_CONFIG_RE.sub(_replace, content) + if modified: + log("Restored real block verification for eval-only mode") + return new_content, modified + + register("dynamo-vllm", sys.modules[__name__]) +register("vllm", sys.modules[__name__]) diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index c027cc9b2f..a89a3354bb 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -1,9 +1,14 @@ +import json +import os import subprocess from pathlib import Path +import yaml REPO_ROOT = Path(__file__).resolve().parents[1] SLURM_UTILS = REPO_ROOT / "runners" / "slurm_utils.sh" +PATCH_SRT_EVAL = REPO_ROOT / "runners" / "patch_srt_eval_dispatch.py" +INJECT_ACCEPTANCE = REPO_ROOT / "runners" / "inject_synthetic_acceptance.py" def run_bash(command: str, *args: Path | str) -> subprocess.CompletedProcess[str]: @@ -55,3 +60,213 @@ def test_copy_agentic_results_fails_when_aggregate_is_missing( assert result.returncode != 0 assert "no run_conc*.json results found" in result.stderr + + +def test_patch_srt_eval_dispatch_forwards_selection_and_is_idempotent( + tmp_path: Path, +) -> None: + do_sweep = tmp_path / "src/srtctl/cli/do_sweep.py" + eval_script = tmp_path / "src/srtctl/benchmarks/scripts/lm-eval/bench.sh" + do_sweep.parent.mkdir(parents=True) + eval_script.parent.mkdir(parents=True) + do_sweep.write_text( + " for var in [\n" + ' "RUN_EVAL",\n' + ' "EVAL_ONLY",\n' + ' "IS_MULTINODE",\n' + " ]:\n" + ) + eval_script.write_text( + 'run_eval --framework lm-eval --port "$PORT" || eval_rc=$?\n' + "cp -v results*.json /logs/eval_results/ 2>/dev/null || true\n" + "cp -v sample*.jsonl /logs/eval_results/ 2>/dev/null || true\n" + ) + + first = subprocess.run( + ["python3", str(PATCH_SRT_EVAL), str(tmp_path)], + check=False, + capture_output=True, + text=True, + ) + second = subprocess.run( + ["python3", str(PATCH_SRT_EVAL), str(tmp_path)], + check=False, + capture_output=True, + text=True, + ) + + assert first.returncode == 0, first.stderr + assert second.returncode == 0, second.stderr + assert do_sweep.read_text().count('"EVAL_FRAMEWORK"') == 1 + assert do_sweep.read_text().count('"EVAL_SUITE"') == 1 + assert 'run_eval --port "$PORT"' in eval_script.read_text() + assert "--framework lm-eval" not in eval_script.read_text() + assert "*_vendor_report.json" in eval_script.read_text() + assert "already patched" in second.stdout + + +def test_patch_srt_eval_dispatch_preflights_before_writing(tmp_path: Path) -> None: + do_sweep = tmp_path / "src/srtctl/cli/do_sweep.py" + eval_script = tmp_path / "src/srtctl/benchmarks/scripts/lm-eval/bench.sh" + do_sweep.parent.mkdir(parents=True) + eval_script.parent.mkdir(parents=True) + original_do_sweep = ' "EVAL_ONLY",\n "IS_MULTINODE",\n' + original_eval_script = "unsupported eval hook\n" + do_sweep.write_text(original_do_sweep) + eval_script.write_text(original_eval_script) + + result = subprocess.run( + ["python3", str(PATCH_SRT_EVAL), str(tmp_path)], + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert do_sweep.read_text() == original_do_sweep + assert eval_script.read_text() == original_eval_script + + +def test_patch_srt_eval_dispatch_rejects_mixed_patch_state(tmp_path: Path) -> None: + do_sweep = tmp_path / "src/srtctl/cli/do_sweep.py" + eval_script = tmp_path / "src/srtctl/benchmarks/scripts/lm-eval/bench.sh" + do_sweep.parent.mkdir(parents=True) + eval_script.parent.mkdir(parents=True) + original_do_sweep = ( + ' "EVAL_ONLY",\n' + ' "IS_MULTINODE",\n' + ' "EVAL_ONLY",\n' + ' "EVAL_FRAMEWORK",\n' + ' "EVAL_SUITE",\n' + ' "IS_MULTINODE",\n' + ) + original_eval_script = ( + 'run_eval --framework lm-eval --port "$PORT" || eval_rc=$?\n' + "cp -v results*.json /logs/eval_results/ 2>/dev/null || true\n" + "cp -v sample*.jsonl /logs/eval_results/ 2>/dev/null || true\n" + ) + do_sweep.write_text(original_do_sweep) + eval_script.write_text(original_eval_script) + + result = subprocess.run( + ["python3", str(PATCH_SRT_EVAL), str(tmp_path)], + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 1 + assert "invalid patch state" in result.stderr + assert do_sweep.read_text() == original_do_sweep + assert eval_script.read_text() == original_eval_script + + +def test_eval_only_restores_real_vllm_acceptance(tmp_path: Path) -> None: + recipe = tmp_path / "recipe.yaml" + recipe.write_text( + "speculative-config: " + """'{\"method\":\"dspark\",\"num_speculative_tokens\":2,""" + """\"rejection_sample_method\":\"synthetic\",""" + """\"synthetic_acceptance_length\":2.51}'\n""" + ) + env = { + **os.environ, + "EVAL_ONLY": "true", + "SYNTHETIC_ACCEPTANCE": "true", + } + + result = subprocess.run( + ["python3", str(INJECT_ACCEPTANCE), str(recipe), "vllm"], + env=env, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + rewritten = recipe.read_text() + assert '"rejection_sample_method":"block"' in rewritten + assert "synthetic_acceptance_length" not in rewritten + + +def test_eval_only_acceptance_rewrite_allows_non_speculative_recipe( + tmp_path: Path, +) -> None: + recipe = tmp_path / "recipe.yaml" + original = "backend:\n type: vllm\n" + recipe.write_text(original) + + result = subprocess.run( + ["python3", str(INJECT_ACCEPTANCE), str(recipe), "dynamo-vllm"], + env={**os.environ, "EVAL_ONLY": "true"}, + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + assert recipe.read_text() == original + + +def test_nvidia_srt_launchers_prepare_kimi_eval_dispatch() -> None: + launchers = ( + REPO_ROOT / "runners/launch_h200-dgxc-slurm.sh", + REPO_ROOT / "runners/launch_b200-dgxc.sh", + REPO_ROOT / "runners/launch_gb200-nv.sh", + ) + + for launcher in launchers: + content = launcher.read_text() + assert "patch_srt_eval_dispatch.py" in content + assert 'EVAL_FRAMEWORK:-lm-eval}" == "kimi-vendor"' in content + assert "inject_synthetic_acceptance.py" in content + + +def test_gb200_kimi_compilation_config_preserves_all_settings() -> None: + recipes = { + "agg-gb200-tep16-balanced-agentic.yaml": 96, + "agg-gb200-tp16-latency-agentic.yaml": 24, + } + recipe_dir = ( + REPO_ROOT / "benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" + ) + + for filename, largest_capture in recipes.items(): + recipe = yaml.safe_load((recipe_dir / filename).read_text()) + raw_config = recipe["backend"]["vllm_config"]["aggregated"][ + "compilation-config" + ] + compilation_config = json.loads(raw_config) + + assert compilation_config["cudagraph_capture_sizes"][-1] == largest_capture + assert compilation_config["pass_config"]["fuse_allreduce_rms"] is False + + +def test_gb200_dynamo_kimi_recipes_enable_structural_tool_constraints() -> None: + recipe_dir = ( + REPO_ROOT / "benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" + ) + recipes = [ + yaml.safe_load(path.read_text()) + for path in recipe_dir.glob("agg-gb200-*-agentic.yaml") + ] + + assert recipes + for recipe in recipes: + frontend = recipe["frontend"] + assert frontend["type"] == "dynamo" + config = recipe["backend"]["vllm_config"]["aggregated"] + assert config["dyn-tool-call-parser"] == "kimi_k3" + assert config["dyn-enable-structural-tag"] is True + + +def test_b200_kimi_recipe_uses_available_roce_devices() -> None: + recipe_path = ( + REPO_ROOT + / "benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" + / "agg-b200-tp8pp2-agentic.yaml" + ) + recipe = yaml.safe_load(recipe_path.read_text()) + devices = recipe["backend"]["aggregated_environment"]["UCX_NET_DEVICES"] + + assert devices == ",".join(f"mlx5_{index}:1" for index in range(8)) diff --git a/utils/collect_eval_results.py b/utils/collect_eval_results.py index 667e60bc6f..545ef3a855 100644 --- a/utils/collect_eval_results.py +++ b/utils/collect_eval_results.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import sys import json +import math import re from pathlib import Path from typing import Any, Dict, List, Optional, Tuple @@ -32,6 +33,7 @@ SPEC_DECODING = "Spec Decode" CONC_SUFFIX_RE = re.compile(r"_conc(\d+)(?:_\d+)?\.json$") +EVAL_RESULT_FORMAT = "inferencex-eval-v1" def load_json(path: Path) -> Optional[Dict[str, Any]]: @@ -71,10 +73,10 @@ def result_concurrency(path: Path) -> Optional[int]: def detect_lm_eval_jsons(d: Path, batched: bool = False) -> List[Path]: - """Return lm-eval result JSONs from one artifact directory. + """Return the latest collector-compatible eval result JSONs. - Legacy artifacts contribute their latest result file. Batched artifacts - contribute the latest result file for each `_concN` suffix. + Result filenames contain sortable timestamps. Mtime remains a fallback for + legacy names, with the filename as a deterministic tie-breaker. """ immediate_jsons = set(d.glob('results*.json')) immediate_jsons.update( @@ -82,17 +84,28 @@ def detect_lm_eval_jsons(d: Path, batched: bool = False) -> List[Path]: ) lm_paths = [] + def recency_key(path: Path) -> Tuple[str, int, str]: + match = re.search( + r"\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}(?:\.\d+)?", + path.name, + ) + return ( + match.group(0) if match else "", + path.stat().st_mtime_ns, + path.name, + ) + for p in immediate_jsons: data = load_json(p) if not isinstance(data, dict): continue - if 'lm_eval_version' in data: + if data.get('result_format') == EVAL_RESULT_FORMAT or 'lm_eval_version' in data: lm_paths.append(p) if not lm_paths: return [] if not batched: - return [max(lm_paths, key=lambda path: path.stat().st_mtime)] + return [max(lm_paths, key=recency_key)] latest_by_conc: Dict[int, Path] = {} for path in lm_paths: @@ -100,15 +113,28 @@ def detect_lm_eval_jsons(d: Path, batched: bool = False) -> List[Path]: if conc is None: continue current = latest_by_conc.get(conc) - if current is None or path.stat().st_mtime > current.stat().st_mtime: + if current is None or recency_key(path) > recency_key(current): latest_by_conc[conc] = path return [latest_by_conc[conc] for conc in sorted(latest_by_conc)] -def detect_eval_jsons(d: Path) -> Tuple[Optional[Path], Optional[Path]]: - """Return the latest legacy lm-eval JSON and deprecated second slot.""" - lm_paths = detect_lm_eval_jsons(d) - return (lm_paths[0] if lm_paths else None), None +def has_invalid_effective_count(data: Dict[str, Any], task: str) -> bool: + """Return whether a task has an explicitly invalid effective count.""" + if 'n-samples' not in data: + return False + sample_counts = data['n-samples'] + if not isinstance(sample_counts, dict) or task not in sample_counts: + return True + task_samples = sample_counts[task] + if not isinstance(task_samples, dict) or 'effective' not in task_samples: + return True + effective = task_samples['effective'] + return ( + isinstance(effective, bool) + or not isinstance(effective, (int, float)) + or not math.isfinite(effective) + or effective <= 0 + ) def extract_lm_metrics(json_path: Path) -> List[Dict[str, Any]]: @@ -123,6 +149,8 @@ def extract_lm_metrics(json_path: Path) -> List[Dict[str, Any]]: - Values from results[task][metric,filter] """ data = load_json(json_path) or {} + if 'integration_error' in data: + return [] results = data.get('results', {}) configs = data.get('configs', {}) @@ -132,6 +160,8 @@ def extract_lm_metrics(json_path: Path) -> List[Dict[str, Any]]: extracted = [] for task in results.keys(): + if has_invalid_effective_count(data, task): + continue task_results = results[task] task_config = configs.get(task, {}) @@ -284,6 +314,9 @@ def build_row(meta: Dict[str, Any], m: Dict[str, Any]) -> Dict[str, Any]: 'source': m.get('source'), } + if 'eval_suite' in meta: + row['eval_suite'] = meta['eval_suite'] + # Add universal score field (primary metric for unified comparison) if m.get('strict') is not None: row['score'] = m.get('strict') @@ -404,7 +437,7 @@ def main(): f"{pct(r['score'])}{se(r['score_se'])}", f"{pct(r['em_strict'])}{se(r['em_strict_se'])}", f"{pct(r['em_flexible'])}{se(r['em_flexible_se'])}", - r['n_eff'] or '', + r['n_eff'] if r['n_eff'] is not None else '', r['model'], ] for r in single_node_rows @@ -442,7 +475,7 @@ def main(): f"{pct(r['score'])}{se(r['score_se'])}", f"{pct(r['em_strict'])}{se(r['em_strict_se'])}", f"{pct(r['em_flexible'])}{se(r['em_flexible_se'])}", - r['n_eff'] or '', + r['n_eff'] if r['n_eff'] is not None else '', r['model'], ] for r in multinode_rows diff --git a/utils/evals/EVALS.md b/utils/evals/EVALS.md index 0a7bf213df..9c1375e7c0 100644 --- a/utils/evals/EVALS.md +++ b/utils/evals/EVALS.md @@ -11,11 +11,10 @@ from throughput. Selection lives in `mark_eval_entries()` in runner, framework, precision, TP, and decoding configuration. - **Multi-node:** 8k1k only, with one job per parallelism topology at its highest eligible concurrency. Rows differing only by concurrency share a topology. -- **Agentic (SWE-bench), single-node:** highest-conc entry per (model, - runner, framework, precision) group. -- **Agentic (SWE-bench), multi-node:** same policy as multi-node fixed-seq-len - above (highest eligible conc per parallelism topology), since SWE-bench - doesn't support batched concurrencies the way lm-eval does. +- **Agentic (GSM8K), single-node:** highest-conc entry per (model, runner, + framework, precision) group. +- **Agentic (GSM8K), multi-node:** highest eligible concurrency per + parallelism topology. Generator eval modes: @@ -44,10 +43,75 @@ malformed metadata, duplicates, and raw/aggregate mismatches are not. See [workflow reuse](../../.github/workflows/README.md#reusing-an-approved-pr-full-sweep). ## How? -`run_eval` in `benchmarks/benchmark_lib.sh` runs EleutherAI/lm-evaluation-harness against the server's OpenAI-compatible endpoint. Concurrency is set via `EVAL_CONCURRENT_REQUESTS` env var (not a CLI flag). Results are collected by `utils/collect_eval_results.py` and published as a summary table. +`run_eval` in `benchmarks/benchmark_lib.sh` dispatches to the selected eval +runner. Existing jobs continue to use lm-eval with GSM8K by default. The default eval framework is [lm-evaluation-harness](https://github.com/EleutherAI/lm-evaluation-harness) (`lm-eval`). Agentic eval-only matrix jobs inherit this default and therefore run the same GSM8K task as 8k1k. Explicit agentic runs can still select SWE-bench. +The Phase 1 Kimi smoke is opt-in. It supports single-node jobs and Kimi K3 +aggregate H200, B200, and GB200 srt-slurm jobs. Select +`eval-framework: kimi-vendor` and `eval-suite: kimi_tool_call_schema` on +`e2e-tests.yml`, or invoke it from the repository root after a server is ready: + +```bash +source benchmarks/benchmark_lib.sh +export EVAL_FRAMEWORK=kimi-vendor +export EVAL_SUITE=kimi_tool_call_schema +export EVAL_RESULT_DIR="$(mktemp -d /tmp/eval_out-XXXXXX)" +run_eval --port "$PORT" +append_lm_eval_summary +python3 utils/evals/validate_scores.py +``` + +The framework selects a provider-specific subprocess adapter, while the suite +selects a case set understood by that adapter. Each adapter owns its endpoint +format, dependencies, native report, metrics, and pass policy. Future MiniMax +or BFCL support should add explicit `run_eval` cases rather than a shared +request or report abstraction. +Agentic eval jobs forward the matrix `spec-decoding` value, so MTP entries +launch their existing `*_mtp.sh` server instead of silently falling back to STP. + +### Stock Kimi tool-call schema smoke + +The smoke runs the unmodified +[MoonshotAI/Kimi-Vendor-Verifier](https://github.com/MoonshotAI/Kimi-Vendor-Verifier) +at commit `b9ed3a6665bdff2c943246f7d2903cd003d6ddd6`. Each run downloads the +fresh pinned GitHub source archive and safely extracts only the upstream pytest +configuration, tool-call schema tests, and bundled Walle cases. InferenceX does +not install the verifier package or reimplement its request, streaming, or +validation logic. + +Python 3.12 or newer is required. The runner installs the minimal pinned runtime +(`httpx[http2]`, `openai`, `jsonschema`, `pytest`, and +`pytest-rerunfailures`) into a temporary isolated package directory, then runs +upstream +`tests/tool_call_json_schema/test_tool_call_json_schema.py` with: + +- the local OpenAI-compatible endpoint, `EMPTY` API key, and served model name; +- `--think-mode none --selection object --max-cases 1 --max-tokens 2048`; +- the upstream-recommended `--reruns 3 --reruns-delay 2`; +- the bundled Walle case directory and `--tool-json-report`. + +The selection is `TestAdditionalProperties:1`, parametrized upstream in +non-streaming and streaming modes. Pytest makes one initial attempt and up to +three reruns of each failing mode, with a two-second delay before each rerun. +These retries reduce transient transport and model-sampling flakes; they do not +make the smoke deterministic. The unchanged native report remains one final +outcome per mode because the upstream report deduplicates rerun records by case +and mode. It is uploaded as `kimi_vendor_report.json`, and +`utils/evals/kimi_vendor_eval.py` projects those two outcomes into the existing +eval result shape. Both must pass, so the `kimi_tool_call_schema` threshold is +`1.0`. Setup, timeout, and collection failures emit a zero-score result with +error metadata. The adapter's 900-second global timeout bounds the entire +upstream pytest process, including all attempts and rerun delays. + +This smoke validates one object-schema tool call. It does not cover tool choice, +parallel calls, multi-turn execution, or general agent quality. Multi-value +batched concurrency is unsupported. Multi-node aggregate jobs run the same +two-case smoke against their OpenAI-compatible frontend. Eval-only launchers +restore real block verification before submitting recipes that otherwise use +synthetic acceptance for throughput. + ### Benchmark script flow All benchmark scripts in `benchmarks/` follow one of two flows: @@ -77,14 +141,17 @@ Key eval functions in `benchmarks/benchmark_lib.sh`: |----------|-------------| | `run_eval` | Unified entrypoint - dispatches to framework-specific runner | | `run_lm_eval` | Runs lm-eval harness against the OpenAI-compatible endpoint | +| `run_kimi_vendor_eval` | Selects and runs a pinned Kimi Vendor Verifier suite | | `append_lm_eval_summary` | Writes `meta_env.json` and moves eval artifacts to workspace | | `_install_lm_eval_deps` | Installs lm-eval dependencies | +| `_prepare_kimi_vendor_runtime` | Installs the pinned verifier dependencies in an isolated temp path | +| `_prepare_kimi_vendor_verifier` | Downloads and safely extracts a fresh subset of the pinned source archive | | `_patch_lm_eval` | Patches lm-eval for reasoning tokens and TRT compatibility | | `compute_eval_context_length` | Computes eval context length (requested benchmark context, capped at model native max) | | `get_native_max_context_length` | Extracts model's native max context length from HF config | ### Single-node -In eval-only mode (`EVAL_ONLY=true`), the benchmark script computes `EVAL_MAX_MODEL_LEN` via `compute_eval_context_length`, starts the server with that context length, skips throughput, and runs lm-eval directly. Each framework wires that context differently (`--context-length` for SGLang, `--max_seq_len` for TRT-LLM). +For default lm-eval jobs in eval-only mode (`EVAL_ONLY=true`), the benchmark script computes `EVAL_MAX_MODEL_LEN` via `compute_eval_context_length`, starts the server with that context length, skips throughput, and runs lm-eval. Each framework wires that context differently (`--context-length` for SGLang, `--max_seq_len` for TRT-LLM). ### Multi-node Multi-node evals support two hardware paths: @@ -92,13 +159,13 @@ Multi-node evals support two hardware paths: **MI355X (AMD)** — `benchmarks/multi_node/amd_utils/server_sglang.sh` - Skips throughput when `EVAL_ONLY=true` - Fixed-seq-len: runs lm-eval via `run_eval --framework lm-eval` against the router on port 30000 -- Agentic-coding (disaggregated, `IS_AGENTIC=1`): runs SWE-bench via `run_eval --port 30000` (no - `--framework` override, same auto-selection as single-node agentic eval-only). Since there's no - single "TP" for a disaggregated topology, and the workflow spells a couple of metadata fields - differently (`PREFILL_DP_ATTN`/`DECODE_DP_ATTN`) than `append_lm_eval_summary` expects - (`PREFILL_DP_ATTENTION`/`DECODE_DP_ATTENTION`), the agentic branch bridges those before calling - `run_eval`; `append_lm_eval_summary` itself runs automatically inside `run_eval()` (same - `EVAL_ONLY=true && IS_AGENTIC` auto-staging as single-node), not as a separate call. +- Agentic-coding (disaggregated, `IS_AGENTIC=1`): follows the same GSM8K/lm-eval path via + `run_eval --framework lm-eval`. Since there's no single "TP" for a disaggregated topology, + and the workflow spells a couple of metadata fields differently + (`PREFILL_DP_ATTN`/`DECODE_DP_ATTN`) than `append_lm_eval_summary` expects + (`PREFILL_DP_ATTENTION`/`DECODE_DP_ATTENTION`), the agentic branch bridges those before + calling `run_eval`; `append_lm_eval_summary` itself runs automatically inside `run_eval()` + (same `EVAL_ONLY=true && IS_AGENTIC` auto-staging as single-node), not as a separate call. - Concurrency uses workflow-provided `EVAL_CONC` when set, otherwise falls back to max of `BENCH_MAX_CONCURRENCY` (x-separated values) - Eval artifacts copied to `/run_logs/slurm_job-*/eval_results/` - `runners/launch_mi355x-amds.sh` skips benchmark result collection when `EVAL_ONLY=true` and uses `find` to locate eval results @@ -106,10 +173,10 @@ Multi-node evals support two hardware paths: **NVIDIA Slurm multi-node (GB200, GB300, B200, B300, H100, H200)** runs through [srt-slurm](https://github.com/NVIDIA/srt-slurm) on the `sa-submission-q2-2026` branch. - `do_sweep.py` skips the benchmark stage when `EVAL_ONLY=true`, runs `_run_post_eval()` directly - In eval-only mode, uses the full `wait_for_model()` health check (same as benchmark stage) since the benchmark health check was skipped -- `lm-eval` runner (`benchmarks/lm_eval.py`) is invoked by `do_sweep.py` as a post/eval-only step and sources InferenceX's `benchmark_lib.sh` from the mounted workspace (`/infmax-workspace`) +- The registered srt-slurm `lm-eval` post-runner sources InferenceX's `benchmark_lib.sh` from the mounted workspace (`/infmax-workspace`). Kimi-selected launches patch that hook to use generic `run_eval` dispatch while preserving lm-eval as the default. - Eval artifacts written to `/logs/eval_results/` inside the container, collected by launch scripts - NVIDIA Slurm launch scripts always collect server logs for debugging but skip benchmark result collection when `EVAL_ONLY=true` -- Env vars threaded: `RUN_EVAL`, `EVAL_ONLY`, `IS_MULTINODE`, `FRAMEWORK`, `PRECISION`, `MODEL_PREFIX`, `RUNNER_TYPE`, `RESULT_FILENAME`, `SPEC_DECODING`, `ISL`, `OSL`, `PREFILL_TP/EP/NUM_WORKERS/DP_ATTN`, `DECODE_TP/EP/NUM_WORKERS/DP_ATTN`, `MODEL_NAME`, `EVAL_CONC` +- Env vars threaded: `RUN_EVAL`, `EVAL_ONLY`, `EVAL_FRAMEWORK`, `EVAL_SUITE`, `IS_MULTINODE`, `FRAMEWORK`, `PRECISION`, `MODEL_PREFIX`, `RUNNER_TYPE`, `RESULT_FILENAME`, `SPEC_DECODING`, `ISL`, `OSL`, `PREFILL_TP/EP/NUM_WORKERS/DP_ATTN`, `DECODE_TP/EP/NUM_WORKERS/DP_ATTN`, `MODEL_NAME`, `EVAL_CONC` For multi-node `all-evals`, `EVAL_CONC` is a space-separated list. When it contains multiple values, `run_eval` runs those concurrency points sequentially against the same live engine, stages each result with a `_concN` filename suffix, and records expected/completed/failed points in `meta_env.json`. @@ -159,6 +226,7 @@ cat ./evals/agg_eval_all.json | jq '[.[] | select(.hw == "B200")]' | `em_flexible` | Flexible extraction (looser number matching) | | `n_eff` | Number of samples evaluated | | `task` | Eval task name (e.g., `gsm8k`) | +| `eval_suite` | Explicit suite identity used for collection and artifact reuse | ### Environment variables @@ -166,7 +234,8 @@ cat ./evals/agg_eval_all.json | jq '[.[] | select(.hw == "B200")]' |----------|---------|-------------| | `RUN_EVAL` | `false` | Enable eval after throughput benchmark | | `EVAL_ONLY` | `false` | Skip throughput, only run evals (set by workflow) | -| `EVAL_FRAMEWORK` | `lm-eval` | Eval framework to use | +| `EVAL_FRAMEWORK` | `lm-eval` | Eval runner (`lm-eval`, `swebench`, or `kimi-vendor`) | +| `EVAL_SUITE` | basename of `EVAL_TASKS_DIR`, else `gsm8k` | Provider suite selector and artifact identity. External override is currently supported only by `kimi-vendor`; other runners derive it from their task | | `EVAL_TASKS_DIR` | `utils/evals/gsm8k.yaml` | Path to lm-eval task YAML | | `EVAL_RESULT_DIR` | `/tmp/eval_out-*` | Output directory for eval results | | `EVAL_MAX_MODEL_LEN` | `16384` | Max context for eval (set by `compute_eval_context_length`) | @@ -182,6 +251,15 @@ cat ./evals/agg_eval_all.json | jq '[.[] | select(.hw == "B200")]' 2. Set `EVAL_TASKS_DIR=utils/evals/.yaml` when running benchmarks. 3. Update `utils/collect_eval_results.py` if new metrics need extraction. +### Adding a provider verifier + +1. Add a provider-specific adapter under `utils/evals/`. +2. Add an explicit framework case in `run_eval`; keep suite-specific policy in + that adapter's shell runner. +3. Install dependencies in a provider-specific isolated runtime. +4. Emit `result_format: inferencex-eval-v1`, preserve the native report as + `*_vendor_report.json`, set `EVAL_SUITE`, and add a threshold. + ### Runtime patches (`utils/evals/patches/`) The benchmark helpers invoke these standalone scripts against pinned dependencies. diff --git a/utils/evals/kimi_vendor_eval.py b/utils/evals/kimi_vendor_eval.py new file mode 100755 index 0000000000..8851f2be66 --- /dev/null +++ b/utils/evals/kimi_vendor_eval.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +"""Run the stock Kimi Vendor Verifier and project its native report.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from collections.abc import Mapping, Sequence +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +TASK_NAME = "kimi_tool_call_schema" +NATIVE_REPORT_FILENAME = "kimi_vendor_report.json" +COMPATIBILITY_GLOB = "results_kimi_vendor_*.json" +EXPECTED_MODES = {"non-stream", "stream"} +DEFAULT_TIMEOUT_SECONDS = 900 +RESULT_FORMAT = "inferencex-eval-v1" +ADAPTER_NAME = "kimi-vendor-verifier" + + +def prepare_compatibility_path(output_dir: Path) -> Path: + """Remove stale projections and return a timestamped collector artifact path.""" + for stale_path in output_dir.glob(COMPATIBILITY_GLOB): + stale_path.unlink() + timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S.%f") + return output_dir / f"results_kimi_vendor_{timestamp}.json" + + +def build_pytest_command( + *, base_url: str, api_key: str, model: str, report_path: Path +) -> list[str]: + """Build the fixed Phase 1 invocation of the upstream verifier.""" + return [ + sys.executable, + "-m", + "pytest", + "tests/tool_call_json_schema/test_tool_call_json_schema.py", + "--reruns", + "3", + "--reruns-delay", + "2", + "--base-url", + base_url, + "--api-key", + api_key, + "--smoke-model", + model, + "--think-mode", + "none", + "--selection", + "object", + "--max-cases", + "1", + "--case-dir", + "testdata/walle_validator_cases/validator_cases", + "--max-tokens", + "2048", + "--tool-json-report", + str(report_path), + ] + + +def _mapping(value: Any, name: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ValueError(f"{name} must be an object") + return value + + +def _project_report(model: str, report: Any) -> tuple[dict[str, Any], bool]: + root = _mapping(report, "report") + summary = _mapping(root.get("summary"), "report.summary") + results = root.get("results") + if not isinstance(results, list): + raise ValueError("report.results must be an array") + + total = summary.get("total") + by_status = _mapping(summary.get("by_status"), "report.summary.by_status") + if not isinstance(total, int) or isinstance(total, bool): + raise ValueError("report summary contains invalid counts") + for status, count in by_status.items(): + if ( + status not in {"passed", "failed"} + or not isinstance(count, int) + or isinstance(count, bool) + or count < 0 + ): + raise ValueError("report summary contains invalid counts") + if sum(by_status.values()) != total: + raise ValueError("report summary does not match total") + passed = by_status.get("passed", 0) + + modes: list[str] = [] + result_passes = 0 + for index, result in enumerate(results): + record = _mapping(result, f"report.results[{index}]") + mode = record.get("mode") + status = record.get("status") + if ( + not isinstance(mode, str) + or not isinstance(status, str) + or status not in {"passed", "failed"} + ): + raise ValueError(f"report.results[{index}] has invalid mode or status") + modes.append(mode) + result_passes += status == "passed" + + if total != len(results) or passed != result_passes: + raise ValueError("report summary does not match result records") + + if ( + total != 2 + or len(results) != 2 + or set(modes) != EXPECTED_MODES + or len(modes) != len(set(modes)) + ): + raise ValueError("report does not contain the expected stream modes") + score = passed / 2.0 + return _compatibility_result(model, score, n_samples=2), passed == 2 + + +def _compatibility_result( + model: str, + score: float, + *, + n_samples: int, + integration_error: BaseException | None = None, +) -> dict[str, Any]: + result: dict[str, Any] = { + "result_format": RESULT_FORMAT, + "eval_adapter": ADAPTER_NAME, + "model_name": model, + "results": { + TASK_NAME: { + "exact_match,strict-match": score, + "exact_match_stderr,strict-match": 0.0, + } + }, + "configs": { + TASK_NAME: { + "metric_list": [{"metric": "exact_match"}], + "filter_list": [{"name": "strict-match"}], + } + }, + "n-samples": { + TASK_NAME: { + "original": len(EXPECTED_MODES), + "effective": n_samples, + } + }, + } + if integration_error is not None: + result["integration_error"] = { + "type": type(integration_error).__name__, + "message": str(integration_error), + } + return result + + +def _write_compatibility(path: Path, result: Mapping[str, Any]) -> None: + path.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + + +def run_evaluation( + *, + verifier_dir: Path, + base_url: str, + api_key: str, + model: str, + output_dir: Path, + timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS, +) -> bool: + """Run upstream pytest and always attempt to publish a compatibility result.""" + output_dir.mkdir(parents=True, exist_ok=True) + native_report = output_dir / NATIVE_REPORT_FILENAME + compatibility_path = prepare_compatibility_path(output_dir) + subprocess_rc: int | None = None + integration_error: BaseException | None = None + compatibility = _compatibility_result(model, 0.0, n_samples=0) + complete_pass = False + try: + native_report.unlink(missing_ok=True) + completed = subprocess.run( + build_pytest_command( + base_url=base_url, + api_key=api_key, + model=model, + report_path=native_report.resolve(), + ), + cwd=verifier_dir, + check=False, + timeout=timeout_seconds, + ) + subprocess_rc = completed.returncode + report = json.loads(native_report.read_text(encoding="utf-8")) + compatibility, complete_pass = _project_report(model, report) + if subprocess_rc != 0 and complete_pass: + integration_error = RuntimeError( + f"upstream verifier exited with code {subprocess_rc}" + ) + compatibility = _compatibility_result( + model, 0.0, n_samples=2, integration_error=integration_error + ) + complete_pass = False + except (OSError, ValueError, subprocess.TimeoutExpired) as exc: + integration_error = exc + compatibility = _compatibility_result( + model, 0.0, n_samples=0, integration_error=exc + ) + finally: + try: + _write_compatibility(compatibility_path, compatibility) + except OSError as exc: + if integration_error is not None: + exc.add_note(f"Earlier integration error: {integration_error}") + raise + + return subprocess_rc == 0 and complete_pass and integration_error is None + + +def _positive_int(value: str) -> int: + parsed = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("must be a positive integer") + return parsed + + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run the pinned stock Kimi Vendor Verifier tool-schema smoke test." + ) + parser.add_argument("--verifier-dir", type=Path) + parser.add_argument("--base-url") + parser.add_argument("--api-key", default="EMPTY") + parser.add_argument("--model", required=True) + parser.add_argument("--output-dir", required=True, type=Path) + parser.add_argument( + "--timeout-seconds", type=_positive_int, default=DEFAULT_TIMEOUT_SECONDS + ) + parser.add_argument("--integration-error") + args = parser.parse_args(argv) + if args.integration_error is None: + missing = [ + option + for option, value in ( + ("--verifier-dir", args.verifier_dir), + ("--base-url", args.base_url), + ) + if value is None + ] + if missing: + parser.error( + f"{', '.join(missing)} required unless --integration-error is provided" + ) + return args + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + if args.integration_error is not None: + args.output_dir.mkdir(parents=True, exist_ok=True) + (args.output_dir / NATIVE_REPORT_FILENAME).unlink(missing_ok=True) + _write_compatibility( + prepare_compatibility_path(args.output_dir), + _compatibility_result( + args.model, + 0.0, + n_samples=0, + integration_error=RuntimeError(args.integration_error), + ), + ) + return 0 + passed = run_evaluation( + verifier_dir=args.verifier_dir, + base_url=args.base_url, + api_key=args.api_key, + model=args.model, + output_dir=args.output_dir, + timeout_seconds=args.timeout_seconds, + ) + return 0 if passed else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/utils/evals/test_batched_eval.py b/utils/evals/test_batched_eval.py index a5d6df0085..56c219e558 100644 --- a/utils/evals/test_batched_eval.py +++ b/utils/evals/test_batched_eval.py @@ -227,6 +227,106 @@ def test_validate_scores_checks_threshold_for_every_concurrency( assert "FAIL: [conc=4] gsm8k exact_match,strict-match" in captured.err +def test_validate_scores_reports_integration_failure_without_thresholding( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + result_path = tmp_path / "results_test.json" + result_path.write_text(json.dumps({ + "integration_error": { + "type": "RuntimeError", + "message": "vendor verifier checkout failed", + }, + "results": { + "gsm8k": { + "exact_match,strict-match": 0.0, + }, + }, + "n-samples": {"gsm8k": {"effective": 0}}, + })) + monkeypatch.setattr(sys, "argv", [ + "validate_scores.py", + "--meta-env", + str(tmp_path / "meta_env.json"), + "--results-glob", + str(result_path), + ]) + + assert validate_scores_main() == 1 + captured = capsys.readouterr() + assert "integration failure: RuntimeError: vendor verifier checkout failed" in captured.err + assert "gsm8k exact_match,strict-match" not in captured.err + + +def test_validate_scores_rejects_invalid_effective_count_without_thresholding( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + result_path = tmp_path / "results_test.json" + result_path.write_text(json.dumps({ + "results": { + "gsm8k": { + "exact_match,strict-match": 0.0, + }, + "other": { + "exact_match,strict-match": 0.0, + }, + "nonfinite": { + "exact_match,strict-match": 1.0, + }, + }, + "n-samples": { + "gsm8k": {"effective": "unknown"}, + "other": {"effective": 0}, + "nonfinite": {"effective": float("inf")}, + }, + })) + monkeypatch.setattr(sys, "argv", [ + "validate_scores.py", + "--meta-env", + str(tmp_path / "meta_env.json"), + "--results-glob", + str(result_path), + ]) + + assert validate_scores_main() == 1 + captured = capsys.readouterr() + assert "gsm8k invalid effective sample count: 'unknown'" in captured.err + assert "gsm8k exact_match,strict-match" not in captured.err + assert "other invalid effective sample count: 0" in captured.err + assert "other exact_match,strict-match" not in captured.err + + assert "nonfinite invalid effective sample count: inf" in captured.err + assert "nonfinite exact_match,strict-match" not in captured.err + +def test_validate_scores_accepts_legacy_result_without_effective_count( + tmp_path: Path, + monkeypatch, + capsys, +) -> None: + result_path = tmp_path / "results_test.json" + result_path.write_text(json.dumps({ + "results": { + "gsm8k": { + "exact_match,strict-match": 1.0, + }, + }, + })) + monkeypatch.setattr(sys, "argv", [ + "validate_scores.py", + "--meta-env", + str(tmp_path / "meta_env.json"), + "--results-glob", + str(result_path), + ]) + + assert validate_scores_main() == 0 + captured = capsys.readouterr() + assert "PASS: gsm8k exact_match,strict-match" in captured.out + + def test_amd_multinode_container_forwards_eval_concurrency_list() -> None: job_slurm = ( Path(__file__).resolve().parents[2] diff --git a/utils/evals/test_kimi_vendor_eval.py b/utils/evals/test_kimi_vendor_eval.py new file mode 100644 index 0000000000..a71d3d962f --- /dev/null +++ b/utils/evals/test_kimi_vendor_eval.py @@ -0,0 +1,242 @@ +import json +import re +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import kimi_vendor_eval as kve + + +def _report(stream_status: str = "passed") -> dict[str, Any]: + statuses = ["passed", stream_status] + by_status = {status: statuses.count(status) for status in set(statuses)} + return { + "summary": {"total": 2, "by_status": by_status}, + "results": [ + {"mode": "non-stream", "status": "passed"}, + {"mode": "stream", "status": stream_status}, + ], + } + + +def _report_with_inconsistent_counts() -> dict[str, Any]: + report = _report("failed") + report["summary"]["by_status"]["failed"] = 2 + return report + + +def _result(output_dir: Path) -> dict[str, Any]: + paths = list(output_dir.glob(kve.COMPATIBILITY_GLOB)) + assert len(paths) == 1 + assert re.fullmatch( + r"results_kimi_vendor_\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}\.\d{6}\.json", + paths[0].name, + ) + return json.loads(paths[0].read_text()) + + +def _score(output_dir: Path) -> float: + return _result(output_dir)["results"][kve.TASK_NAME]["exact_match,strict-match"] + + +def _n_eff(output_dir: Path) -> int: + return _result(output_dir)["n-samples"][kve.TASK_NAME]["effective"] + + +def test_builds_fixed_upstream_pytest_command(tmp_path: Path) -> None: + report = tmp_path / kve.NATIVE_REPORT_FILENAME + + assert kve.build_pytest_command( + base_url="http://127.0.0.1:8000/v1", + api_key="EMPTY", + model="test-model", + report_path=report, + ) == [ + sys.executable, + "-m", + "pytest", + "tests/tool_call_json_schema/test_tool_call_json_schema.py", + "--reruns", + "3", + "--reruns-delay", + "2", + "--base-url", + "http://127.0.0.1:8000/v1", + "--api-key", + "EMPTY", + "--smoke-model", + "test-model", + "--think-mode", + "none", + "--selection", + "object", + "--max-cases", + "1", + "--case-dir", + "testdata/walle_validator_cases/validator_cases", + "--max-tokens", + "2048", + "--tool-json-report", + str(report), + ] + + +@pytest.mark.parametrize( + ("stream_status", "return_code", "expected_pass", "expected_score"), + ( + ("passed", 0, True, 1.0), + ("passed", 1, False, 0.0), + ("failed", 1, False, 0.5), + ), +) +def test_projects_upstream_outcomes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + stream_status: str, + return_code: int, + expected_pass: bool, + expected_score: float, +) -> None: + output_dir = tmp_path / "output" + native_bytes = json.dumps(_report(stream_status)).encode() + invocation: dict[str, Any] = {} + + def fake_run( + command: list[str], *, cwd: Path, check: bool, timeout: int + ) -> SimpleNamespace: + invocation.update(command=command, cwd=cwd, check=check, timeout=timeout) + Path(command[command.index("--tool-json-report") + 1]).write_bytes(native_bytes) + return SimpleNamespace(returncode=return_code) + + monkeypatch.setattr(kve.subprocess, "run", fake_run) + + assert ( + kve.run_evaluation( + verifier_dir=tmp_path, + base_url="http://localhost/v1", + api_key="EMPTY", + model="model-a", + output_dir=output_dir, + ) + is expected_pass + ) + assert invocation["cwd"] == tmp_path + assert invocation["check"] is False + assert invocation["timeout"] == kve.DEFAULT_TIMEOUT_SECONDS + assert _score(output_dir) == expected_score + assert _n_eff(output_dir) == 2 + projected = _result(output_dir) + assert projected["result_format"] == kve.RESULT_FORMAT + assert projected["eval_adapter"] == kve.ADAPTER_NAME + assert "lm_eval_version" not in projected + assert (output_dir / kve.NATIVE_REPORT_FILENAME).read_bytes() == native_bytes + + +@pytest.mark.parametrize( + ("failure", "error_type"), + ( + (None, "FileNotFoundError"), + ("{bad-json", "JSONDecodeError"), + (OSError("boom"), "OSError"), + (json.dumps(_report("skipped")), "ValueError"), + (json.dumps(_report_with_inconsistent_counts()), "ValueError"), + ( + subprocess.TimeoutExpired("pytest", kve.DEFAULT_TIMEOUT_SECONDS), + "TimeoutExpired", + ), + ), +) +def test_collection_failures_write_zero_score( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + failure: str | BaseException | None, + error_type: str, +) -> None: + def fake_run( + command: list[str], *, cwd: Path, check: bool, timeout: int + ) -> SimpleNamespace: + if isinstance(failure, BaseException): + raise failure + if failure is not None: + Path(command[command.index("--tool-json-report") + 1]).write_text(failure) + return SimpleNamespace(returncode=1) + + monkeypatch.setattr(kve.subprocess, "run", fake_run) + output_dir = tmp_path / "output" + + assert not kve.run_evaluation( + verifier_dir=tmp_path, + base_url="http://localhost/v1", + api_key="EMPTY", + model="model-a", + output_dir=output_dir, + ) + projected = _result(output_dir) + assert _score(output_dir) == 0.0 + assert projected["integration_error"]["type"] == error_type + assert _n_eff(output_dir) == 0 + + +def test_failure_cannot_reuse_stale_outputs( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + output_dir = tmp_path / "output" + output_dir.mkdir() + native_report = output_dir / kve.NATIVE_REPORT_FILENAME + native_report.write_text(json.dumps(_report())) + (output_dir / "results_kimi_vendor_2000-01-01T00-00-00.000000.json").write_text( + "{}" + ) + foreign_result = output_dir / "results_other_eval.json" + foreign_result.write_text("{}") + + def fail_collection(*args: Any, **kwargs: Any) -> SimpleNamespace: + assert not native_report.exists() + return SimpleNamespace(returncode=2) + + monkeypatch.setattr(kve.subprocess, "run", fail_collection) + + assert not kve.run_evaluation( + verifier_dir=tmp_path, + base_url="http://localhost/v1", + api_key="EMPTY", + model="model-a", + output_dir=output_dir, + ) + assert _score(output_dir) == 0.0 + assert not native_report.exists() + assert foreign_result.exists() + + +def test_cli_setup_failure_writes_zero_score_artifact(tmp_path: Path) -> None: + output_dir = tmp_path / "output" + output_dir.mkdir() + (output_dir / kve.NATIVE_REPORT_FILENAME).write_text(json.dumps(_report())) + (output_dir / "results_kimi_vendor_2000-01-01T00-00-00.000000.json").write_text( + "{}" + ) + + assert ( + kve.main( + [ + "--model", + "model-a", + "--output-dir", + str(output_dir), + "--integration-error", + "checkout failed", + ] + ) + == 0 + ) + projected = _result(output_dir) + assert not (output_dir / kve.NATIVE_REPORT_FILENAME).exists() + assert _score(output_dir) == 0.0 + assert projected["integration_error"]["message"] == "checkout failed" + assert _n_eff(output_dir) == 0 diff --git a/utils/evals/test_run_eval_dispatch.py b/utils/evals/test_run_eval_dispatch.py index 45be3a6e4d..cbcf25b7e3 100644 --- a/utils/evals/test_run_eval_dispatch.py +++ b/utils/evals/test_run_eval_dispatch.py @@ -1,25 +1,44 @@ from __future__ import annotations +import io +import json import os +import re import stat import subprocess +import tarfile +import threading +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path -BENCHMARK_LIB = Path(__file__).resolve().parents[2] / "benchmarks" / "benchmark_lib.sh" +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +BENCHMARK_LIB = REPO_ROOT / "benchmarks" / "benchmark_lib.sh" +MULTINODE_WORKFLOW = REPO_ROOT / ".github/workflows/benchmark-multinode-tmpl.yml" +E2E_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "e2e-tests.yml" _SCRIPT = r''' source "$BENCHMARK_LIB" run_lm_eval() { echo "DISPATCH=lm-eval"; } run_swebench_eval() { echo "DISPATCH=swebench"; } +run_kimi_vendor_eval() { echo "DISPATCH=kimi-vendor"; } append_lm_eval_summary() { echo "STAGED=summary"; } export EVAL_MAX_MODEL_LEN=16384 -unset EVAL_CONCURRENT_REQUESTS +export EVAL_CONCURRENT_REQUESTS="" run_eval ${CLI_FW:+--framework "$CLI_FW"} --port 8888 ''' -def _dispatch(*, is_agentic: str = "0", eval_only: str = "false", cli_fw=None, env_fw=None) -> str: +def _dispatch( + *, + is_agentic: str = "0", + eval_only: str = "false", + cli_fw=None, + env_fw=None, +) -> str: env = { **os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB), @@ -71,10 +90,73 @@ def test_env_can_force_swebench_on_fixed_seqlen(): assert "DISPATCH=swebench" in _dispatch(is_agentic="0", env_fw="swebench") + + +def test_env_can_force_kimi_vendor_on_agentic_eval() -> None: + assert "DISPATCH=kimi-vendor" in _dispatch( + is_agentic="1", + eval_only="true", + env_fw="kimi-vendor", + ) + + +def test_kimi_vendor_skips_unused_model_context_loading() -> None: + script = r''' +source "$BENCHMARK_LIB" +unset EVAL_MAX_MODEL_LEN +compute_eval_context_length() { echo "UNEXPECTED_CONTEXT_LOAD"; return 99; } +run_kimi_vendor_eval() { echo "DISPATCH=kimi-vendor"; } +export EVAL_FRAMEWORK=kimi-vendor +export EVAL_CONCURRENT_REQUESTS="" +export EVAL_ONLY=false +export IS_AGENTIC=0 +run_eval --port 8888 +''' + result = subprocess.run( + ["bash", "-c", script], + env={**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB)}, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert "DISPATCH=kimi-vendor" in result.stdout + assert "UNEXPECTED_CONTEXT_LOAD" not in result.stdout + + +def test_kimi_failure_preserves_rc_without_eval_only() -> None: + script = r''' +set -u +source "$BENCHMARK_LIB" +run_kimi_vendor_eval() { return 7; } +export EVAL_FRAMEWORK=kimi-vendor +export EVAL_CONCURRENT_REQUESTS="" +export EVAL_MAX_MODEL_LEN=16384 +export IS_AGENTIC=0 +unset EVAL_ONLY +run_eval --port 8888 +''' + result = subprocess.run( + ["bash", "-c", script], + env={**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB)}, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 7 + assert "unbound variable" not in result.stderr + + + + def test_recipe_lm_eval_arg_still_lm_eval_on_fixed_seqlen(): assert "DISPATCH=lm-eval" in _dispatch(is_agentic="0", cli_fw="lm-eval") + + def _run_invalid_call(call: str) -> subprocess.CompletedProcess: env = { **os.environ, @@ -95,6 +177,482 @@ def test_run_eval_rejects_missing_framework_value(): assert "--framework requires a value" in result.stderr +def test_run_eval_rejects_unsafe_suite_name() -> None: + result = _run_invalid_call( + "EVAL_SUITE='kimi\"suite' run_eval --framework kimi-vendor" + ) + + assert result.returncode == 2 + assert "EVAL_SUITE may contain only" in result.stderr + + +def test_run_eval_rejects_suite_override_for_lm_eval() -> None: + result = _run_invalid_call( + "EVAL_SUITE=gpqa_diamond run_eval --framework lm-eval" + ) + + assert result.returncode == 2 + assert "only supported with EVAL_FRAMEWORK=kimi-vendor" in result.stderr + + +def test_run_eval_scopes_runner_selected_suite_to_one_call() -> None: + script = r''' +source "$BENCHMARK_LIB" +run_kimi_vendor_eval() { + export EVAL_SUITE=kimi_tool_call_schema + echo "DISPATCH=kimi-vendor SUITE=$EVAL_SUITE" +} +run_lm_eval() { + echo "DISPATCH=lm-eval SUITE=${EVAL_SUITE:-unset} COMPLETED=${EVAL_COMPLETED_SUITE:-unset}" +} +append_lm_eval_summary() { + echo "METADATA=${EVAL_COMPLETED_SUITE:-gsm8k}" +} +export EVAL_MAX_MODEL_LEN=16384 +export EVAL_CONCURRENT_REQUESTS="" +export EVAL_ONLY=false +export IS_AGENTIC=0 +unset EVAL_SUITE +export EVAL_FRAMEWORK=kimi-vendor +run_eval --port 8888 +printf 'KIMI_COMPLETED=%s\n' "${EVAL_COMPLETED_SUITE:-unset}" +append_lm_eval_summary +export EVAL_FRAMEWORK=lm-eval +run_eval --port 8888 +printf 'LM_COMPLETED=%s\n' "${EVAL_COMPLETED_SUITE:-unset}" +append_lm_eval_summary +printf 'FINAL_SUITE=%s\n' "${EVAL_SUITE-unset}" +''' + result = subprocess.run( + ["bash", "-c", script], + env={**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB)}, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert "DISPATCH=kimi-vendor SUITE=kimi_tool_call_schema" in result.stdout + assert "KIMI_COMPLETED=kimi_tool_call_schema" in result.stdout + assert "METADATA=kimi_tool_call_schema" in result.stdout + assert "DISPATCH=lm-eval SUITE=unset COMPLETED=unset" in result.stdout + assert "LM_COMPLETED=unset" in result.stdout + assert "METADATA=gsm8k" in result.stdout + assert "FINAL_SUITE=unset" in result.stdout + + +def test_kimi_default_suite_reaches_eval_only_metadata() -> None: + script = r''' +source "$BENCHMARK_LIB" +run_kimi_vendor_eval() { echo "DISPATCH=$EVAL_SUITE"; } +append_lm_eval_summary() { echo "METADATA=$EVAL_COMPLETED_SUITE"; } +export EVAL_FRAMEWORK=kimi-vendor +export EVAL_ONLY=true +export IS_AGENTIC=1 +export EVAL_CONCURRENT_REQUESTS="" +unset EVAL_SUITE +run_eval --port 8888 +''' + result = subprocess.run( + ["bash", "-c", script], + env={**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB)}, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert "DISPATCH=kimi_tool_call_schema" in result.stdout + assert "METADATA=kimi_tool_call_schema" in result.stdout + + +def test_kimi_vendor_rejects_batched_concurrency() -> None: + result = _run_invalid_call( + "EVAL_MAX_MODEL_LEN=16384 " + "EVAL_CONCURRENT_REQUESTS='1 4' " + "run_eval --framework kimi-vendor" + ) + assert result.returncode == 1 + assert "batched eval concurrency is only supported for lm-eval" in result.stderr + + +def test_kimi_vendor_rejects_unsupported_suite() -> None: + result = _run_invalid_call( + "EVAL_SUITE=gsm8k run_kimi_vendor_eval" + ) + assert result.returncode == 2 + assert "unsupported Kimi Vendor Verifier suite 'gsm8k'" in result.stderr + + + +def test_kimi_vendor_setup_failure_writes_compatibility_result( + tmp_path: Path, +) -> None: + results_dir = tmp_path / "results" + script = r''' +source "$BENCHMARK_LIB" +_require_kimi_vendor_python() { :; } +_prepare_kimi_vendor_runtime() { return 12; } +run_kimi_vendor_eval --results-dir "$RESULTS_DIR" +printf 'SETUP_RC=%s\n' "$?" +''' + env = { + **os.environ, + "BENCHMARK_LIB": str(BENCHMARK_LIB), + "RESULTS_DIR": str(results_dir), + "MODEL": "test-model", + "IS_MULTINODE": "false", + "KV_OFFLOADING": "none", + } + for key in ( + "EVAL_SUITE", + "EVAL_RESULT_DIR", + "MODEL_NAME", + ): + env.pop(key, None) + + result = subprocess.run( + ["bash", "-c", script], + env=env, + text=True, + capture_output=True, + check=True, + ) + message = "Kimi Vendor Verifier dependency installation failed with exit code 12" + score_files = list(results_dir.glob("results*.json")) + + assert "SETUP_RC=12" in result.stdout + assert message in result.stderr + assert "failed to write Kimi verifier failure artifact" not in result.stderr + assert len(score_files) == 1 + score_result = json.loads(score_files[0].read_text()) + assert ( + score_result["results"]["kimi_tool_call_schema"][ + "exact_match,strict-match" + ] + == 0.0 + ) + assert score_result["integration_error"]["message"] == message + assert not (results_dir / "kimi_vendor_report.json").exists() + + +_KIMI_VERIFIER_REQUIRED_FILES = { + "pyproject.toml", + "tests/conftest.py", + "tests/__init__.py", + "tests/tool_call_json_schema/conftest.py", + "tests/tool_call_json_schema/__init__.py", + "tests/tool_call_json_schema/test_tool_call_json_schema.py", + "tests/tool_call_json_schema/validator.py", + *{ + f"testdata/walle_validator_cases/validator_cases/{case}/valid.jsonl" + for case in ( + "TestAdditionalProperties", + "TestAnyOf", + "TestBasicTypes", + "TestDefs", + "TestDescription", + "TestEnforcerCases", + "TestID", + "TestKeywordsValidation", + "TestNestedDefsDepth", + "TestNumberFormat", + "TestRangeConstraints", + "TestRefInProperties", + "TestReferences", + "TestRequired", + "TestSingleTypeInArray", + "TestTypeLocation", + ) + }, +} + + +def _kimi_verifier_archive( + *, + missing: str | None = None, + unsafe_member: tarfile.TarInfo | None = None, +) -> bytes: + output = io.BytesIO() + with tarfile.open(fileobj=output, mode="w:gz") as archive: + for relative_path in sorted(_KIMI_VERIFIER_REQUIRED_FILES - {missing}): + payload = relative_path.encode() + member = tarfile.TarInfo(f"verifier-pinned/{relative_path}") + member.size = len(payload) + archive.addfile(member, io.BytesIO(payload)) + extra = b"must not be extracted" + member = tarfile.TarInfo("verifier-pinned/README.md") + member.size = len(extra) + archive.addfile(member, io.BytesIO(extra)) + if unsafe_member is not None: + archive.addfile( + unsafe_member, + io.BytesIO(b"unsafe") if unsafe_member.isfile() else None, + ) + return output.getvalue() + + +@contextmanager +def _serve_archive(payload: bytes, *, transient_failures: int = 0): + request_paths = [] + request_count = 0 + + class ArchiveHandler(BaseHTTPRequestHandler): + def do_GET(self): + nonlocal request_count + request_paths.append(self.path) + request_count += 1 + if request_count <= transient_failures: + self.send_response(503) + self.end_headers() + return + self.send_response(200) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), ArchiveHandler) + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + yield ( + f"http://127.0.0.1:{server.server_port}/owner/verifier.git", + request_paths, + ) + finally: + server.shutdown() + thread.join() + server.server_close() + + +def _prepare_local_kimi_verifier( + tmp_path: Path, + payload: bytes, + verifier_ref: str = "1" * 40, + transient_failures: int = 0, +) -> tuple[subprocess.CompletedProcess[str], Path, list[str]]: + checkout = tmp_path / "checkout" + script = r''' +source "$BENCHMARK_LIB" +git() { echo "git must not be invoked" >&2; return 127; } +mktemp() { mkdir "$CHECKOUT"; printf '%s\n' "$CHECKOUT"; } +_prepare_kimi_vendor_verifier "$REPO_URL" "$VERIFIER_REF" +''' + with _serve_archive( + payload, + transient_failures=transient_failures, + ) as (repo_url, request_paths): + result = subprocess.run( + ["bash", "-c", script], + env={ + **os.environ, + "BENCHMARK_LIB": str(BENCHMARK_LIB), + "CHECKOUT": str(checkout), + "REPO_URL": repo_url, + "VERIFIER_REF": verifier_ref, + }, + text=True, + capture_output=True, + ) + return result, checkout, request_paths + + +def test_kimi_vendor_verifier_fetches_expected_subset_without_git(tmp_path: Path) -> None: + result, checkout, request_paths = _prepare_local_kimi_verifier( + tmp_path, + _kimi_verifier_archive(), + ) + verifier_ref = "1" * 40 + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == str(checkout) + assert request_paths == [ + f"/owner/verifier/archive/{verifier_ref}.tar.gz", + ] + assert { + path.relative_to(checkout).as_posix() + for path in checkout.rglob("*") + if path.is_file() + } == _KIMI_VERIFIER_REQUIRED_FILES + assert "git must not be invoked" not in result.stderr + + +def test_kimi_vendor_verifier_retries_transient_archive_failure( + tmp_path: Path, +) -> None: + result, checkout, request_paths = _prepare_local_kimi_verifier( + tmp_path, + _kimi_verifier_archive(), + transient_failures=1, + ) + verifier_ref = "1" * 40 + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == str(checkout) + assert request_paths == [ + f"/owner/verifier/archive/{verifier_ref}.tar.gz", + f"/owner/verifier/archive/{verifier_ref}.tar.gz", + ] + assert "archive download attempt 1/3 failed" in result.stderr + + +def test_kimi_vendor_verifier_removes_partial_checkout_when_member_missing( + tmp_path: Path, +) -> None: + missing = "tests/tool_call_json_schema/validator.py" + result, checkout, _ = _prepare_local_kimi_verifier( + tmp_path, + _kimi_verifier_archive(missing=missing), + ) + + assert result.returncode == 1 + assert missing in result.stderr + assert not checkout.exists() + + +def test_kimi_vendor_verifier_rejects_unsafe_archive_members(tmp_path: Path) -> None: + unsafe = tarfile.TarInfo("verifier-pinned/../../escaped") + unsafe.size = len(b"unsafe") + result, checkout, _ = _prepare_local_kimi_verifier( + tmp_path, + _kimi_verifier_archive(unsafe_member=unsafe), + ) + + assert result.returncode == 1 + assert "unsafe archive member path" in result.stderr + assert not checkout.exists() + assert not (tmp_path / "escaped").exists() + + +def test_kimi_vendor_dependency_install_is_isolated(tmp_path: Path) -> None: + runtime_dir = tmp_path / "runtime" + script = r''' +source "$BENCHMARK_LIB" +python3() { printf 'PYTHON_ARG=<%s>\n' "$@"; } +_install_kimi_vendor_eval_deps "$RUNTIME_DIR" +''' + result = subprocess.run( + ["bash", "-c", script], + env={ + **os.environ, + "BENCHMARK_LIB": str(BENCHMARK_LIB), + "RUNTIME_DIR": str(runtime_dir), + }, + text=True, + capture_output=True, + check=True, + ) + + assert "PYTHON_ARG=<--target>" in result.stdout + assert f"PYTHON_ARG=<{runtime_dir}>" in result.stdout + assert "PYTHON_ARG=" in result.stdout + assert "--break-system-packages" not in result.stdout + + +def test_kimi_vendor_surfaces_failure_artifact_error(tmp_path: Path) -> None: + script = r''' +source "$BENCHMARK_LIB" +_require_kimi_vendor_python() { return 12; } +_write_kimi_vendor_integration_error() { return 23; } +run_kimi_vendor_eval --results-dir "$RESULTS_DIR" +printf 'EVAL_RC=%s\n' "$?" +''' + result = subprocess.run( + ["bash", "-c", script], + env={ + **os.environ, + "BENCHMARK_LIB": str(BENCHMARK_LIB), + "RESULTS_DIR": str(tmp_path / "results"), + "MODEL": "test-model", + "IS_MULTINODE": "false", + }, + text=True, + capture_output=True, + check=True, + ) + + assert "EVAL_RC=12" in result.stdout + assert "failed to write Kimi verifier failure artifact" in result.stderr + + + + +def test_kimi_vendor_multinode_runner_uses_fixed_upstream_contract( + tmp_path: Path, +) -> None: + results_dir = tmp_path / "results" + verifier_dir = tmp_path / "verifier" + runtime_dir = tmp_path / "runtime" + runtime_dir.mkdir() + verifier_dir.mkdir() + script = r''' +source "$BENCHMARK_LIB" +_require_kimi_vendor_python() { :; } +_prepare_kimi_vendor_runtime() { printf '%s\n' "$RUNTIME_DIR"; } +_prepare_kimi_vendor_verifier() { + printf 'CHECKOUT=%s@%s\n' "$1" "$2" >&2 + printf '%s\n' "$VERIFIER_DIR" +} +python3() { + printf 'PYTHONPATH=<%s>\n' "$PYTHONPATH" + printf 'PYTHON_ARG=<%s>\n' "$@" +} +run_kimi_vendor_eval --port 9999 --results-dir "$RESULTS_DIR" +printf 'EVAL_SUITE=%s\n' "$EVAL_SUITE" +printf 'EVAL_RESULT_DIR=%s\n' "$EVAL_RESULT_DIR" +''' + env = { + **os.environ, + "BENCHMARK_LIB": str(BENCHMARK_LIB), + "RESULTS_DIR": str(results_dir), + "VERIFIER_DIR": str(verifier_dir), + "MODEL": "test-model", + "RUNTIME_DIR": str(runtime_dir), + "OPENAI_API_KEY": "must-not-be-forwarded", + "KV_OFFLOADING": "none", + "IS_MULTINODE": "true", + } + for key in ( + "EVAL_SUITE", + "EVAL_RESULT_DIR", + "MODEL_NAME", + ): + env.pop(key, None) + + result = subprocess.run( + ["bash", "-c", script], + env=env, + text=True, + capture_output=True, + check=True, + ) + output = result.stdout + result.stderr + adapter = BENCHMARK_LIB.parents[1] / "utils/evals/kimi_vendor_eval.py" + + assert f"PYTHONPATH=<{tmp_path / 'runtime'}" in output + assert ( + "CHECKOUT=https://github.com/MoonshotAI/Kimi-Vendor-Verifier.git" + "@b9ed3a6665bdff2c943246f7d2903cd003d6ddd6" + ) in output + for value in ( + adapter, + verifier_dir, + "http://127.0.0.1:9999/v1", + "EMPTY", + "test-model", + results_dir, + ): + assert f"PYTHON_ARG=<{value}>" in output + assert "must-not-be-forwarded" not in output + assert "EVAL_SUITE=kimi_tool_call_schema" in output + assert f"EVAL_RESULT_DIR={results_dir}" in output + assert not (tmp_path / "runtime").exists() + assert not verifier_dir.exists() + + def test_run_lm_eval_rejects_missing_option_value(): result = _run_invalid_call("run_lm_eval --port") assert result.returncode == 2 @@ -177,6 +735,140 @@ def test_lm_eval_defaults_to_gsm8k(): assert "utils/evals/gsm8k.yaml" in out +def _summary_metadata(tmp_path: Path, **overrides: str) -> dict: + work_dir = tmp_path / "work" + results_dir = tmp_path / "results" + work_dir.mkdir(parents=True) + results_dir.mkdir() + script = r''' +source "$BENCHMARK_LIB" +cd "$WORK_DIR" +append_lm_eval_summary >/dev/null +''' + env = { + **os.environ, + "BENCHMARK_LIB": str(BENCHMARK_LIB), + "WORK_DIR": str(work_dir), + "EVAL_RESULT_DIR": str(results_dir), + "MODEL": "test-model", + "CONC": "7", + "KV_OFFLOADING": "none", + } + for key in ("EVAL_COMPLETED_SUITE", "EVAL_SUITE", "EVAL_TASKS_DIR"): + env.pop(key, None) + env.update(overrides) + subprocess.run(["bash", "-c", script], env=env, check=True) + return json.loads((work_dir / "meta_env.json").read_text()) + + +def test_summary_metadata_preserves_lm_eval_gsm8k_defaults(tmp_path: Path) -> None: + meta = _summary_metadata(tmp_path) + + assert meta["eval_suite"] == "gsm8k" + assert meta["conc"] == 7 + + +def test_run_lm_eval_exports_cli_task_path(tmp_path: Path) -> None: + script = r''' +source "$BENCHMARK_LIB" +python3() { :; } +export EVAL_MAX_MODEL_LEN=16384 +export INFERENCEX_LM_EVAL_RUNTIME_READY=true +run_lm_eval --task custom.yaml --results-dir "$RESULTS_DIR" +printf 'EVAL_TASKS_DIR=%s\n' "$EVAL_TASKS_DIR" +''' + env = { + **os.environ, + "BENCHMARK_LIB": str(BENCHMARK_LIB), + "RESULTS_DIR": str(tmp_path / "results"), + "MODEL_NAME": "test-model", + "OPENAI_API_KEY": "EMPTY", + "KV_OFFLOADING": "none", + } + env.pop("EVAL_TASKS_DIR", None) + result = subprocess.run( + ["bash", "-c", script], + env=env, + text=True, + capture_output=True, + check=True, + ) + + assert "EVAL_TASKS_DIR=custom.yaml" in result.stdout + + +def test_summary_metadata_prefers_explicit_suite_then_task_basename( + tmp_path: Path, +) -> None: + from_task = _summary_metadata( + tmp_path / "task", + EVAL_TASKS_DIR="/tmp/custom_reasoning.yaml", + ) + explicit = _summary_metadata( + tmp_path / "explicit", + EVAL_SUITE="kimi_tool_call_schema", + EVAL_TASKS_DIR="/tmp/ignored.yaml", + ) + + assert from_task["eval_suite"] == "custom_reasoning" + assert explicit["eval_suite"] == "kimi_tool_call_schema" + + +def test_summary_metadata_prefers_completed_eval_identity(tmp_path: Path) -> None: + meta = _summary_metadata( + tmp_path, + EVAL_COMPLETED_SUITE="kimi_tool_call_schema", + EVAL_SUITE="stale_input_selector", + EVAL_TASKS_DIR="/tmp/ignored.yaml", + ) + + assert meta["eval_suite"] == "kimi_tool_call_schema" + + +def test_env_is_true_is_case_insensitive_and_unset_safe() -> None: + script = r''' +set -u +source "$BENCHMARK_LIB" +for value in TrUe yEs oN 1 false 0; do + if _env_is_true "$value"; then + echo true + else + echo false + fi +done +for empty_call in with-argument without-argument; do + if [ "$empty_call" = "with-argument" ]; then + _env_is_true "" + else + _env_is_true + fi + if [ "$?" -eq 0 ]; then + echo true + else + echo false + fi +done +''' + result = subprocess.run( + ["bash", "-c", script], + env={**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB)}, + text=True, + capture_output=True, + check=True, + ) + + assert result.stdout.splitlines() == [ + "true", + "true", + "true", + "true", + "false", + "false", + "false", + "false", + ] + + _MODAL_CREDS_SCRIPT = r''' source "$BENCHMARK_LIB" @@ -547,19 +1239,212 @@ def test_agentic_eval_limit_full_runs_whole_split(tmp_path): assert "GEN_RC=0" in res.stdout, res.stdout + res.stderr +def test_multinode_eval_artifact_names_are_bounded_and_distinct() -> None: + workflow = yaml.safe_load(MULTINODE_WORKFLOW.read_text()) + upload = next( + step + for step in workflow["jobs"]["benchmark"]["steps"] + if step.get("name") == "Upload eval results (if any)" + ) + expression = upload["with"]["name"] + assert expression.startswith("eval_") + assert "RESULT_FILENAME" not in expression + + targets = [ + { + "EXP_NAME": "kimik3_p2x16ep32dpa_d0x16ep32dpa_conc12", + "PRECISION": "fp4", + "FRAMEWORK": "vllm", + "PREFILL_NUM_WORKERS": "2", + "PREFILL_TP": "16", + "PREFILL_PP_SIZE": "1", + "SPEC_DECODING": "mtp", + "PREFILL_DCP_SIZE": "1", + "PREFILL_PCP_SIZE": "1", + "PREFILL_EP": "32", + "PREFILL_DP_ATTN": "true", + "DECODE_NUM_WORKERS": "0", + "DECODE_TP": "16", + "DECODE_PP_SIZE": "1", + "DECODE_DCP_SIZE": "1", + "DECODE_PCP_SIZE": "1", + "DECODE_EP": "32", + "DECODE_DP_ATTN": "true", + "KV_OFFLOADING": "none", + "KV_OFFLOAD_BACKEND": "", + "conc-list": ["1", "12", "16"], + "runner.name": "h200-dgxc-slurm_00", + }, + { + "EXP_NAME": "kimik3_p4x8ep32dpa_d0x8ep32dpa_conc12", + "PRECISION": "fp4", + "FRAMEWORK": "vllm", + "PREFILL_NUM_WORKERS": "4", + "PREFILL_TP": "8", + "PREFILL_PP_SIZE": "1", + "PREFILL_DCP_SIZE": "1", + "PREFILL_PCP_SIZE": "1", + "PREFILL_EP": "32", + "PREFILL_DP_ATTN": "true", + "DECODE_NUM_WORKERS": "0", + "SPEC_DECODING": "mtp", + "DECODE_TP": "8", + "DECODE_PP_SIZE": "1", + "DECODE_DCP_SIZE": "1", + "DECODE_PCP_SIZE": "1", + "DECODE_EP": "32", + "DECODE_DP_ATTN": "true", + "KV_OFFLOADING": "none", + "KV_OFFLOAD_BACKEND": "", + "conc-list": ["1", "12", "16"], + "runner.name": "h200-dgxc-slurm_01", + }, + { + "EXP_NAME": "kimik3_p4x8ep32dpa_d0x8ep32dpa_conc12_kvdram-vllm-simple", + "PRECISION": "fp4", + "FRAMEWORK": "vllm", + "PREFILL_NUM_WORKERS": "4", + "PREFILL_TP": "8", + "PREFILL_PP_SIZE": "1", + "PREFILL_DCP_SIZE": "1", + "PREFILL_PCP_SIZE": "1", + "PREFILL_EP": "32", + "PREFILL_DP_ATTN": "true", + "DECODE_NUM_WORKERS": "0", + "DECODE_TP": "8", + "DECODE_PP_SIZE": "1", + "DECODE_DCP_SIZE": "1", + "DECODE_PCP_SIZE": "1", + "DECODE_EP": "32", + "SPEC_DECODING": "mtp", + "DECODE_DP_ATTN": "true", + "KV_OFFLOADING": "dram", + "KV_OFFLOAD_BACKEND": "vllm-simple", + "conc-list": ["1", "12", "16"], + "runner.name": "h200-dgxc-slurm_02", + }, + { + "EXP_NAME": "kimik3_p1x8_d0x8_conc16", + "PRECISION": "fp4", + "FRAMEWORK": "dynamo-vllm", + "PREFILL_NUM_WORKERS": "1", + "PREFILL_TP": "8", + "PREFILL_PP_SIZE": "2", + "PREFILL_DCP_SIZE": "1", + "PREFILL_PCP_SIZE": "1", + "PREFILL_EP": "1", + "PREFILL_DP_ATTN": "false", + "DECODE_NUM_WORKERS": "0", + "DECODE_TP": "8", + "DECODE_PP_SIZE": "2", + "DECODE_DCP_SIZE": "1", + "DECODE_PCP_SIZE": "1", + "DECODE_EP": "1", + "DECODE_DP_ATTN": "false", + "SPEC_DECODING": "mtp", + "KV_OFFLOADING": "none", + "KV_OFFLOAD_BACKEND": "", + "conc-list": ["1", "16"], + "runner.name": "b200-dgxc_00", + }, + { + "EXP_NAME": "kimik3_p1x16ep16_d0x16ep16_conc16", + "PRECISION": "fp4", + "FRAMEWORK": "dynamo-vllm", + "PREFILL_NUM_WORKERS": "1", + "PREFILL_TP": "16", + "PREFILL_PP_SIZE": "1", + "PREFILL_DCP_SIZE": "1", + "PREFILL_PCP_SIZE": "1", + "PREFILL_EP": "16", + "PREFILL_DP_ATTN": "false", + "DECODE_NUM_WORKERS": "0", + "DECODE_TP": "16", + "DECODE_PP_SIZE": "1", + "DECODE_DCP_SIZE": "1", + "DECODE_PCP_SIZE": "1", + "DECODE_EP": "16", + "DECODE_DP_ATTN": "false", + "KV_OFFLOADING": "none", + "SPEC_DECODING": "mtp", + "KV_OFFLOAD_BACKEND": "", + "conc-list": ["16"], + "runner.name": "gb200-nv_00", + }, + { + "EXP_NAME": "kimik3_p1x16_d0x16_conc1", + "PRECISION": "fp4", + "FRAMEWORK": "dynamo-vllm", + "PREFILL_NUM_WORKERS": "1", + "PREFILL_TP": "16", + "PREFILL_PP_SIZE": "1", + "PREFILL_DCP_SIZE": "1", + "PREFILL_PCP_SIZE": "1", + "PREFILL_EP": "1", + "PREFILL_DP_ATTN": "false", + "DECODE_NUM_WORKERS": "0", + "DECODE_TP": "16", + "DECODE_PP_SIZE": "1", + "DECODE_DCP_SIZE": "1", + "DECODE_PCP_SIZE": "1", + "DECODE_EP": "1", + "DECODE_DP_ATTN": "false", + "KV_OFFLOADING": "none", + "KV_OFFLOAD_BACKEND": "", + "SPEC_DECODING": "mtp", + "conc-list": ["1"], + "runner.name": "gb200-nv_01", + }, + ] + non_mtp_twin = {**targets[3], "SPEC_DECODING": "none"} + + def render(values: dict[str, object]) -> str: + name = expression + name = re.sub( + r"\$\{\{ join\(fromJson\(inputs\.conc-list\), 'x'\) \}\}", + "x".join(values["conc-list"]), + name, + ) + for key, value in values.items(): + if key != "conc-list": + name = name.replace(f"${{{{ env.{key} }}}}", str(value)) + name = name.replace("${{ runner.name }}", str(values["runner.name"])) + assert "${{" not in name + return name + + names = [render(target) for target in targets] + assert len(names) == len(set(names)) == 6 + assert render(targets[3]) != render(non_mtp_twin) + assert all(name.startswith("eval_") and len(name.encode()) <= 256 for name in names) + + _GENMODE_SCRIPT = r''' source "$BENCHMARK_LIB" 2>/dev/null _install_swebench_agent_deps() { :; } _ensure_modal_credentials() { :; } -_run_swebench_agentic_generation() { echo "GEN=agentic"; return 42; } -run_lm_eval() { echo "GEN=single-shot"; return 42; } +_run_swebench_agentic_generation() { + echo "GEN=agentic" + echo "SUITE=$EVAL_SUITE" + return 42 +} +run_lm_eval() { + echo "GEN=single-shot" + echo "SUITE=$EVAL_SUITE" + return 42 +} run_swebench_eval --port 8888 echo "RC=$?" ''' -def _gen_mode(tmp_path, *, is_agentic, gen_mode=None) -> str: +def _gen_mode( + tmp_path: Path, + *, + is_agentic, + gen_mode=None, + eval_suite=None, +) -> str: env = {**os.environ, "BENCHMARK_LIB": str(BENCHMARK_LIB), "KV_OFFLOADING": "none", @@ -567,8 +1452,11 @@ def _gen_mode(tmp_path, *, is_agentic, gen_mode=None) -> str: "EVAL_RESULT_DIR": str(tmp_path / "out")} env.pop("SWEBENCH_GEN_MODE", None) env.pop("SCENARIO_TYPE", None) + env.pop("EVAL_SUITE", None) if gen_mode is not None: env["SWEBENCH_GEN_MODE"] = gen_mode + if eval_suite is not None: + env["EVAL_SUITE"] = eval_suite res = subprocess.run(["bash", "-c", _GENMODE_SCRIPT], env=env, text=True, capture_output=True, cwd=BENCHMARK_LIB.parents[1]) @@ -577,7 +1465,9 @@ def _gen_mode(tmp_path, *, is_agentic, gen_mode=None) -> str: def test_gen_mode_defaults_to_agentic(tmp_path): - assert "GEN=agentic" in _gen_mode(tmp_path, is_agentic="1") + output = _gen_mode(tmp_path, is_agentic="1") + assert "GEN=agentic" in output + assert "SUITE=swebench_lite" in output def test_gen_mode_agentic_even_without_agentic_scenario(tmp_path): @@ -585,7 +1475,20 @@ def test_gen_mode_agentic_even_without_agentic_scenario(tmp_path): def test_explicit_single_shot_escape_hatch(tmp_path): - assert "GEN=single-shot" in _gen_mode(tmp_path, is_agentic="1", gen_mode="single-shot") + output = _gen_mode(tmp_path, is_agentic="1", gen_mode="single-shot") + assert "GEN=single-shot" in output + assert "SUITE=swebench_lite" in output + + +def test_swebench_generation_modes_preserve_explicit_suite(tmp_path): + for gen_mode in ("agentic", "single-shot"): + output = _gen_mode( + tmp_path / gen_mode, + is_agentic="1", + gen_mode=gen_mode, + eval_suite="explicit_swebench", + ) + assert "SUITE=explicit_swebench" in output def test_agent_sandbox_cpu_knob(tmp_path): @@ -640,3 +1543,44 @@ def test_eval_limit_full_and_zero_accepted(tmp_path): assert "GEN_RC=0" in res.stdout, f"EVAL_LIMIT={sentinel!r}: {res.stdout}{res.stderr}" argv = (shim / "argv.log").read_text() assert "--slice" not in argv + + +def test_agentic_eval_workflow_forwards_runner_contract() -> None: + workflow = yaml.safe_load(E2E_WORKFLOW.read_text()) + forwarded = workflow["jobs"]["test-sweep-agentic-evals"]["with"] + + assert forwarded["spec-decoding"] == "${{ matrix.config.spec-decoding }}" + assert forwarded["eval-framework"] == "${{ inputs.eval-framework }}" + assert forwarded["eval-suite"] == "${{ inputs.eval-suite }}" + + +def test_multinode_agentic_eval_workflow_forwards_runner_contract() -> None: + workflow = yaml.safe_load(E2E_WORKFLOW.read_text()) + forwarded = workflow["jobs"]["test-sweep-multi-node-agentic-evals"]["with"] + reusable_workflow = yaml.safe_load(MULTINODE_WORKFLOW.read_text()) + + assert forwarded["eval-framework"] == "${{ inputs.eval-framework }}" + assert forwarded["eval-suite"] == "${{ inputs.eval-suite }}" + assert reusable_workflow["env"]["EVAL_FRAMEWORK"] == "${{ inputs.eval-framework }}" + assert reusable_workflow["env"]["EVAL_SUITE"] == "${{ inputs.eval-suite }}" + assert "*_vendor_report.json" in MULTINODE_WORKFLOW.read_text() + + + +def test_trusted_changelog_matrix_keeps_multinode_agentic_evals() -> None: + workflow = yaml.safe_load(E2E_WORKFLOW.read_text()) + get_jobs = next( + step + for step in workflow["jobs"]["get-jobs"]["steps"] + if step.get("id") == "get-jobs" + ) + flatten_command = next( + line + for line in get_jobs["run"].splitlines() + if "rows.extend" in line + ) + + assert '"multinode_agentic_evals"' in flatten_command + get_jobs_command = get_jobs["run"] + assert "EVALS=$(" in get_jobs_command + assert "score_matrix eval" in get_jobs_command \ No newline at end of file diff --git a/utils/evals/thresholds.yaml b/utils/evals/thresholds.yaml index 6ff9731c45..bb3a4f58b6 100644 --- a/utils/evals/thresholds.yaml +++ b/utils/evals/thresholds.yaml @@ -1,6 +1,7 @@ # Model thresholds override task defaults. default: gsm8k: 0.90 + kimi_tool_call_schema: 1.0 gpqa_diamond_cot_n_shot: 0.30 swebench_lite: 0.50 models: diff --git a/utils/evals/validate_scores.py b/utils/evals/validate_scores.py index ba7fc13962..bf4b391728 100644 --- a/utils/evals/validate_scores.py +++ b/utils/evals/validate_scores.py @@ -5,6 +5,7 @@ import argparse import glob import json +import math import os import re import sys @@ -68,6 +69,34 @@ def resolve_threshold(config: dict, prefix: str | None, task: str, fallback: flo return default[task], "default" return fallback, "min-score" +def invalid_effective_count(data: dict, task: str) -> tuple[bool, object]: + """Return whether an explicitly present effective count is invalid.""" + if "n-samples" not in data: + return False, None + sample_counts = data["n-samples"] + if not isinstance(sample_counts, dict) or task not in sample_counts: + return True, sample_counts + task_samples = sample_counts[task] + if not isinstance(task_samples, dict) or "effective" not in task_samples: + return True, task_samples + effective = task_samples["effective"] + invalid = ( + isinstance(effective, bool) + or not isinstance(effective, (int, float)) + or not math.isfinite(effective) + or effective <= 0 + ) + return invalid, effective + + +def integration_error_message(error: object) -> str: + """Render the structured integration error fields for a direct failure.""" + if isinstance(error, dict): + error_type = error.get("type", "unknown") + message = error.get("message", "") + return f"{error_type}: {message}" + return f"unknown: {error}" + def validate_batch_manifest( meta_env_path: str, @@ -277,7 +306,24 @@ def main() -> int: conc_label = f"[conc={match.group(1)}] " if match else "" with open(f) as fh: data = json.load(fh) + if "integration_error" in data: + print( + f"FAIL: {conc_label}integration failure: " + f"{integration_error_message(data['integration_error'])}", + file=sys.stderr, + ) + failed = True + continue for task, metrics in data.get("results", {}).items(): + invalid_effective, effective = invalid_effective_count(data, task) + if invalid_effective: + print( + f"FAIL: {conc_label}{task} invalid effective sample count: " + f"{effective!r}", + file=sys.stderr, + ) + failed = True + continue min_score, source = resolve_threshold(config, prefix, task, args.min_score) for name, val in metrics.items(): if not name.startswith(args.metric_prefix) or "stderr" in name: diff --git a/utils/matrix_logic/generate_sweep_configs.py b/utils/matrix_logic/generate_sweep_configs.py index 5b2dee4108..815eefb01a 100644 --- a/utils/matrix_logic/generate_sweep_configs.py +++ b/utils/matrix_logic/generate_sweep_configs.py @@ -289,10 +289,9 @@ def mark_eval_entries(matrix_values: list[dict], include_agentic: bool = False) - Single-node: run GSM8K through the same lm-eval path as fixed-sequence 8k1k evals, marking the highest-conc entry per (model, runner, framework, precision) group. - - Multi-node: same policy as the fixed-seq-len multi-node case above - (highest eligible conc per distinct parallelism config, via - eval-conc), using SWE-bench since it doesn't support batched - concurrencies. + - Multi-node: run GSM8K through the same lm-eval path, selecting the + highest eligible concurrency per distinct parallelism config via + eval-conc. """ from collections import defaultdict @@ -360,8 +359,7 @@ def _eligible_eval_concs(entry): ag_sn_groups = defaultdict(list) # Multi-node agentic: same "highest eligible conc per distinct # parallelism config" policy as the fixed-seq-len mn_groups above. - # SWE-bench doesn't support batched concurrencies (unlike lm-eval), - # so exactly one conc is picked per group, never the full list. + # The selected eval subset uses exactly one conc per group. ag_mn_groups = defaultdict(list) for i, entry in enumerate(matrix_values): if entry.get(Fields.SCENARIO_TYPE.value) != 'agentic-coding': @@ -402,12 +400,10 @@ def mark_all_eval_entries(matrix_values: list[dict]) -> list[dict]: Evals only run at 8k1k (matching mark_eval_entries), so entries at other sequence lengths (e.g. 1k1k) are passed through untouched rather than expanded into eval rows. - Single-node agentic entries use GSM8K through the same lm-eval path as - fixed-sequence 8k1k evals. Multi-node agentic entries use SWE-bench, - which doesn't support batched concurrencies (unlike lm-eval): multi-node - agentic rows with the same topology are merged (to recombine any chunking - split), but only the highest resulting conc is marked for eval via - eval-conc, not the full list. + Single- and multi-node agentic entries use GSM8K through lm-eval. + Multi-node agentic rows with the same topology are merged (to recombine + any chunking split), but only the highest resulting conc is marked for + eval via eval-conc, matching the default agentic selection policy. Multi-node fixed-seq-len rows with the same engine topology are merged into one eval row whose full concurrency list is run sequentially against the same engine. diff --git a/utils/test_collect_eval_results.py b/utils/test_collect_eval_results.py index 019bbdf123..13f1cefe2e 100644 --- a/utils/test_collect_eval_results.py +++ b/utils/test_collect_eval_results.py @@ -3,7 +3,16 @@ import json from pathlib import Path -from collect_eval_results import build_row, collect_eval_rows +from collect_eval_results import ( + EVAL_RESULT_FORMAT, + build_row, + collect_eval_rows, +) +from evals.kimi_vendor_eval import RESULT_FORMAT as KIMI_VENDOR_RESULT_FORMAT + + +def test_kimi_vendor_result_format_matches_collector_contract() -> None: + assert KIMI_VENDOR_RESULT_FORMAT == EVAL_RESULT_FORMAT def test_build_row_preserves_sequence_lengths() -> None: @@ -21,25 +30,39 @@ def test_build_row_preserves_sequence_lengths() -> None: assert row["isl"] == 1024 assert row["osl"] == 1024 + assert "eval_suite" not in row + + +def test_build_row_preserves_explicit_eval_suite() -> None: + row = build_row( + {"eval_suite": "kimi_tool_call_schema"}, + {"task": "kimi_tool_call_schema"}, + ) + assert row["eval_suite"] == "kimi_tool_call_schema" -def _write_lm_eval_result(path: Path, score: float) -> None: + +def _write_lm_eval_result( + path: Path, + score: float, + task: str = "gsm8k", +) -> None: path.write_text(json.dumps({ "lm_eval_version": "0.4.0", "model_name": "test-model", "results": { - "gsm8k": { + task: { "exact_match,strict-match": score, "exact_match_stderr,strict-match": 0.01, }, }, "configs": { - "gsm8k": { + task: { "metric_list": [{"metric": "exact_match"}], "filter_list": [{"name": "strict-match"}], }, }, - "n-samples": {"gsm8k": {"effective": 10}}, + "n-samples": {task: {"effective": 10}}, })) @@ -67,6 +90,7 @@ def test_collect_eval_rows_expands_batched_concurrencies( "completed_eval_concs": [4, 16], "failed_eval_concs": [], "conc": 4, + "eval_suite": "gsm8k", })) _write_lm_eval_result( artifact_dir / "results_test_conc4.json", @@ -81,6 +105,7 @@ def test_collect_eval_rows_expands_batched_concurrencies( assert [row["conc"] for row in rows] == [4, 16] assert [row["score"] for row in rows] == [0.90, 0.91] + assert {row["eval_suite"] for row in rows} == {"gsm8k"} def test_collect_eval_rows_ignores_failed_batch_points( @@ -107,3 +132,103 @@ def test_collect_eval_rows_ignores_failed_batch_points( rows = collect_eval_rows(tmp_path) assert [row["conc"] for row in rows] == [4] + + + +def test_collect_eval_rows_accepts_neutral_result_format(tmp_path: Path) -> None: + artifact_dir = tmp_path / "eval_provider" + artifact_dir.mkdir() + (artifact_dir / "meta_env.json").write_text( + json.dumps({"eval_suite": "provider_smoke"}) + ) + result_path = artifact_dir / "results_provider.json" + _write_lm_eval_result(result_path, 1.0) + result = json.loads(result_path.read_text()) + result.pop("lm_eval_version") + result["result_format"] = EVAL_RESULT_FORMAT + result_path.write_text(json.dumps(result)) + + rows = collect_eval_rows(tmp_path) + + assert len(rows) == 1 + assert rows[0]["score"] == 1.0 + assert rows[0]["eval_suite"] == "provider_smoke" + + +def test_collect_eval_rows_excludes_integration_and_sample_failures( + tmp_path: Path, +) -> None: + for name, invalid in ( + ("integration", "integration"), + ("zero", 0), + ("nonnumeric", "unknown"), + ("nonfinite", float("nan")), + ("malformed", []), + ): + artifact_dir = tmp_path / f"eval_{name}" + artifact_dir.mkdir() + (artifact_dir / "meta_env.json").write_text(json.dumps({ + "eval_suite": "gsm8k", + })) + result_path = artifact_dir / f"results_{name}.json" + _write_lm_eval_result(result_path, 0.0) + result = json.loads(result_path.read_text()) + if invalid == "integration": + result["integration_error"] = { + "type": "RuntimeError", + "message": "vendor verifier checkout failed", + } + else: + result["n-samples"]["gsm8k"]["effective"] = invalid + result_path.write_text(json.dumps(result)) + + assert collect_eval_rows(tmp_path) == [] + + +def test_collect_eval_rows_accepts_legacy_missing_effective_count( + tmp_path: Path, +) -> None: + artifact_dir = tmp_path / "eval_legacy" + artifact_dir.mkdir() + (artifact_dir / "meta_env.json").write_text(json.dumps({ + "eval_suite": "gsm8k", + })) + result_path = artifact_dir / "results_legacy.json" + _write_lm_eval_result(result_path, 0.9) + result = json.loads(result_path.read_text()) + result.pop("n-samples") + result_path.write_text(json.dumps(result)) + + rows = collect_eval_rows(tmp_path) + + assert len(rows) == 1 + assert rows[0]["score"] == 0.9 + assert rows[0]["n_eff"] is None + + +def test_collect_eval_rows_does_not_resurrect_stale_valid_result( + tmp_path: Path, +) -> None: + artifact_dir = tmp_path / "eval_retry" + artifact_dir.mkdir() + (artifact_dir / "meta_env.json").write_text(json.dumps({ + "eval_suite": "kimi_tool_call_schema", + })) + stale_path = ( + artifact_dir / "results_kimi_vendor_2026-08-12T01-00-00.000000.json" + ) + _write_lm_eval_result(stale_path, 1.0, task="kimi_tool_call_schema") + current_path = ( + artifact_dir / "results_kimi_vendor_2026-08-12T02-00-00.000000.json" + ) + _write_lm_eval_result(current_path, 0.0, task="kimi_tool_call_schema") + result = json.loads(current_path.read_text()) + result["integration_error"] = { + "type": "RuntimeError", + "message": "vendor verifier checkout failed", + } + current_path.write_text(json.dumps(result)) + current_path.touch() + stale_path.touch() + + assert collect_eval_rows(tmp_path) == [] \ No newline at end of file diff --git a/utils/test_validate_reusable_sweep_artifacts.py b/utils/test_validate_reusable_sweep_artifacts.py index 69eb633cdc..cfa0a1df1a 100644 --- a/utils/test_validate_reusable_sweep_artifacts.py +++ b/utils/test_validate_reusable_sweep_artifacts.py @@ -32,8 +32,9 @@ def single_eval_result( runner: str = "h100-dgxc-slurm", isl: int = 8192, osl: int = 1024, + eval_suite: str | None = None, ) -> dict: - return { + row = { "is_multinode": False, "hw": runner.upper(), "model_prefix": "gptoss", @@ -51,6 +52,9 @@ def single_eval_result( "conc": conc, "task": "gsm8k", } + if eval_suite is not None: + row["eval_suite"] = eval_suite + return row def single_eval_meta( @@ -58,8 +62,9 @@ def single_eval_meta( runner: str = "h100-dgxc-slurm", isl: int = 8192, osl: int = 1024, + eval_suite: str | None = None, ) -> dict: - row = single_eval_result(conc, runner, isl, osl) + row = single_eval_result(conc, runner, isl, osl, eval_suite) row["infmax_model_prefix"] = row.pop("model_prefix") return row @@ -72,11 +77,20 @@ def write_raw_eval_artifact( physical_runner: str = "h100-dgxc-slurm_00", isl: int = 8192, osl: int = 1024, + eval_suite: str | None = None, ) -> None: artifact_dir = root / f"eval_result_conc{conc}_{physical_runner}" artifact_dir.mkdir() (artifact_dir / "meta_env.json").write_text( - json.dumps(single_eval_meta(conc, logical_runner, isl, osl)) + json.dumps( + single_eval_meta( + conc, + logical_runner, + isl, + osl, + eval_suite, + ) + ) ) @@ -330,7 +344,7 @@ def test_eval_validation_requires_raw_result_dirs_not_eval_debug_dirs( assert any("unexpected" in error for error in errors) -def test_eval_validation_accepts_matching_raw_and_aggregate( +def test_eval_validation_accepts_matching_legacy_artifacts_without_suite( tmp_path: Path, ) -> None: write_eval_aggregate( @@ -347,6 +361,31 @@ def test_eval_validation_accepts_matching_raw_and_aggregate( assert validate_eval_artifacts(tmp_path) == [] +def test_eval_validation_separates_explicit_suite_identities( + tmp_path: Path, +) -> None: + gsm8k = single_eval_result(32, eval_suite="gsm8k") + tool_use = single_eval_result( + 32, + eval_suite="kimi_tool_call_schema", + ) + write_eval_aggregate(tmp_path, [gsm8k, tool_use]) + write_raw_eval_artifact( + tmp_path, + 32, + eval_suite="gsm8k", + ) + write_raw_eval_artifact( + tmp_path, + 32, + physical_runner="h100-dgxc-slurm_01", + eval_suite="kimi_tool_call_schema", + ) + + assert eval_key(gsm8k) != eval_key(tool_use) + assert validate_eval_artifacts(tmp_path) == [] + + def test_eval_validation_distinguishes_sequence_lengths(tmp_path: Path) -> None: write_eval_aggregate( tmp_path, diff --git a/utils/validate_reusable_sweep_artifacts.py b/utils/validate_reusable_sweep_artifacts.py index 0cfd1d2662..8942d88fbf 100644 --- a/utils/validate_reusable_sweep_artifacts.py +++ b/utils/validate_reusable_sweep_artifacts.py @@ -313,6 +313,10 @@ def normalized_runner(value: Any) -> str: return str(value or "").lower() +LEGACY_EVAL_SUITE = "" + + + def eval_key(row: dict[str, Any]) -> tuple[Any, ...]: """Build an eval identity from one aggregate row.""" if as_bool(row.get("is_multinode", False)): @@ -322,6 +326,7 @@ def eval_key(row: dict[str, Any]) -> tuple[Any, ...]: row.get("model_prefix", row.get("infmax_model_prefix")), row.get("framework"), row.get("precision"), + row.get("eval_suite", LEGACY_EVAL_SUITE), row.get("spec_decoding", "none"), as_int(row.get("isl", 8192), 8192), as_int(row.get("osl", 1024), 1024), @@ -347,6 +352,7 @@ def eval_key(row: dict[str, Any]) -> tuple[Any, ...]: row.get("model_prefix", row.get("infmax_model_prefix")), row.get("framework"), row.get("precision"), + row.get("eval_suite", LEGACY_EVAL_SUITE), row.get("spec_decoding", "none"), as_int(row.get("isl", 8192), 8192), as_int(row.get("osl", 1024), 1024),