Add pinned Kimi Vendor Verifier smoke / 添加固定版本的 Kimi Vendor Verifier 冒烟评估

.github/workflows/benchmark-multinode-tmpl.yml

@@ -133,6 +133,16 @@ on:
         type: boolean
         required: false
         default: false
+      eval-framework:
+        description: "Eval runner (lm-eval, swebench, or kimi-vendor)"
+        type: string
+        required: false
+        default: "lm-eval"
+      eval-suite:
+        description: "Kimi Vendor Verifier suite; leave empty for other eval runners"
+        type: string
+        required: false
+        default: ""
       eval-conc:
         description: "Concurrency value or space-separated list for eval requests (overrides default max-of-conc-list)"
         type: string
@@ -232,6 +242,8 @@ env:
   DECODE_HARDWARE: ${{ inputs.decode-hardware }}
   RUN_EVAL: ${{ inputs.run-eval }}
   EVAL_ONLY: ${{ inputs.eval-only }}
+  EVAL_FRAMEWORK: ${{ inputs.eval-framework }}
+  EVAL_SUITE: ${{ inputs.eval-suite }}
   EVAL_CONC: ${{ inputs.eval-conc }}
   EVAL_LIMIT: ${{ inputs.eval-limit }}
   SWEBENCH_GEN_MODE: ${{ inputs.swebench-gen-mode }}
@@ -467,10 +479,11 @@ jobs:
         if: ${{ always() && (env.RUN_EVAL == 'true' || inputs.eval-only) }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: eval_${{ env.RESULT_FILENAME }}
+          name: eval_${{ env.EXP_NAME }}_${{ env.PRECISION }}_${{ env.FRAMEWORK }}_p${{ env.PREFILL_NUM_WORKERS }}x${{ env.PREFILL_TP }}p${{ env.PREFILL_PP_SIZE }}c${{ env.PREFILL_DCP_SIZE }}k${{ env.PREFILL_PCP_SIZE }}e${{ env.PREFILL_EP }}d${{ env.PREFILL_DP_ATTN }}_d${{ env.DECODE_NUM_WORKERS }}x${{ env.DECODE_TP }}p${{ env.DECODE_PP_SIZE }}c${{ env.DECODE_DCP_SIZE }}k${{ env.DECODE_PCP_SIZE }}e${{ env.DECODE_EP }}d${{ env.DECODE_DP_ATTN }}_kv${{ env.KV_OFFLOADING }}-${{ env.KV_OFFLOAD_BACKEND }}_spec${{ env.SPEC_DECODING }}_c${{ join(fromJson(inputs.conc-list), 'x') }}_${{ runner.name }}
           path: |
             meta_env.json
             results*.json
+            *_vendor_report.json
             sample*.jsonl
             agent_preds.json
             predictions.jsonl
@@ -492,6 +505,7 @@ jobs:
         run: |
           rm -f meta_env.json || true
           rm -f results*.json || true
+          rm -f *_vendor_report.json || true
           rm -f sample*.jsonl || true
           rm -f agent_preds.json predictions.jsonl swebench_report_*.json *.traj* || true
 

.github/workflows/benchmark-tmpl.yml

@@ -85,6 +85,16 @@ on:
         type: boolean
         required: false
         default: false
+      eval-framework:
+        description: "Eval runner (lm-eval, swebench, or kimi-vendor)"
+        type: string
+        required: false
+        default: "lm-eval"
+      eval-suite:
+        description: "Kimi Vendor Verifier suite; leave empty for other eval runners"
+        type: string
+        required: false
+        default: ""
       random-range-ratio:
         required: false
         type: string
@@ -173,6 +183,8 @@ env:
   DISAGG: ${{ inputs.disagg }}
   RUN_EVAL: ${{ inputs.run-eval }}
   EVAL_ONLY: ${{ inputs.eval-only }}
+  EVAL_FRAMEWORK: ${{ inputs.eval-framework }}
+  EVAL_SUITE: ${{ inputs.eval-suite }}
   # Agentic-coding env. Fixed-seq-len jobs leave these empty.
   SCENARIO_TYPE: ${{ inputs.scenario-type }}
   SCENARIO_SUBDIR: ${{ inputs.scenario-type == 'agentic-coding' && 'agentic/' || 'fixed_seq_len/' }}
@@ -197,7 +209,7 @@ env:
   MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
   MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
   # These b300 nodes are currently broken.
-  SALLOC_EXCLUDE: 'b300-005,b300-006'
+  SALLOC_EXCLUDE: 'b300-005,b300-006,b300-017'
 
 permissions:
   contents: read
@@ -392,6 +404,7 @@ jobs:
           path: |
             meta_env.json
             results*.json
+            *_vendor_report.json
             sample*.jsonl
             agent_preds.json
             predictions.jsonl
@@ -409,6 +422,7 @@ jobs:
           rm -f meta_env.json || true
           # Remove any eval results JSONs that were moved into workspace
           rm -f results*.json || true
+          rm -f -- ./*_vendor_report.json || true
           rm -f sample*.jsonl || true
           rm -f agent_preds.json predictions.jsonl swebench_report_*.json *.traj* || true
 

.github/workflows/e2e-tests.yml

@@ -45,6 +45,16 @@ on:
                 required: false
                 type: string
                 default: ""
+            eval-framework:
+                description: "Agentic eval runner (lm-eval, swebench, or kimi-vendor)"
+                required: false
+                type: string
+                default: "lm-eval"
+            eval-suite:
+                description: "Kimi Vendor Verifier suite; empty for other runners"
+                required: false
+                type: string
+                default: ""
             swebench-gen-mode:
                 description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)."
                 required: false
@@ -125,6 +135,16 @@ on:
                 required: false
                 type: string
                 default: ""
+            eval-framework:
+                description: "Agentic eval runner (lm-eval, swebench, or kimi-vendor)"
+                required: false
+                type: string
+                default: "lm-eval"
+            eval-suite:
+                description: "Kimi Vendor Verifier suite; empty for other runners"
+                required: false
+                type: string
+                default: ""
             swebench-gen-mode:
                 description: "SWE-bench generation mode (single-shot | agentic). Empty = agentic (single-shot is an explicit debugging escape hatch)."
                 required: false
@@ -236,7 +256,7 @@ jobs:
                       CMD+=(--evals-only)
                     fi
                     RAW_CONFIG_JSON=$("${CMD[@]}")
-                    CONFIG_JSON=$(python3 -c 'import json,sys; data=json.load(sys.stdin); rows=[row for family in ("single_node","multi_node") for group in data.get(family,{}).values() for row in group]; rows.extend(row for family in ("evals","agentic_evals","multinode_evals") for row in data.get(family,[])); print(json.dumps(rows))' <<<"$RAW_CONFIG_JSON")
+                    CONFIG_JSON=$(python3 -c 'import json,sys; data=json.load(sys.stdin); rows=[row for family in ("single_node","multi_node") for group in data.get(family,{}).values() for row in group]; rows.extend(row for family in ("evals","agentic_evals","multinode_evals","multinode_agentic_evals") for row in data.get(family,[])); print(json.dumps(rows))' <<<"$RAW_CONFIG_JSON")
                   else
                     GENERATE_COMMAND="${{ inputs.generate-cli-command || github.event.inputs.generate-cli-command }}"
                     if [ -z "$GENERATE_COMMAND" ]; then
@@ -260,6 +280,7 @@ jobs:
                   MULTI_AGENTIC=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x and not x.get('run-eval', False)]))" | score_matrix multi-agentic)
                   MULTI_AGENTIC_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if x.get('scenario-type') == 'agentic-coding' and 'prefill' in x and x.get('run-eval', False)]))" | score_matrix multi-agentic-eval)
                   SINGLE=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix single)
+                  EVALS=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' not in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix eval)
                   MULTI=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and not x.get('eval-only', False)]))" | score_matrix multi)
                   MULTI_EVAL=$(echo "$CONFIG_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(json.dumps([x for x in d if 'prefill' in x and x.get('scenario-type') != 'agentic-coding' and x.get('run-eval', False)]))" | score_matrix multi-eval)
                   {
@@ -377,6 +398,8 @@ jobs:
             run-eval: true
             eval-only: true
             eval-conc: ${{ matrix.config['eval-all-concs'] && join(matrix.config.conc, ' ') || matrix.config['eval-conc'] }}
+            eval-framework: ${{ inputs.eval-framework }}
+            eval-suite: ${{ inputs.eval-suite }}
             ref: ${{ inputs.ref }}
 
     test-sweep-agentic:
@@ -455,12 +478,14 @@ jobs:
             isl: '0'
             osl: '0'
             max-model-len: '0'
-            spec-decoding: 'none'
+            spec-decoding: ${{ matrix.config.spec-decoding }}
             disagg: ${{ 'false' }}
             run-eval: true
             eval-only: true
             eval-limit: ${{ inputs.eval-limit }}
             swebench-gen-mode: ${{ inputs.swebench-gen-mode }}
+            eval-framework: ${{ inputs.eval-framework }}
+            eval-suite: ${{ inputs.eval-suite }}
             scenario-type: agentic-coding
             ref: ${{ inputs.ref }}
 
@@ -576,6 +601,8 @@ jobs:
             eval-conc: ${{ matrix.config['eval-conc'] }}
             eval-limit: ${{ inputs.eval-limit }}
             swebench-gen-mode: ${{ inputs.swebench-gen-mode }}
+            eval-framework: ${{ inputs.eval-framework }}
+            eval-suite: ${{ inputs.eval-suite }}
             scenario-type: agentic-coding
             ref: ${{ inputs.ref }}
 
@@ -654,6 +681,8 @@ jobs:
             run-eval: true
             eval-only: true
             eval-limit: ${{ inputs.eval-limit }}
+            eval-framework: ${{ inputs.eval-framework }}
+            eval-suite: ${{ inputs.eval-suite }}
             ref: ${{ inputs.ref }}
 
     collect-results:

.github/workflows/run-sweep.yml

@@ -853,12 +853,12 @@ jobs:
             eval-only: true
             eval-conc: ${{ matrix.config['eval-all-concs'] && join(matrix.config.conc, ' ') || matrix.config['eval-conc'] }}
 
-    # Multi-node agentic (SWE-bench) eval rows carry the agentic input shape,
-    # so they are dispatched with sweep-multi-node-agentic's inputs rather
-    # than sweep-multi-node-evals' fixed-seq-len inputs (isl/osl/max-model-len,
-    # which agentic rows don't have). SWE-bench doesn't support batched
-    # concurrencies (unlike lm-eval), so eval-conc is always a single value,
-    # never the joined-list form sweep-multi-node-evals uses.
+    # Multi-node agentic GSM8K eval rows carry the agentic input shape, so
+    # they are dispatched with sweep-multi-node-agentic's inputs rather than
+    # sweep-multi-node-evals' fixed-seq-len inputs (isl/osl/max-model-len,
+    # which agentic rows don't have). Agentic selection uses one highest
+    # eval-conc per topology; fixed-sequence --all-evals rows may instead pass
+    # a joined concurrency list to lm-eval.
     sweep-multi-node-agentic-evals:
         needs: [setup, canary-select, canary-sweep]
         if: >-