From 23d91fb1ca9475a199f207e175f8254279506248 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 12:06:31 -0500 Subject: [PATCH 1/4] feat: add MI300X MiniMax-M3 EAGLE3 AgentX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a broad TP4/TP8 resident discovery grid plus an isolated LMCache MP offload gate for MiniMax-M3 MXFP8 on MI300X. Use vLLM EAGLE3-GQA with the committed golden acceptance length and require the single aggregate vLLM metrics endpoint. 新增 MI300X MiniMax-M3 MXFP8 的 TP4/TP8 常驻发现矩阵及独立 LMCache MP 卸载验证点。使用 vLLM EAGLE3-GQA 与已提交的黄金接受长度，并要求采集单一聚合 vLLM 指标端点。 --- .../agentic/minimaxm3_fp8_mi300x_mtp.sh | 193 ++++++++++++++++++ configs/amd-master.yaml | 18 ++ perf-changelog.yaml | 8 + 3 files changed, 219 insertions(+) create mode 100755 benchmarks/single_node/agentic/minimaxm3_fp8_mi300x_mtp.sh diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x_mtp.sh new file mode 100755 index 000000000..88d907881 --- /dev/null +++ b/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x_mtp.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash +set -eo pipefail +set -x + +# MiniMax-M3 MXFP8 MI300X AgentX with EAGLE3-GQA and optional LMCache MP. + +source "$(dirname "$0")/../../benchmark_lib.sh" + +export EVAL_FRAMEWORK="lm-eval" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION PORT EVAL_ONLY + +DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" +NUM_SPEC_TOKENS=3 +SYNTHETIC_ACCEPT_LEN=2.78 + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi +if [[ -n "$ROCR_VISIBLE_DEVICES" ]]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +if [[ -n "$MODEL_PATH" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi +hf download "$DRAFT_MODEL" + +rocm-smi || true +amd-smi || true +resolve_trace_source +install_agentic_deps + +export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" +export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" + +SERVER_LOG="$RESULT_DIR/server.log" +LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" +mkdir -p "$RESULT_DIR" + +SERVER_PID="" +LMCACHE_PIDS=() +cleanup_agentic_services() { + local exit_code=$? + trap - EXIT INT TERM + set +e + stop_background_process_tree "$SERVER_PID" "vLLM server" 60 + local i + for i in "${!LMCACHE_PIDS[@]}"; do + stop_background_process_tree "${LMCACHE_PIDS[$i]}" "LMCache server $i" + done + exit "$exit_code" +} +trap cleanup_agentic_services EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +OFFLOAD_ARGS=() +case "$KV_OFFLOAD_BACKEND" in + "") + require_agentic_kv_offload_none + ;; + lmcache) + require_agentic_kv_offload_backend lmcache + LMCACHE_VERSION="0.5.3" + LMCACHE_ROCM_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/v${LMCACHE_VERSION}-rocm" + agentic_pip_install --quiet --no-cache-dir --no-deps \ + "sortedcontainers==2.4.0" \ + "opentelemetry-exporter-prometheus==0.61b0" \ + "cupy-rocm-7-0==14.1.1" \ + "lmcache==${LMCACHE_VERSION}" --find-links "$LMCACHE_ROCM_INDEX" + python3 -c \ + "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \ + >/dev/null + + LMCACHE_L1_SHARD_GB=$((TOTAL_CPU_DRAM_GB / TP)) + if [ "$LMCACHE_L1_SHARD_GB" -lt 1 ]; then + echo "Error: LMCache DRAM budget is less than 1 GB per TP rank." >&2 + exit 1 + fi + + LMCACHE_SERVER_URLS=() + LMCACHE_HTTP_PORTS=() + LMCACHE_LOGS=() + : > "$RESULT_DIR/lmcache_command.txt" + for shard in $(seq 0 $((TP - 1))); do + shard_port=$((5555 + shard)) + shard_http_port=$((8080 + shard)) + shard_log="${LMCACHE_LOG%.log}_${shard}.log" + LMCACHE_CMD=( + lmcache server + --host 127.0.0.1 + --port "$shard_port" + --http-host 127.0.0.1 + --http-port "$shard_http_port" + --l1-size-gb "$LMCACHE_L1_SHARD_GB" + --l1-init-size-gb 10 + --l1-read-ttl-seconds 7200 + --chunk-size 256 + --max-workers 2 + --eviction-policy LRU + --supported-transfer-mode lmcache_driven + ) + append_command "$RESULT_DIR/lmcache_command.txt" "${LMCACHE_CMD[@]}" + "${LMCACHE_CMD[@]}" > "$shard_log" 2>&1 & + LMCACHE_PIDS+=($!) + LMCACHE_HTTP_PORTS+=("$shard_http_port") + LMCACHE_LOGS+=("$shard_log") + LMCACHE_SERVER_URLS+=("tcp://127.0.0.1:${shard_port}") + done + for shard in "${!LMCACHE_PIDS[@]}"; do + wait_for_ready \ + --endpoint "http://127.0.0.1:${LMCACHE_HTTP_PORTS[$shard]}/healthcheck" \ + --log "${LMCACHE_LOGS[$shard]}" \ + --pid "${LMCACHE_PIDS[$shard]}" \ + --sleep-interval 1 \ + --timeout 600 + done + LMCACHE_SERVER_URLS_CSV=$(IFS=,; echo "${LMCACHE_SERVER_URLS[*]}") + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.server_urls\":\"$LMCACHE_SERVER_URLS_CSV\",\"lmcache.mp.mq_timeout\":6000.0}}" + ) + ;; + *) + echo "Unsupported KV_OFFLOAD_BACKEND: $KV_OFFLOAD_BACKEND" >&2 + exit 1 + ;; +esac + +PARALLEL_ARGS=(--tensor-parallel-size "$TP") +if [ "$EP_SIZE" -gt 1 ]; then + PARALLEL_ARGS+=(--enable-expert-parallel) +fi + +if [ "$EVAL_ONLY" = "true" ]; then + SPEC_CONFIG="{\"method\":\"eagle3\",\"model\":\"$DRAFT_MODEL\",\"num_speculative_tokens\":$NUM_SPEC_TOKENS,\"attention_backend\":\"TRITON_ATTN\"}" +else + SPEC_CONFIG="{\"method\":\"eagle3\",\"model\":\"$DRAFT_MODEL\",\"num_speculative_tokens\":$NUM_SPEC_TOKENS,\"attention_backend\":\"TRITON_ATTN\",\"rejection_sample_method\":\"synthetic\",\"synthetic_acceptance_length\":$SYNTHETIC_ACCEPT_LEN}" +fi + +export PYTHONNOUSERSITE=1 +export VLLM_ENGINE_READY_TIMEOUT_S=3600 +export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800 +export VLLM_USE_BREAKABLE_CUDAGRAPH=0 +export VLLM_ROCM_USE_AITER=1 +export VLLM_ROCM_USE_AITER_MHA=0 +export TORCH_BLAS_PREFER_HIPBLASLT=1 +export NCCL_MIN_NCHANNELS=112 +export GPU_MAX_HW_QUEUES=2 + +VLLM_CMD=( + vllm serve "$MODEL_PATH" + --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$PORT" + "${PARALLEL_ARGS[@]}" + --trust-remote-code + --block-size 128 + --gpu-memory-utilization 0.90 + --enable-chunked-prefill + --max-num-batched-tokens 16384 + --language-model-only + --enable-prefix-caching + --attention-backend TRITON_ATTN + --kv-cache-dtype fp8 + --tool-call-parser minimax_m3 + --reasoning-parser minimax_m3 + --enable-auto-tool-choice + --default-chat-template-kwargs '{"thinking_mode":"enabled"}' + --max-num-seqs "$((2 * CONC))" + --stream-interval 20 + --speculative-config "$SPEC_CONFIG" + "${OFFLOAD_ARGS[@]}" +) +write_command "$RESULT_DIR/server_command.txt" "${VLLM_CMD[@]}" +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +if [ "$EVAL_ONLY" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + REPLAY_CMD+=" --apply-chat-template" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index c94029484..bb85733e5 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1522,6 +1522,24 @@ minimaxm3-fp8-mi300x-vllm-agentic: - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } +minimaxm3-fp8-mi300x-vllm-agentic-mtp: + image: vllm/vllm-openai-rocm:v0.27.1 + model: MiniMaxAI/MiniMax-M3-MXFP8 + model-prefix: minimaxm3 + runner: cluster:mi300x-amds + precision: fp8 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 10, 12] } + - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 10, 12] } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 24, 28, 32] } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 10, 12, 16, 20] } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.3" }, conc-list: [16] } + # GLM-5.2 FP8 full-context AgentX refresh on MI325X. This preserves the TP8 # GPU-resident-KV c1/c2/c3/c4/c5/c6/c8 curve from Actions run 29657732517 # and enables EAGLE MTP with the committed thinking-on golden AL. diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6082f2b86..b580ef362 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5918,3 +5918,11 @@ - "Add a TEP2 arm (tp 2, ep 2) to the qwen3.5-fp4-b200-sglang-mtp 8k/1k sweep at concurrency 16, 32, and 64" - "Rides on the NVFP4-V2 checkpoint switch from #2205" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2550 + +- config-keys: + - minimaxm3-fp8-mi300x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add MI300X MiniMax-M3 MXFP8 AgentX with vLLM EAGLE3-GQA, golden synthetic acceptance length 2.78, a broad TP8/EP8 discovery grid, required server metrics, and an isolated LMCache MP DRAM-offload gate." + pr-link: null From 62145ccb6c1f2db3664ca1b5b59f77d99587d1b5 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 12:07:16 -0500 Subject: [PATCH 2/4] chore: link MI300X MiniMax-M3 changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Record PR #2580 in the appended performance changelog entry. 在新增的性能变更日志条目中记录 PR #2580。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b580ef362..a28d4b022 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5925,4 +5925,4 @@ - agentic-coding description: - "Add MI300X MiniMax-M3 MXFP8 AgentX with vLLM EAGLE3-GQA, golden synthetic acceptance length 2.78, a broad TP8/EP8 discovery grid, required server metrics, and an isolated LMCache MP DRAM-offload gate." - pr-link: null + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2580 From 4c9712526909e29e5867f5bb9c11d876203563c7 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 20:23:51 -0500 Subject: [PATCH 3/4] perf: select measured MI300X MiniMax-M3 frontier --- configs/amd-master.yaml | 5 +---- perf-changelog.yaml | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index bb85733e5..c88a52547 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1534,10 +1534,7 @@ minimaxm3-fp8-mi300x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 10, 12] } - - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 10, 12] } - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 24, 28, 32] } - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 10, 12, 16, 20] } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 10] } - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.3" }, conc-list: [16] } # GLM-5.2 FP8 full-context AgentX refresh on MI325X. This preserves the TP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a28d4b022..550482a8e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5924,5 +5924,5 @@ scenario-type: - agentic-coding description: - - "Add MI300X MiniMax-M3 MXFP8 AgentX with vLLM EAGLE3-GQA, golden synthetic acceptance length 2.78, a broad TP8/EP8 discovery grid, required server metrics, and an isolated LMCache MP DRAM-offload gate." + - "Add MI300X MiniMax-M3 MXFP8 AgentX with vLLM EAGLE3-GQA, measured resident TP8 c1-c10, and an LMCache MP DRAM-offload c16 point." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2580 From 074ec53e9e5477eeb852f47ef80b1421edcfcd66 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 13 Aug 2026 01:05:36 -0500 Subject: [PATCH 4/4] perf: drop non-validatable MI300X c1 point --- configs/amd-master.yaml | 2 +- perf-changelog.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index c88a52547..cee570704 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1534,7 +1534,7 @@ minimaxm3-fp8-mi300x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 6, 8, 10] } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [2, 4, 6, 8, 10] } - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.3" }, conc-list: [16] } # GLM-5.2 FP8 full-context AgentX refresh on MI325X. This preserves the TP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 550482a8e..ae3b10e4c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5924,5 +5924,5 @@ scenario-type: - agentic-coding description: - - "Add MI300X MiniMax-M3 MXFP8 AgentX with vLLM EAGLE3-GQA, measured resident TP8 c1-c10, and an LMCache MP DRAM-offload c16 point." + - "Add MI300X MiniMax-M3 MXFP8 AgentX with vLLM EAGLE3-GQA, measured resident TP8 c2-c10, and an LMCache MP DRAM-offload c16 point." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2580