From f97ab37e3e2bbf35af0d040f404364290262423e Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 12:05:26 -0500 Subject: [PATCH 1/4] perf(mi325x): gate MiniMax-M3 EAGLE3 LMCache --- .../agentic/minimaxm3_fp8_mi325x_mtp.sh | 229 ++++++++++++++++++ configs/amd-master.yaml | 14 ++ perf-changelog.yaml | 16 +- 3 files changed, 246 insertions(+), 13 deletions(-) create mode 100755 benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh new file mode 100755 index 000000000..433d7cc79 --- /dev/null +++ b/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh @@ -0,0 +1,229 @@ +#!/usr/bin/env bash +set -eo pipefail +set -x + +source "$(dirname "$0")/../../benchmark_lib.sh" + +export EVAL_FRAMEWORK="lm-eval" + +check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION PORT EVAL_ONLY + +DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" +NUM_SPEC_TOKENS=3 +SYNTHETIC_ACCEPT_LEN=2.78 + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +if [[ -n "$ROCR_VISIBLE_DEVICES" ]]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +if [[ -n "$MODEL_PATH" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi +hf download "$DRAFT_MODEL" + +rocm-smi || true +amd-smi || true + +resolve_trace_source +install_agentic_deps + +SERVER_LOG="$RESULT_DIR/server.log" +ROUTER_LOG="$RESULT_DIR/router.log" +LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" +mkdir -p "$RESULT_DIR" + +SERVER_PID="" +ROUTER_PID="" +LMCACHE_PIDS=() +cleanup_agentic_services() { + local exit_code=$? + trap - EXIT INT TERM + set +e + stop_background_process_tree "$ROUTER_PID" "vLLM router" + stop_background_process_tree "$SERVER_PID" "vLLM server" 60 + local i + for i in "${!LMCACHE_PIDS[@]}"; do + stop_background_process_tree "${LMCACHE_PIDS[$i]}" "LMCache server $i" + done + exit "$exit_code" +} +trap cleanup_agentic_services EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +OFFLOAD_ARGS=() +case "$KV_OFFLOAD_BACKEND" in + "") + require_agentic_kv_offload_none + ;; + lmcache) + require_agentic_kv_offload_backend lmcache + LMCACHE_VERSION="0.5.3" + LMCACHE_ROCM_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/v${LMCACHE_VERSION}-rocm" + agentic_pip_install --quiet --no-cache-dir --no-deps \ + "sortedcontainers==2.4.0" \ + "opentelemetry-exporter-prometheus==0.61b0" \ + "cupy-rocm-7-0==14.1.1" \ + "lmcache==${LMCACHE_VERSION}" --find-links "$LMCACHE_ROCM_INDEX" + python3 -c \ + "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \ + >/dev/null + + LMCACHE_N_SERVERS="$TP" + LMCACHE_L1_SHARD_GB=$((TOTAL_CPU_DRAM_GB / LMCACHE_N_SERVERS)) + if [ "$LMCACHE_L1_SHARD_GB" -lt 1 ]; then + echo "Error: LMCache DRAM budget is less than 1 GB per TP rank." >&2 + exit 1 + fi + + LMCACHE_SERVER_URLS=() + LMCACHE_HTTP_PORTS=() + LMCACHE_LOGS=() + : > "$RESULT_DIR/lmcache_command.txt" + for shard in $(seq 0 $((LMCACHE_N_SERVERS - 1))); do + shard_port=$((5555 + shard)) + shard_http_port=$((8080 + shard)) + shard_log="${LMCACHE_LOG%.log}_${shard}.log" + LMCACHE_CMD=( + lmcache server + --host 127.0.0.1 + --port "$shard_port" + --http-host 127.0.0.1 + --http-port "$shard_http_port" + --l1-size-gb "$LMCACHE_L1_SHARD_GB" + --l1-init-size-gb 10 + --l1-read-ttl-seconds 7200 + --chunk-size 256 + --max-workers 2 + --eviction-policy LRU + --supported-transfer-mode lmcache_driven + ) + append_command "$RESULT_DIR/lmcache_command.txt" "${LMCACHE_CMD[@]}" + "${LMCACHE_CMD[@]}" > "$shard_log" 2>&1 & + LMCACHE_PIDS+=($!) + LMCACHE_HTTP_PORTS+=("$shard_http_port") + LMCACHE_LOGS+=("$shard_log") + LMCACHE_SERVER_URLS+=("tcp://127.0.0.1:${shard_port}") + done + for shard in "${!LMCACHE_PIDS[@]}"; do + wait_for_ready \ + --endpoint "http://127.0.0.1:${LMCACHE_HTTP_PORTS[$shard]}/healthcheck" \ + --log "${LMCACHE_LOGS[$shard]}" \ + --pid "${LMCACHE_PIDS[$shard]}" \ + --sleep-interval 1 \ + --timeout 600 + done + LMCACHE_SERVER_URLS_CSV=$(IFS=,; echo "${LMCACHE_SERVER_URLS[*]}") + OFFLOAD_ARGS=( + --kv-transfer-config + "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.server_urls\":\"$LMCACHE_SERVER_URLS_CSV\",\"lmcache.mp.mq_timeout\":6000.0}}" + ) + ;; + *) + echo "Unsupported KV_OFFLOAD_BACKEND: $KV_OFFLOAD_BACKEND" >&2 + exit 1 + ;; +esac + +PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) +if [ "$DP_ATTENTION" = "true" ]; then + PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") +fi +if [ "$EP_SIZE" -gt 1 ]; then + PARALLEL_ARGS+=(--enable-expert-parallel) +fi + +VLLM_BACKEND_PORT="$PORT" +ROUTER_METRICS_PORT=$((PORT + 10000)) +if [ "$DP_ATTENTION" = "true" ]; then + VLLM_BACKEND_PORT=$((PORT + 1)) + export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 + agentic_pip_install --quiet "vllm-router==0.1.14" + export AIPERF_SERVER_METRICS_URLS="http://localhost:${VLLM_BACKEND_PORT}/metrics,http://localhost:${ROUTER_METRICS_PORT}/metrics" +else + export AIPERF_SERVER_METRICS_URLS="http://localhost:${VLLM_BACKEND_PORT}/metrics" +fi +export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" + +if [ "$EVAL_ONLY" = "true" ]; then + SPEC_CONFIG="{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TRITON_ATTN\"}" +else + SPEC_CONFIG="{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TRITON_ATTN\", \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}" +fi + +export PYTHONNOUSERSITE=1 +export VLLM_ENGINE_READY_TIMEOUT_S=3600 +export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800 +export VLLM_USE_BREAKABLE_CUDAGRAPH=0 + +VLLM_CMD=( + vllm serve "$MODEL_PATH" + --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$VLLM_BACKEND_PORT" + "${PARALLEL_ARGS[@]}" + --gpu-memory-utilization 0.90 + --kv-cache-dtype fp8 + --block-size 128 + --language-model-only + --attention-backend TRITON_ATTN + --enable-prefix-caching + --enable-chunked-prefill + --max-num-batched-tokens 32768 + --max-num-seqs "$((2 * CONC))" + --speculative-config "$SPEC_CONFIG" + --tool-call-parser minimax_m3 + --reasoning-parser minimax_m3 + --enable-auto-tool-choice + --default-chat-template-kwargs '{"thinking_mode":"enabled"}' + --trust-remote-code + --stream-interval 20 + "${OFFLOAD_ARGS[@]}" +) +write_command "$RESULT_DIR/server_command.txt" "${VLLM_CMD[@]}" +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! + +wait_for_ready \ + --endpoint "http://0.0.0.0:${VLLM_BACKEND_PORT}/health" \ + --log "$SERVER_LOG" \ + --pid "$SERVER_PID" + +if [ "$DP_ATTENTION" = "true" ]; then + ROUTER_CMD=( + vllm-router + --worker-urls "http://localhost:$VLLM_BACKEND_PORT" + --policy consistent_hash + --intra-node-data-parallel-size "$TP" + --host 0.0.0.0 + --port "$PORT" + --prometheus-host 127.0.0.1 + --prometheus-port "$ROUTER_METRICS_PORT" + --request-timeout-secs 14400 + --disable-retries + ) + write_command "$RESULT_DIR/router_command.txt" "${ROUTER_CMD[@]}" + "${ROUTER_CMD[@]}" > "$ROUTER_LOG" 2>&1 & + ROUTER_PID=$! + wait_for_ready \ + --endpoint "http://0.0.0.0:${PORT}/health" \ + --log "$ROUTER_LOG" \ + --pid "$ROUTER_PID" +fi + +if [ "$EVAL_ONLY" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + REPLAY_CMD+=" --apply-chat-template" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index c94029484..276d7e5b4 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1565,6 +1565,20 @@ minimaxm3-fp8-mi325x-vllm-agentic: - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [24, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 96], router: { name: vllm-router, version: "0.1.14" } } +minimaxm3-fp8-mi325x-vllm-agentic-mtp: + image: vllm/vllm-openai-rocm:v0.27.1 + model: MiniMaxAI/MiniMax-M3-MXFP8 + model-prefix: minimaxm3 + runner: cluster:mi325x-amds + precision: fp8 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.20 + search-space: + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.3" }, conc-list: [18] } + minimaxm3-fp4-mi355x-vllm-agentic: image: vllm/vllm-openai-rocm:nightly-dcfebf93f4eccf30f71872283331eee757915daf model: amd/MiniMax-M3-MXFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f118b99aa..977730097 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5894,18 +5894,8 @@ - "Cover the measured resident TP2/EP2 and TP4 Pareto ranges through their HBM capacity knees, with required SGLang metrics exports." - "Use SGLang v0.5.17 and disable unstable AITER all-reduce fusion for TP2/EP2 EAGLE rank consistency." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2562 - - config-keys: - - dsv4-fp4-b200-dynamo-sglang-mtp - description: - - "Add a DeepSeek-V4-Pro FP4 Dynamo-SGLang MTP configuration for B200 with five 8k/1k disaggregated topologies." - - "Use NIXL for KV transfer with EAGLE speculative decoding and chat-formatted benchmark inputs." - - "Image: lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2554 - -- config-keys: - - glm5.2-fp4-mi355x-sglang-agentic-mtp + - minimaxm3-fp8-mi325x-vllm-agentic-mtp description: - - "Increase MTP speculative steps from 3 to 5 (num-draft-tokens 4→6) and raise SGLANG_SIMULATE_ACC_LEN from 2.99 to 3.61 to reflect higher measured acceptance rate, targeting ~15% throughput improvement" - - "Replace the TP8/EP8 low-concurrency arm (conc [1,2,4]) hicache offload with kv-offloading: none to reduce per-request latency at low load; conc [1,2,4] are now tested on both tp=4+hicache and tp=8+no-offload so SemiAnalysis can select the Pareto-optimal point per concurrency" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2570 + - "Add MiniMax-M3 MXFP8 AgentX on MI325X with vLLM v0.27.1, EAGLE3-GQA, golden synthetic acceptance length 2.78, complete server metrics, and an isolated LMCache MP functional gate before broad resident discovery." + pr-link: XXX From 09a3442ef7b3e0fcf39e8f2901a223336800facb Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 12:05:54 -0500 Subject: [PATCH 2/4] docs(perf): link MI325X MiniMax-M3 PR --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 977730097..6d2a18b80 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5898,4 +5898,4 @@ - minimaxm3-fp8-mi325x-vllm-agentic-mtp description: - "Add MiniMax-M3 MXFP8 AgentX on MI325X with vLLM v0.27.1, EAGLE3-GQA, golden synthetic acceptance length 2.78, complete server metrics, and an isolated LMCache MP functional gate before broad resident discovery." - pr-link: XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2579 From 1f3a2b5ea4021ef2bc62129731a731f818a9a9df Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 12:56:34 -0500 Subject: [PATCH 3/4] perf(mi325x): expand MiniMax-M3 AgentX discovery --- configs/amd-master.yaml | 7 ++++++- perf-changelog.yaml | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 276d7e5b4..597d25608 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1577,7 +1577,12 @@ minimaxm3-fp8-mi325x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.20 search-space: - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.3" }, conc-list: [18] } + - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12] } + - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12] } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [2, 4, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } + - { tp: 8, ep: 8, dp-attn: true, spec-decoding: mtp, kv-offloading: none, conc-list: [16, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.3" }, conc-list: [18, 20, 22, 24, 28, 32] } minimaxm3-fp4-mi355x-vllm-agentic: image: vllm/vllm-openai-rocm:nightly-dcfebf93f4eccf30f71872283331eee757915daf diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6d2a18b80..813eae66a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5897,5 +5897,5 @@ - config-keys: - minimaxm3-fp8-mi325x-vllm-agentic-mtp description: - - "Add MiniMax-M3 MXFP8 AgentX on MI325X with vLLM v0.27.1, EAGLE3-GQA, golden synthetic acceptance length 2.78, complete server metrics, and an isolated LMCache MP functional gate before broad resident discovery." + - "Add MiniMax-M3 MXFP8 AgentX on MI325X with vLLM v0.27.1, EAGLE3-GQA, golden synthetic acceptance length 2.78, complete server metrics, broad TP4/TP8/TEP/DEP discovery, and LMCache MP around the TP8/EP8 memory knee." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2579 From 7eeea89b2de40637a98d5d20ccb40b4504ba9aed Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 16:18:57 -0500 Subject: [PATCH 4/4] perf(mi325x): select MiniMax-M3 AgentX frontier --- .../agentic/minimaxm3_fp8_mi325x_mtp.sh | 134 +----------------- configs/amd-master.yaml | 7 +- perf-changelog.yaml | 20 ++- 3 files changed, 26 insertions(+), 135 deletions(-) diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh index 433d7cc79..1851139da 100755 --- a/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh +++ b/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh @@ -6,7 +6,7 @@ source "$(dirname "$0")/../../benchmark_lib.sh" export EVAL_FRAMEWORK="lm-eval" -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION PORT EVAL_ONLY +check_env_vars MODEL TP CONC KV_OFFLOADING RESULT_DIR DURATION EP_SIZE DP_ATTENTION PORT EVAL_ONLY DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" NUM_SPEC_TOKENS=3 @@ -37,121 +37,22 @@ resolve_trace_source install_agentic_deps SERVER_LOG="$RESULT_DIR/server.log" -ROUTER_LOG="$RESULT_DIR/router.log" -LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" mkdir -p "$RESULT_DIR" SERVER_PID="" -ROUTER_PID="" -LMCACHE_PIDS=() cleanup_agentic_services() { local exit_code=$? trap - EXIT INT TERM set +e - stop_background_process_tree "$ROUTER_PID" "vLLM router" stop_background_process_tree "$SERVER_PID" "vLLM server" 60 - local i - for i in "${!LMCACHE_PIDS[@]}"; do - stop_background_process_tree "${LMCACHE_PIDS[$i]}" "LMCache server $i" - done exit "$exit_code" } trap cleanup_agentic_services EXIT trap 'exit 130' INT trap 'exit 143' TERM -OFFLOAD_ARGS=() -case "$KV_OFFLOAD_BACKEND" in - "") - require_agentic_kv_offload_none - ;; - lmcache) - require_agentic_kv_offload_backend lmcache - LMCACHE_VERSION="0.5.3" - LMCACHE_ROCM_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/v${LMCACHE_VERSION}-rocm" - agentic_pip_install --quiet --no-cache-dir --no-deps \ - "sortedcontainers==2.4.0" \ - "opentelemetry-exporter-prometheus==0.61b0" \ - "cupy-rocm-7-0==14.1.1" \ - "lmcache==${LMCACHE_VERSION}" --find-links "$LMCACHE_ROCM_INDEX" - python3 -c \ - "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \ - >/dev/null - - LMCACHE_N_SERVERS="$TP" - LMCACHE_L1_SHARD_GB=$((TOTAL_CPU_DRAM_GB / LMCACHE_N_SERVERS)) - if [ "$LMCACHE_L1_SHARD_GB" -lt 1 ]; then - echo "Error: LMCache DRAM budget is less than 1 GB per TP rank." >&2 - exit 1 - fi - - LMCACHE_SERVER_URLS=() - LMCACHE_HTTP_PORTS=() - LMCACHE_LOGS=() - : > "$RESULT_DIR/lmcache_command.txt" - for shard in $(seq 0 $((LMCACHE_N_SERVERS - 1))); do - shard_port=$((5555 + shard)) - shard_http_port=$((8080 + shard)) - shard_log="${LMCACHE_LOG%.log}_${shard}.log" - LMCACHE_CMD=( - lmcache server - --host 127.0.0.1 - --port "$shard_port" - --http-host 127.0.0.1 - --http-port "$shard_http_port" - --l1-size-gb "$LMCACHE_L1_SHARD_GB" - --l1-init-size-gb 10 - --l1-read-ttl-seconds 7200 - --chunk-size 256 - --max-workers 2 - --eviction-policy LRU - --supported-transfer-mode lmcache_driven - ) - append_command "$RESULT_DIR/lmcache_command.txt" "${LMCACHE_CMD[@]}" - "${LMCACHE_CMD[@]}" > "$shard_log" 2>&1 & - LMCACHE_PIDS+=($!) - LMCACHE_HTTP_PORTS+=("$shard_http_port") - LMCACHE_LOGS+=("$shard_log") - LMCACHE_SERVER_URLS+=("tcp://127.0.0.1:${shard_port}") - done - for shard in "${!LMCACHE_PIDS[@]}"; do - wait_for_ready \ - --endpoint "http://127.0.0.1:${LMCACHE_HTTP_PORTS[$shard]}/healthcheck" \ - --log "${LMCACHE_LOGS[$shard]}" \ - --pid "${LMCACHE_PIDS[$shard]}" \ - --sleep-interval 1 \ - --timeout 600 - done - LMCACHE_SERVER_URLS_CSV=$(IFS=,; echo "${LMCACHE_SERVER_URLS[*]}") - OFFLOAD_ARGS=( - --kv-transfer-config - "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.server_urls\":\"$LMCACHE_SERVER_URLS_CSV\",\"lmcache.mp.mq_timeout\":6000.0}}" - ) - ;; - *) - echo "Unsupported KV_OFFLOAD_BACKEND: $KV_OFFLOAD_BACKEND" >&2 - exit 1 - ;; -esac - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [ "$DP_ATTENTION" = "true" ]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi -if [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS+=(--enable-expert-parallel) -fi - -VLLM_BACKEND_PORT="$PORT" -ROUTER_METRICS_PORT=$((PORT + 10000)) -if [ "$DP_ATTENTION" = "true" ]; then - VLLM_BACKEND_PORT=$((PORT + 1)) - export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 - agentic_pip_install --quiet "vllm-router==0.1.14" - export AIPERF_SERVER_METRICS_URLS="http://localhost:${VLLM_BACKEND_PORT}/metrics,http://localhost:${ROUTER_METRICS_PORT}/metrics" -else - export AIPERF_SERVER_METRICS_URLS="http://localhost:${VLLM_BACKEND_PORT}/metrics" -fi +require_agentic_kv_offload_none +export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" if [ "$EVAL_ONLY" = "true" ]; then @@ -169,8 +70,8 @@ VLLM_CMD=( vllm serve "$MODEL_PATH" --served-model-name "$MODEL" --host 0.0.0.0 - --port "$VLLM_BACKEND_PORT" - "${PARALLEL_ARGS[@]}" + --port "$PORT" + --tensor-parallel-size "$TP" --gpu-memory-utilization 0.90 --kv-cache-dtype fp8 --block-size 128 @@ -187,39 +88,16 @@ VLLM_CMD=( --default-chat-template-kwargs '{"thinking_mode":"enabled"}' --trust-remote-code --stream-interval 20 - "${OFFLOAD_ARGS[@]}" ) write_command "$RESULT_DIR/server_command.txt" "${VLLM_CMD[@]}" "${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & SERVER_PID=$! wait_for_ready \ - --endpoint "http://0.0.0.0:${VLLM_BACKEND_PORT}/health" \ + --endpoint "http://0.0.0.0:${PORT}/health" \ --log "$SERVER_LOG" \ --pid "$SERVER_PID" -if [ "$DP_ATTENTION" = "true" ]; then - ROUTER_CMD=( - vllm-router - --worker-urls "http://localhost:$VLLM_BACKEND_PORT" - --policy consistent_hash - --intra-node-data-parallel-size "$TP" - --host 0.0.0.0 - --port "$PORT" - --prometheus-host 127.0.0.1 - --prometheus-port "$ROUTER_METRICS_PORT" - --request-timeout-secs 14400 - --disable-retries - ) - write_command "$RESULT_DIR/router_command.txt" "${ROUTER_CMD[@]}" - "${ROUTER_CMD[@]}" > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - wait_for_ready \ - --endpoint "http://0.0.0.0:${PORT}/health" \ - --log "$ROUTER_LOG" \ - --pid "$ROUTER_PID" -fi - if [ "$EVAL_ONLY" = "true" ]; then run_eval --port "$PORT" else diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 597d25608..bb7bcb339 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1577,12 +1577,7 @@ minimaxm3-fp8-mi325x-vllm-agentic-mtp: agentic-coding: - dram-utilization: 0.20 search-space: - - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12] } - - { tp: 4, ep: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12] } - - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [2, 4, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - - { tp: 8, ep: 8, dp-attn: true, spec-decoding: mtp, kv-offloading: none, conc-list: [16, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64], router: { name: vllm-router, version: "0.1.14" } } - - { tp: 8, ep: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.3" }, conc-list: [18, 20, 22, 24, 28, 32] } + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 14, 16, 18] } minimaxm3-fp4-mi355x-vllm-agentic: image: vllm/vllm-openai-rocm:nightly-dcfebf93f4eccf30f71872283331eee757915daf diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 813eae66a..92c5ff1dc 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5894,8 +5894,26 @@ - "Cover the measured resident TP2/EP2 and TP4 Pareto ranges through their HBM capacity knees, with required SGLang metrics exports." - "Use SGLang v0.5.17 and disable unstable AITER all-reduce fusion for TP2/EP2 EAGLE rank consistency." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2562 + +- config-keys: + - dsv4-fp4-b200-dynamo-sglang-mtp + description: + - "Add a DeepSeek-V4-Pro FP4 Dynamo-SGLang MTP configuration for B200 with five 8k/1k disaggregated topologies." + - "Use NIXL for KV transfer with EAGLE speculative decoding and chat-formatted benchmark inputs." + - "Image: lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2554 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + description: + - "Increase MTP speculative steps from 3 to 5 (num-draft-tokens 4→6) and raise SGLANG_SIMULATE_ACC_LEN from 2.99 to 3.61 to reflect higher measured acceptance rate, targeting ~15% throughput improvement" + - "Replace the TP8/EP8 low-concurrency arm (conc [1,2,4]) hicache offload with kv-offloading: none to reduce per-request latency at low load; conc [1,2,4] are now tested on both tp=4+hicache and tp=8+no-offload so SemiAnalysis can select the Pareto-optimal point per concurrency" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2570 + - config-keys: - minimaxm3-fp8-mi325x-vllm-agentic-mtp + scenario-type: + - agentic-coding description: - - "Add MiniMax-M3 MXFP8 AgentX on MI325X with vLLM v0.27.1, EAGLE3-GQA, golden synthetic acceptance length 2.78, complete server metrics, broad TP4/TP8/TEP/DEP discovery, and LMCache MP around the TP8/EP8 memory knee." + - "Add MiniMax-M3 MXFP8 AgentX on MI325X with vLLM v0.27.1, EAGLE3-GQA, golden synthetic acceptance length 2.78, and the measured resident TP8 frontier through its c16-c18 latency knee." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2579