diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh new file mode 100755 index 000000000..1851139da --- /dev/null +++ b/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x_mtp.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +set -eo pipefail +set -x + +source "$(dirname "$0")/../../benchmark_lib.sh" + +export EVAL_FRAMEWORK="lm-eval" + +check_env_vars MODEL TP CONC KV_OFFLOADING RESULT_DIR DURATION EP_SIZE DP_ATTENTION PORT EVAL_ONLY + +DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" +NUM_SPEC_TOKENS=3 +SYNTHETIC_ACCEPT_LEN=2.78 + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +if [[ -n "$ROCR_VISIBLE_DEVICES" ]]; then + export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" +fi + +if [[ -n "$MODEL_PATH" ]]; then + if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then + hf download "$MODEL" --local-dir "$MODEL_PATH" + fi +else + hf download "$MODEL" + export MODEL_PATH="$MODEL" +fi +hf download "$DRAFT_MODEL" + +rocm-smi || true +amd-smi || true + +resolve_trace_source +install_agentic_deps + +SERVER_LOG="$RESULT_DIR/server.log" +mkdir -p "$RESULT_DIR" + +SERVER_PID="" +cleanup_agentic_services() { + local exit_code=$? + trap - EXIT INT TERM + set +e + stop_background_process_tree "$SERVER_PID" "vLLM server" 60 + exit "$exit_code" +} +trap cleanup_agentic_services EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +require_agentic_kv_offload_none +export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" +export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" + +if [ "$EVAL_ONLY" = "true" ]; then + SPEC_CONFIG="{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TRITON_ATTN\"}" +else + SPEC_CONFIG="{\"method\": \"eagle3\", \"model\": \"$DRAFT_MODEL\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"attention_backend\": \"TRITON_ATTN\", \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": $SYNTHETIC_ACCEPT_LEN}" +fi + +export PYTHONNOUSERSITE=1 +export VLLM_ENGINE_READY_TIMEOUT_S=3600 +export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800 +export VLLM_USE_BREAKABLE_CUDAGRAPH=0 + +VLLM_CMD=( + vllm serve "$MODEL_PATH" + --served-model-name "$MODEL" + --host 0.0.0.0 + --port "$PORT" + --tensor-parallel-size "$TP" + --gpu-memory-utilization 0.90 + --kv-cache-dtype fp8 + --block-size 128 + --language-model-only + --attention-backend TRITON_ATTN + --enable-prefix-caching + --enable-chunked-prefill + --max-num-batched-tokens 32768 + --max-num-seqs "$((2 * CONC))" + --speculative-config "$SPEC_CONFIG" + --tool-call-parser minimax_m3 + --reasoning-parser minimax_m3 + --enable-auto-tool-choice + --default-chat-template-kwargs '{"thinking_mode":"enabled"}' + --trust-remote-code + --stream-interval 20 +) +write_command "$RESULT_DIR/server_command.txt" "${VLLM_CMD[@]}" +"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & +SERVER_PID=$! + +wait_for_ready \ + --endpoint "http://0.0.0.0:${PORT}/health" \ + --log "$SERVER_LOG" \ + --pid "$SERVER_PID" + +if [ "$EVAL_ONLY" = "true" ]; then + run_eval --port "$PORT" +else + build_replay_cmd "$RESULT_DIR" + REPLAY_CMD+=" --apply-chat-template" + run_agentic_replay_and_write_outputs "$RESULT_DIR" +fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index c94029484..bb7bcb339 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1565,6 +1565,20 @@ minimaxm3-fp8-mi325x-vllm-agentic: - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [24, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 96], router: { name: vllm-router, version: "0.1.14" } } +minimaxm3-fp8-mi325x-vllm-agentic-mtp: + image: vllm/vllm-openai-rocm:v0.27.1 + model: MiniMaxAI/MiniMax-M3-MXFP8 + model-prefix: minimaxm3 + runner: cluster:mi325x-amds + precision: fp8 + framework: vllm + multinode: false + scenarios: + agentic-coding: + - dram-utilization: 0.20 + search-space: + - { tp: 8, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 14, 16, 18] } + minimaxm3-fp4-mi355x-vllm-agentic: image: vllm/vllm-openai-rocm:nightly-dcfebf93f4eccf30f71872283331eee757915daf model: amd/MiniMax-M3-MXFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 696039af7..998671c91 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -1738,7 +1738,7 @@ - "TP=2 and TP=4, concurrency 4-256 for 1k1k and 8k1k sequence lengths" - "Add --gpu-memory-utilization 0.9 to server launch" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1133 - + - config-keys: - dsv4-fp8-h200-vllm @@ -1860,7 +1860,6 @@ - "Image pinned to lmsysorg/sglang:deepseek-v4-b300@sha256:26e116bd211e300dbb76924d56c5cbe6cc3ee5ee2fe314859cb8774f5bc070f3" - "DP-attention path enables SGLANG_OPT_SWA_EVICT_DROP_PAGE_MARGIN=1 for better SWA eviction behavior" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1185 - - config-keys: - dsv4-fp4-b200-sglang @@ -5910,6 +5909,14 @@ - "Replace the TP8/EP8 low-concurrency arm (conc [1,2,4]) hicache offload with kv-offloading: none to reduce per-request latency at low load; conc [1,2,4] are now tested on both tp=4+hicache and tp=8+no-offload so SemiAnalysis can select the Pareto-optimal point per concurrency" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2570 +- config-keys: + - minimaxm3-fp8-mi325x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add MiniMax-M3 MXFP8 AgentX on MI325X with vLLM v0.27.1, EAGLE3-GQA, golden synthetic acceptance length 2.78, and the measured resident TP8 frontier through its c16-c18 latency knee." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2579 + - config-keys: - qwen3.5-fp4-b200-sglang-mtp scenario-type: @@ -5928,7 +5935,7 @@ - "Use native EAGLE MTP (3 steps, top-k 1, 4 draft tokens) and golden synthetic acceptance length 2.49 for throughput; eval retains real verification." - "Follow the official SGLang DeepSeek-V4 Blackwell recipe, require nonempty SGLang server metrics, and keep pooled AgentX connections alive across inter-turn gaps." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2578 - + - config-keys: - dsv4-fp4-b300-sglang-agentic-hicache-mtp scenario-type: @@ -5938,4 +5945,3 @@ - "Use native EAGLE MTP (3 steps, top-k 1, 4 draft tokens) and golden synthetic acceptance length 2.49 for throughput; eval retains real verification." - "Follow the official SGLang DeepSeek-V4 Blackwell recipe, require nonempty SGLang server metrics, keep pooled AgentX connections alive, let AIPerf own HiCache warmup, and reserve transient MoE workspace at DEP8 c512." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2577 -