[AMD][AgentX] Add MI300X MiniMax-M3 EAGLE3 MXFP8 #2580
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,193 @@ | ||
| #!/usr/bin/env bash | ||
| set -eo pipefail | ||
| set -x | ||
|
|
||
| # MiniMax-M3 MXFP8 MI300X AgentX with EAGLE3-GQA and optional LMCache MP. | ||
|
|
||
| source "$(dirname "$0")/../../benchmark_lib.sh" | ||
|
|
||
| export EVAL_FRAMEWORK="lm-eval" | ||
|
|
||
| check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION PORT EVAL_ONLY | ||
|
|
||
| DRAFT_MODEL="Inferact/MiniMax-M3-EAGLE3-GQA" | ||
| NUM_SPEC_TOKENS=3 | ||
| SYNTHETIC_ACCEPT_LEN=2.78 | ||
|
|
||
| if [[ -n "$SLURM_JOB_ID" ]]; then | ||
| echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" | ||
| fi | ||
| if [[ -n "$ROCR_VISIBLE_DEVICES" ]]; then | ||
| export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" | ||
| fi | ||
|
|
||
| if [[ -n "$MODEL_PATH" ]]; then | ||
| if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then | ||
| hf download "$MODEL" --local-dir "$MODEL_PATH" | ||
| fi | ||
| else | ||
| hf download "$MODEL" | ||
| export MODEL_PATH="$MODEL" | ||
| fi | ||
| hf download "$DRAFT_MODEL" | ||
|
|
||
| rocm-smi || true | ||
| amd-smi || true | ||
| resolve_trace_source | ||
| install_agentic_deps | ||
|
|
||
| export AIPERF_SERVER_METRICS_URLS="http://localhost:${PORT}/metrics" | ||
| export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" | ||
|
|
||
| SERVER_LOG="$RESULT_DIR/server.log" | ||
| LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" | ||
| mkdir -p "$RESULT_DIR" | ||
|
|
||
| SERVER_PID="" | ||
| LMCACHE_PIDS=() | ||
| cleanup_agentic_services() { | ||
| local exit_code=$? | ||
| trap - EXIT INT TERM | ||
| set +e | ||
| stop_background_process_tree "$SERVER_PID" "vLLM server" 60 | ||
| local i | ||
| for i in "${!LMCACHE_PIDS[@]}"; do | ||
| stop_background_process_tree "${LMCACHE_PIDS[$i]}" "LMCache server $i" | ||
| done | ||
| exit "$exit_code" | ||
| } | ||
| trap cleanup_agentic_services EXIT | ||
| trap 'exit 130' INT | ||
| trap 'exit 143' TERM | ||
|
|
||
| OFFLOAD_ARGS=() | ||
| case "$KV_OFFLOAD_BACKEND" in | ||
| "") | ||
| require_agentic_kv_offload_none | ||
| ;; | ||
| lmcache) | ||
| require_agentic_kv_offload_backend lmcache | ||
| LMCACHE_VERSION="0.5.3" | ||
| LMCACHE_ROCM_INDEX="https://github.com/LMCache/LMCache/releases/expanded_assets/v${LMCACHE_VERSION}-rocm" | ||
| agentic_pip_install --quiet --no-cache-dir --no-deps \ | ||
| "sortedcontainers==2.4.0" \ | ||
| "opentelemetry-exporter-prometheus==0.61b0" \ | ||
| "cupy-rocm-7-0==14.1.1" \ | ||
| "lmcache==${LMCACHE_VERSION}" --find-links "$LMCACHE_ROCM_INDEX" | ||
| python3 -c \ | ||
| "import cupy; import lmcache.integration.vllm.lmcache_mp_connector; import opentelemetry.exporter.prometheus" \ | ||
| >/dev/null | ||
|
|
||
|
Comment on lines
+62
to
+80
Contributor
There was a problem hiding this comment. 🟡 This LMCache MP shard spin-up block (lines 62-128) is a near-verbatim duplicate (~55 lines) of the one in Extended reasoning...What's duplicated: Lines 62-128 of |
||
| LMCACHE_L1_SHARD_GB=$((TOTAL_CPU_DRAM_GB / TP)) | ||
| if [ "$LMCACHE_L1_SHARD_GB" -lt 1 ]; then | ||
| echo "Error: LMCache DRAM budget is less than 1 GB per TP rank." >&2 | ||
| exit 1 | ||
| fi | ||
|
|
||
| LMCACHE_SERVER_URLS=() | ||
| LMCACHE_HTTP_PORTS=() | ||
| LMCACHE_LOGS=() | ||
| : > "$RESULT_DIR/lmcache_command.txt" | ||
| for shard in $(seq 0 $((TP - 1))); do | ||
| shard_port=$((5555 + shard)) | ||
| shard_http_port=$((8080 + shard)) | ||
| shard_log="${LMCACHE_LOG%.log}_${shard}.log" | ||
| LMCACHE_CMD=( | ||
| lmcache server | ||
| --host 127.0.0.1 | ||
| --port "$shard_port" | ||
| --http-host 127.0.0.1 | ||
| --http-port "$shard_http_port" | ||
| --l1-size-gb "$LMCACHE_L1_SHARD_GB" | ||
| --l1-init-size-gb 10 | ||
| --l1-read-ttl-seconds 7200 | ||
| --chunk-size 256 | ||
| --max-workers 2 | ||
| --eviction-policy LRU | ||
| --supported-transfer-mode lmcache_driven | ||
| ) | ||
| append_command "$RESULT_DIR/lmcache_command.txt" "${LMCACHE_CMD[@]}" | ||
| "${LMCACHE_CMD[@]}" > "$shard_log" 2>&1 & | ||
| LMCACHE_PIDS+=($!) | ||
| LMCACHE_HTTP_PORTS+=("$shard_http_port") | ||
| LMCACHE_LOGS+=("$shard_log") | ||
| LMCACHE_SERVER_URLS+=("tcp://127.0.0.1:${shard_port}") | ||
| done | ||
| for shard in "${!LMCACHE_PIDS[@]}"; do | ||
| wait_for_ready \ | ||
| --endpoint "http://127.0.0.1:${LMCACHE_HTTP_PORTS[$shard]}/healthcheck" \ | ||
| --log "${LMCACHE_LOGS[$shard]}" \ | ||
| --pid "${LMCACHE_PIDS[$shard]}" \ | ||
| --sleep-interval 1 \ | ||
| --timeout 600 | ||
| done | ||
| LMCACHE_SERVER_URLS_CSV=$(IFS=,; echo "${LMCACHE_SERVER_URLS[*]}") | ||
| OFFLOAD_ARGS=( | ||
| --kv-transfer-config | ||
| "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.server_urls\":\"$LMCACHE_SERVER_URLS_CSV\",\"lmcache.mp.mq_timeout\":6000.0}}" | ||
| ) | ||
| ;; | ||
| *) | ||
| echo "Unsupported KV_OFFLOAD_BACKEND: $KV_OFFLOAD_BACKEND" >&2 | ||
| exit 1 | ||
| ;; | ||
| esac | ||
|
|
||
| PARALLEL_ARGS=(--tensor-parallel-size "$TP") | ||
| if [ "$EP_SIZE" -gt 1 ]; then | ||
| PARALLEL_ARGS+=(--enable-expert-parallel) | ||
| fi | ||
|
|
||
| if [ "$EVAL_ONLY" = "true" ]; then | ||
| SPEC_CONFIG="{\"method\":\"eagle3\",\"model\":\"$DRAFT_MODEL\",\"num_speculative_tokens\":$NUM_SPEC_TOKENS,\"attention_backend\":\"TRITON_ATTN\"}" | ||
| else | ||
| SPEC_CONFIG="{\"method\":\"eagle3\",\"model\":\"$DRAFT_MODEL\",\"num_speculative_tokens\":$NUM_SPEC_TOKENS,\"attention_backend\":\"TRITON_ATTN\",\"rejection_sample_method\":\"synthetic\",\"synthetic_acceptance_length\":$SYNTHETIC_ACCEPT_LEN}" | ||
| fi | ||
|
|
||
| export PYTHONNOUSERSITE=1 | ||
| export VLLM_ENGINE_READY_TIMEOUT_S=3600 | ||
| export VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800 | ||
| export VLLM_USE_BREAKABLE_CUDAGRAPH=0 | ||
| export VLLM_ROCM_USE_AITER=1 | ||
| export VLLM_ROCM_USE_AITER_MHA=0 | ||
| export TORCH_BLAS_PREFER_HIPBLASLT=1 | ||
| export NCCL_MIN_NCHANNELS=112 | ||
| export GPU_MAX_HW_QUEUES=2 | ||
|
|
||
| VLLM_CMD=( | ||
| vllm serve "$MODEL_PATH" | ||
| --served-model-name "$MODEL" | ||
| --host 0.0.0.0 | ||
| --port "$PORT" | ||
| "${PARALLEL_ARGS[@]}" | ||
| --trust-remote-code | ||
| --block-size 128 | ||
| --gpu-memory-utilization 0.90 | ||
| --enable-chunked-prefill | ||
| --max-num-batched-tokens 16384 | ||
| --language-model-only | ||
| --enable-prefix-caching | ||
| --attention-backend TRITON_ATTN | ||
| --kv-cache-dtype fp8 | ||
| --tool-call-parser minimax_m3 | ||
| --reasoning-parser minimax_m3 | ||
| --enable-auto-tool-choice | ||
| --default-chat-template-kwargs '{"thinking_mode":"enabled"}' | ||
| --max-num-seqs "$((2 * CONC))" | ||
| --stream-interval 20 | ||
| --speculative-config "$SPEC_CONFIG" | ||
| "${OFFLOAD_ARGS[@]}" | ||
| ) | ||
| write_command "$RESULT_DIR/server_command.txt" "${VLLM_CMD[@]}" | ||
| "${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & | ||
| SERVER_PID=$! | ||
|
|
||
| wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" | ||
|
|
||
| if [ "$EVAL_ONLY" = "true" ]; then | ||
| run_eval --port "$PORT" | ||
| else | ||
| build_replay_cmd "$RESULT_DIR" | ||
| REPLAY_CMD+=" --apply-chat-template" | ||
| run_agentic_replay_and_write_outputs "$RESULT_DIR" | ||
| fi | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🟡 The new
minimaxm3_fp8_mi300x_mtp.shrequiresDP_ATTENTIONviacheck_env_vars(line 11) but never branches on it —PARALLEL_ARGSonly checksEP_SIZE. It's harmless today since every row inminimaxm3-fp8-mi300x-vllm-agentic-mtpdefaultsdp-attnto false, but sibling recipes with the same required var (e.g.minimaxm3_fp8_mi300x.sh,dsv4_fp4_mi355x_vllm_mtp.sh) do consume it to build DP-attention flags, so if a futuredp-attn: truerow is added here (as already exists for the sibling mi325x recipe), the server would silently launch plain TP/EP while the sweep still labels it DEP.Extended reasoning...
The bug:
benchmarks/single_node/agentic/minimaxm3_fp8_mi300x_mtp.sh:11listsDP_ATTENTIONin itscheck_env_varscall, asserting the script depends on that variable. But grepping the rest of the script shows it is never referenced again —PARALLEL_ARGS(around theEP_SIZE-gated--enable-expert-parallelblock) only branches onEP_SIZE. There is no analogous branch that would add--data-parallel-size, bump the port, or start a router whenDP_ATTENTION=true.Contrast with siblings that do consume it:
minimaxm3_fp8_mi300x.sh(lines 150/160/188),minimaxm3_fp8_mi325x.sh(150/165/198), anddsv4_fp4_mi355x_vllm_mtp.sh(70/365) all require the same variable and use it to switch between plain TP/EP and DP-attention serving (different parallel args, a different port, and launchingvllm-router). Those scripts show what 'consuming'DP_ATTENTIONis supposed to look like in this codebase, and the new MTP script has the requirement without the corresponding behavior.Why this doesn't currently break anything: every search-space row under
minimaxm3-fp8-mi300x-vllm-agentic-mtpinconfigs/amd-master.yamlomitsdp-attn, whichgenerate_sweep_configs.pydefaults tofalse. WithDP_ATTENTION=false, the correct behavior (plain TP/EP) is exactly what the script does anyway, so no current run is mislabeled.Why it's still worth flagging (as a nit, not a blocker): it's a latent footgun rather than an active bug.
dp-attn: truerows already exist for sibling recipes in the very sameamd-master.yamlfile (e.g.minimaxm3-fp8-mi325x-vllm-agentic,dsv4-fp4-mi355x-vllm-agentic-mtp). If a future row addsdp-attn: trueto this recipe, the sweep would setDP_ATTENTION=trueand record the point as a DEP config, but the script would silently serve plain TP/EP — corrupting the benchmark's performance-curve labeling with no error surfaced anywhere in the pipeline.Step-by-step proof: (1) A maintainer adds
{ tp: 8, ep: 8, dp-attn: true, ... }tominimaxm3-fp8-mi300x-vllm-agentic-mtpinamd-master.yaml, mirroring the pattern already used forminimaxm3-fp8-mi325x-vllm-agentic. (2)generate_sweep_configs.pyemits a job withDP_ATTENTION=true. (3)check_env_varspasses since the variable is set. (4) The script buildsPARALLEL_ARGS=(--tensor-parallel-size "")plus--enable-expert-parallelfromEP_SIZEalone —DP_ATTENTIONis never read, so no--data-parallel-sizeflag is added and no router is started. (5) The server runs as plain TEP8, but the result is recorded and labeled as a DEP8 data point in the benchmark output, silently corrupting the recorded performance curve.On the refutation: one verifier correctly points out that this exact required-but-unconsumed pattern already exists in
glm5.2_fp8_mi325x_mtp.sh:11andminimaxm3_fp4_mi355x_mtp.sh:16(confirmed by inspection — both requireDP_ATTENTIONviacheck_env_varsand never branch on it, the latter only echoing it for logging). So this specific script isn't introducing a wholly novel anti-pattern into the repo, and removingDP_ATTENTIONfrom the required-var list would arguably regress the intended uniform env-contract assertion across the agentic family. That's a fair point, and it's why this should stay a nit rather than escalate to normal — but it doesn't fully neutralize the finding: several other siblings in the same model family (minimaxm3_fp8_mi300x.sh,minimaxm3_fp8_mi325x.sh) do wireDP_ATTENTIONinto real behavior, and this recipe's ownamd-master.yamlneighbors already shipdp-attn: truerows for other configs. The inconsistency is real even if the specific failure mode is speculative and not triggered by this PR's search space.Suggested fix: either wire
DP_ATTENTIONintoPARALLEL_ARGS(mirroringminimaxm3_fp8_mi300x.sh) so a futuredp-attn: truerow behaves correctly, or drop it fromcheck_env_varsif DP-attention is intentionally out of scope for this MTP recipe. Either fix removes the silent-mislabeling risk.