diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh index 1b9311eda..20d5f6280 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh @@ -371,10 +371,22 @@ if [ "$EP_SIZE" -gt 1 ]; then EP_ARGS=(--enable-expert-parallel) fi +DP_SCHED_ARGS=() +if [ "$DP_ATTENTION" = "true" ]; then + DP_SCHED_ARGS=( + --prefill-schedule-interval 8 + --max-num-batched-tokens 8192 + --long-prefill-token-threshold 16384 + ) +fi + # AgentX concurrency counts live session trees, not individual requests. # Subagent fan-out can push instantaneous request concurrency above CONC, so # leave 2x headroom rather than clipping those bursts at the scheduler. MAX_NUM_SEQS=$((2 * CONC)) +if [ "$DP_ATTENTION" = "true" ]; then + MAX_NUM_SEQS="$CONC" +fi # DeepSeek-V4-Pro ships a native MTP head. AgentX throughput pins its # three-token draft to the committed thinking-on golden acceptance length; @@ -392,6 +404,7 @@ set -x export VLLM_ROCM_USE_AITER=1 #export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 export VLLM_ROCM_USE_AITER_MOE=1 +export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 sleep 180 @@ -406,7 +419,8 @@ VLLM_CMD=( --kv-cache-dtype fp8 "${PARALLEL_ARGS[@]}" "${EP_ARGS[@]}" - --gpu-memory-utilization 0.8 + "${DP_SCHED_ARGS[@]}" + --gpu-memory-utilization 0.86 --moe-backend aiter --compilation-config '{"mode":3,"cudagraph_mode":"FULL_AND_PIECEWISE"}' --speculative-config "$SPEC_CONFIG" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index c94029484..b406168e3 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1301,7 +1301,7 @@ qwen3.5-fp8-mi355x-sglang-agentic-hicache: # is no longer available from Docker Hub. Throughput uses the three-token golden # synthetic acceptance length, while eval-only runs retain real verification. dsv4-fp4-mi355x-vllm-agentic-mtp: - image: vllm/vllm-openai-rocm:nightly-821717118fc26667dd474b9b0ab81d29259dfc5c + image: vllm/vllm-openai-rocm:nightly-f8d03e77416bf90c49acbe50e233275722f02c4b model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds @@ -1313,7 +1313,7 @@ dsv4-fp4-mi355x-vllm-agentic-mtp: - dram-utilization: 0.60 search-space: - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16, 32, 40, 48] } - - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 1, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [32, 48, 64, 96], router: { name: vllm-router, version: "0.1.14" } } # LMCache invalid-block recovery currently assumes one KV-cache group, # while MTP creates two. Restore these points after the upstream hybrid # KV recovery fix lands: https://github.com/vllm-project/vllm/pull/45497 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6082f2b86..c0b13d2af 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5918,3 +5918,18 @@ - "Add a TEP2 arm (tp 2, ep 2) to the qwen3.5-fp4-b200-sglang-mtp 8k/1k sweep at concurrency 16, 32, and 64" - "Rides on the NVFP4-V2 checkpoint switch from #2205" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2550 + +- config-keys: + - dsv4-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Repin the image to the 08-09 nightly (f8d03e77) so the config can start at all: the previously pinned nightly no longer serves this recipe, and the 08-12 nightly (3ee2df30) memory-faults during the profile run -- eight TP workers serialize on /tmp/aiter_configs/bf16_tuned_gemm.csv.lock and the worker holding the baton dies in the post-MoE bf16 GEMM" + - "Extend the dp-attn arm (tp 8, ep 1, dp-attn true) from a single concurrency 64 point to 32, 48, 64, and 96. dp-attn true maps to --tensor-parallel-size 1 --data-parallel-size 8, so each rank holds a full copy of the weights; one scheduler per rank keeps the per-rank batch small where the single TP8 scheduler collapses, so this arm carries the high-concurrency half of the curve. Concurrency 64 overlaps the previously published point" + - "Keep pure TP8 at 1, 4, 8, 16, 32, 40, and 48. Measured on this base it peaks at 48 (7,259.7 tok/s/GPU) and falls off a cliff past it -- 2,372.4 at 64 and 1,609.3 at 96 -- so the ladder stops at the knee rather than spending runner hours below it" + - "No EP8 arm. EP8 only pays off on this model with the MegaMoE backend, which needs mori.ir.flydsl -- absent from the pinned image. On the aiter backend this config uses, EP8 has no measured advantage over the two arms above" + - "Cap --max-num-seqs at CONC rather than 2*CONC under DP-attention. The limit is per scheduler and DP-attention runs one scheduler per rank, so the existing 2x headroom admitted 16x the intended batch across the eight ranks" + - "Set --max-num-batched-tokens 8192 with --prefill-schedule-interval 8 and --long-prefill-token-threshold 16384 under DP-attention. 16384 measured 23% below 8192 at concurrency 96 (9,424.1 against 12,244.0 tok/s/GPU); that comparison was taken on the expert-parallel MegaMoE topology rather than on this arm, so it motivates the value without establishing it here and the sweep measures it" + - "Export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1. DSv4-Pro is a mixed checkpoint (MXFP4 routed experts, FP8 shared expert) and vllm/models/deepseek_v4/amd/model.py gates the fused shared-expert path on this flag; the aiter side is aiter/fused_moe_dp_shared_expert.py, present on this base. The flag defaults to False, so the checked-in recipe was not running the configuration every validated manual run of this recipe used. Note it is mutually exclusive with expert parallelism -- _fuse_shared_experts_enabled() returns False when enable_expert_parallel is set -- which is consistent with both arms here running ep 1" + - "Raise --gpu-memory-utilization from 0.8 to 0.86. This base predates vllm #51473, so the mxfp4 oracle rounds inter_dim 384 up to 512 and the weights inflate; under this topology on the previously pinned nightly, also pre-#51473, weight residency measured 160.66 GiB per rank and left only 34.17 GiB of KV at 0.86. Utilization is the only lever in the recipe that offsets any of that" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2590