From f10fcb68b2065ec0661b2a2cd1d685e3e7cfb36b Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 11 Aug 2026 16:56:54 -0500 Subject: [PATCH 01/11] Add GB200 DeepSeek V4 vLLM MTP AgentX sweep --- .../agentic/agg-gb200-dep8-mtp-agentic.yaml | 96 ++++++++++++++++ .../agentic/agg-gb200-tp8-mtp-agentic.yaml | 105 +++++++++++++++++ ...sagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml | 107 ++++++++++++++++++ ...sagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml | 102 +++++++++++++++++ configs/nvidia-master.yaml | 99 ++++++++++++++++ runners/launch_gb200-nv.sh | 17 ++- 6 files changed, 524 insertions(+), 2 deletions(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml new file mode 100644 index 0000000000..f9246df4cd --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml @@ -0,0 +1,96 @@ +name: dsv4-gb200-vllm-agentic-mtp-agg-dep8 + +model: + path: deepseek-v4-pro + container: vllm/vllm-openai:v0.27.1 + precision: fp4 + +identity: + model: { repo: deepseek-ai/DeepSeek-V4-Pro } + container: { image: vllm/vllm-openai:v0.27.1 } + frameworks: { dynamo: "1.3.1" } + +dynamo: + version: "1.3.1" + install: true + +setup_script: vllm-container-deps.sh +slurm: { time_limit: "8:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } + +resources: + gpu_type: gb200 + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: random + router-session-affinity-ttl-secs: 900 + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + aggregated_environment: + HF_HUB_CACHE: /hf_hub_cache + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_SERVER_DEV_MODE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + vllm_config: + aggregated: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 256 + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + gpu-memory-utilization: 0.90 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml new file mode 100644 index 0000000000..383b008c52 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml @@ -0,0 +1,105 @@ +name: dsv4-gb200-vllm-agentic-mtp-agg-tp8 + +model: + path: deepseek-v4-pro + container: vllm/vllm-openai:v0.27.1 + precision: fp4 + +identity: + model: + repo: deepseek-ai/DeepSeek-V4-Pro + container: + image: vllm/vllm-openai:v0.27.1 + frameworks: + dynamo: "1.3.1" + +dynamo: + version: "1.3.1" + install: true + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: gb200 + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: random + router-session-affinity-ttl-secs: 900 + +backend: + type: vllm + connector: null + aggregated_environment: + HF_HUB_CACHE: /hf_hub_cache + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_SERVER_DEV_MODE: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + vllm_config: + aggregated: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + kv-cache-dtype: fp8 + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 128 + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + gpu-memory-utilization: 0.90 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml new file mode 100644 index 0000000000..9b0a2c833d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml @@ -0,0 +1,107 @@ +name: dsv4-gb200-vllm-agentic-mtp-disagg-1p1d-dep8-dep8 + +model: + path: deepseek-v4-pro + container: vllm/vllm-openai:v0.27.1 + precision: fp4 + +identity: + model: { repo: deepseek-ai/DeepSeek-V4-Pro } + container: { image: vllm/vllm-openai:v0.27.1 } + frameworks: { dynamo: "1.3.1" } + +dynamo: + version: "1.3.1" + install: true + +setup_script: vllm-container-deps.sh +slurm: { time_limit: "8:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } + +resources: + gpu_type: gb200 + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: random + router-session-affinity-ttl-secs: 900 + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + prefill_environment: &worker_environment + HF_HUB_CACHE: /hf_hub_cache + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: *worker_environment + vllm_config: + prefill: &dep8_config + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + served-model-name: deepseek-ai/DeepSeek-V4-Pro + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + decode: + <<: *dep8_config + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_fp4_indexer_cache":true}' + max-num-seqs: 512 + max-num-batched-tokens: 1024 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 512 + stream-interval: 10 + +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml new file mode 100644 index 0000000000..38ec72fc10 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml @@ -0,0 +1,102 @@ +name: dsv4-gb200-vllm-agentic-mtp-disagg-2p1d-dep8-dep8 + +model: + path: deepseek-v4-pro + container: vllm/vllm-openai:v0.27.1 + precision: fp4 + +identity: + model: { repo: deepseek-ai/DeepSeek-V4-Pro } + container: { image: vllm/vllm-openai:v0.27.1 } + frameworks: { dynamo: "1.3.1" } + +dynamo: { version: "1.3.1", install: true } +setup_script: vllm-container-deps.sh +slurm: { time_limit: "8:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } + +resources: + gpu_type: gb200 + gpus_per_node: 4 + prefill_nodes: 4 + decode_nodes: 2 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: random + router-session-affinity-ttl-secs: 900 + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + prefill_environment: &worker_environment + HF_HUB_CACHE: /hf_hub_cache + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: *worker_environment + vllm_config: + prefill: &dep8_config + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + served-model-name: deepseek-ai/DeepSeek-V4-Pro + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + decode: + <<: *dep8_config + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_fp4_indexer_cache":true}' + max-num-seqs: 512 + max-num-batched-tokens: 1024 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 512 + stream-interval: 10 + +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a376a9a16b..fda9cb3a1c 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7330,6 +7330,105 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep8: ep: 8 dp-attn: true +dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: + image: vllm/vllm-openai:v0.27.1 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.1" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - search-space: + # The checkpoint needs two GB200 NVL4 trays. TP8 anchors latency; DEP8 + # follows the public vLLM recommendation and probes the throughput curve. + - spec-decoding: mtp + conc-list: [1, 2, 4, 8] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [4, 8, 16, 24, 32, 48, 64, 96] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 8 + dp-attn: true + +dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: + image: vllm/vllm-openai:v0.27.1 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + # Public-recipe baseline: one two-tray DEP8 prefill worker and one + # two-tray DEP8 decode worker. Dense sampling finds the MTP knee. + - spec-decoding: mtp + conc-list: [32, 64, 96, 128, 192, 256, 384] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # Add prefill capacity only after the balanced baseline reaches its + # high-concurrency region; this separates prefill from decode scaling. + - spec-decoding: mtp + conc-list: [192, 256, 384, 512] + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f model: deepseek-ai/DeepSeek-V4-Pro diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 34d7c4c806..b8d3525ce2 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -388,8 +388,21 @@ if [ -d "$SRT_REPO_DIR" ]; then rm -rf "$SRT_REPO_DIR" fi -# TODO(CJQ): make first class upon srt-slurm upstream refactor -if [[ "$IS_AGENTIC" == "1" ]]; then +# Agentic DeepSeek-V4 uses the latest released srt-slurm. v1.0.45 carries +# custom-benchmark support plus ordered logical-worker Prometheus endpoint +# injection, so disaggregated AIPerf runs receive every P/D metrics URL. +if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "dsv4" && "$FRAMEWORK" == "dynamo-vllm" ]]; then + git clone --branch v1.0.45 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + test "$(git rev-parse HEAD)" = "9d8d92b20c350a5d42f0709f5a0b64e30eb37d33" || { + echo "Error: NVIDIA/srt-slurm v1.0.45 resolved to an unexpected commit" >&2 + exit 1 + } + mkdir -p recipes/vllm/deepseek-v4/agentic + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ + recipes/vllm/deepseek-v4/agentic +# TODO(CJQ): migrate the remaining Agentic model paths to released srt-slurm. +elif [[ "$IS_AGENTIC" == "1" ]]; then # Agentic multi-node pins cquil11/srt-slurm-nv revisions that provide: # - BenchmarkType.CUSTOM + benchmark.command + benchmark.env # (the hook that hands off to benchmarks/multi_node/agentic_srt.sh) From f2845546ff17aa0dc60f473ba2658fcd3dff17da Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 11 Aug 2026 16:58:06 -0500 Subject: [PATCH 02/11] Document GB200 DeepSeek V4 AgentX submission --- perf-changelog.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7ae8479441..e92865ff9c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5794,3 +5794,15 @@ - "Keep DSpark K=2 probabilistic throughput on synthetic golden AL 2.51 while the generated EVAL_ONLY row uses real block verification" - "Cap the GPU-resident search at concurrency 8 after concurrency 16 failed to complete deterministic warmup; retain concurrency 16 for the DRAM-offload capacity tier" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2476 + +- config-keys: + - dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg + - dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Add GB200 DeepSeek-V4-Pro FP4 Dynamo-vLLM MTP AgentX coverage across TP8, DEP8, 1P1D, and 2P1D discovery arms." + - "Use released vLLM v0.27.1, Dynamo 1.3.1, and NVIDIA/srt-slurm v1.0.45." + - "Collect every logical vLLM prefill/decode leader endpoint and require nonempty vllm: server metrics before publishing results." + - "Use K=3 golden synthetic acceptance length 2.49 for throughput; eval retains real MTP verification." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2567 From 6c2df11234932e26e1faa329a04fceb56bc93cee Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 11 Aug 2026 20:01:27 -0500 Subject: [PATCH 03/11] Align GB200 DeepSeek V4 with vLLM recipe --- .../agentic/agg-gb200-dep8-mtp-agentic.yaml | 8 ++++---- .../agentic/agg-gb200-tp8-mtp-agentic.yaml | 6 ++---- ...sagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml | 20 ++++++++++--------- ...sagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml | 20 ++++++++++--------- configs/nvidia-master.yaml | 8 ++++---- perf-changelog.yaml | 2 +- 6 files changed, 33 insertions(+), 31 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml index f9246df4cd..8462e90547 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml @@ -62,16 +62,16 @@ backend: data-parallel-size: 8 enable-expert-parallel: true enable-ep-weight-filter: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: deep_gemm_mega_moe + attention-config: '{"use_fp4_indexer_cache":true}' max-model-len: 1048576 max-num-seqs: 256 max-num-batched-tokens: 8192 trust-remote-code: true - no-enable-flashinfer-autotune: true block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' max-cudagraph-capture-size: 256 - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' gpu-memory-utilization: 0.90 stream-interval: 10 no-disable-hybrid-kv-cache-manager: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml index 383b008c52..44da87c803 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml @@ -67,16 +67,14 @@ backend: tensor-parallel-size: 8 pipeline-parallel-size: 1 disable-custom-all-reduce: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + attention-config: '{"use_fp4_indexer_cache":true}' max-model-len: 1048576 max-num-seqs: 128 max-num-batched-tokens: 8192 trust-remote-code: true no-enable-flashinfer-autotune: true block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 128 - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' gpu-memory-utilization: 0.90 stream-interval: 10 no-disable-hybrid-kv-cache-manager: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml index 9b0a2c833d..75568eb019 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml @@ -68,24 +68,26 @@ backend: data-parallel-size: 8 enable-expert-parallel: true enable-ep-weight-filter: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: deep_gemm_mega_moe + attention-config: '{"use_fp4_indexer_cache":true}' max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 + max-num-seqs: 16 + max-num-batched-tokens: 16384 trust-remote-code: true - no-enable-flashinfer-autotune: true + enforce-eager: true block-size: 256 gpu-memory-utilization: 0.90 no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true tokenizer-mode: deepseek_v4 - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' decode: <<: *dep8_config - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_fp4_indexer_cache":true}' - max-num-seqs: 512 + enforce-eager: false + max-num-seqs: 1024 max-num-batched-tokens: 1024 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 512 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + max-cudagraph-capture-size: 1024 stream-interval: 10 sbatch_directives: { cpus-per-task: "144", mem: "0" } diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml index 38ec72fc10..5e1daf5131 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml @@ -63,24 +63,26 @@ backend: data-parallel-size: 8 enable-expert-parallel: true enable-ep-weight-filter: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + moe-backend: deep_gemm_mega_moe + attention-config: '{"use_fp4_indexer_cache":true}' max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 + max-num-seqs: 16 + max-num-batched-tokens: 16384 trust-remote-code: true - no-enable-flashinfer-autotune: true + enforce-eager: true block-size: 256 gpu-memory-utilization: 0.90 no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true tokenizer-mode: deepseek_v4 - speculative-config: '{"method":"mtp","num_speculative_tokens":3}' + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' decode: <<: *dep8_config - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_fp4_indexer_cache":true}' - max-num-seqs: 512 + enforce-eager: false + max-num-seqs: 1024 max-num-batched-tokens: 1024 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 512 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + max-cudagraph-capture-size: 1024 stream-interval: 10 sbatch_directives: { cpus-per-task: "144", mem: "0" } diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index fda9cb3a1c..5fde63c079 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7346,7 +7346,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: # The checkpoint needs two GB200 NVL4 trays. TP8 anchors latency; DEP8 # follows the public vLLM recommendation and probes the throughput curve. - spec-decoding: mtp - conc-list: [1, 2, 4, 8] + conc-list: [1] prefill: num-worker: 1 tp: 8 @@ -7362,7 +7362,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: ep: 1 dp-attn: false - spec-decoding: mtp - conc-list: [4, 8, 16, 24, 32, 48, 64, 96] + conc-list: [16] prefill: num-worker: 1 tp: 8 @@ -7395,7 +7395,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: # Public-recipe baseline: one two-tray DEP8 prefill worker and one # two-tray DEP8 decode worker. Dense sampling finds the MTP knee. - spec-decoding: mtp - conc-list: [32, 64, 96, 128, 192, 256, 384] + conc-list: [128] prefill: num-worker: 1 tp: 8 @@ -7413,7 +7413,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: # Add prefill capacity only after the balanced baseline reaches its # high-concurrency region; this separates prefill from decode scaling. - spec-decoding: mtp - conc-list: [192, 256, 384, 512] + conc-list: [384] prefill: num-worker: 2 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e92865ff9c..6356dd820b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5804,5 +5804,5 @@ - "Add GB200 DeepSeek-V4-Pro FP4 Dynamo-vLLM MTP AgentX coverage across TP8, DEP8, 1P1D, and 2P1D discovery arms." - "Use released vLLM v0.27.1, Dynamo 1.3.1, and NVIDIA/srt-slurm v1.0.45." - "Collect every logical vLLM prefill/decode leader endpoint and require nonempty vllm: server metrics before publishing results." - - "Use K=3 golden synthetic acceptance length 2.49 for throughput; eval retains real MTP verification." + - "Use the official K=2 MTP configuration with golden synthetic acceptance length 2.49 for throughput; eval retains real MTP verification." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2567 From 4c98ed76aff033cc6bae229ecab94f07136a7407 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 11 Aug 2026 23:12:33 -0500 Subject: [PATCH 04/11] Fix GB200 DeepSeek V4 MTP startup paths --- .../deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml | 3 ++- .../vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml | 5 ++++- .../agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml | 6 ++++-- .../agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml | 6 ++++-- perf-changelog.yaml | 3 +++ 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml index 8462e90547..9eeb705b9e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml @@ -15,6 +15,7 @@ dynamo: install: true setup_script: vllm-container-deps.sh +environment: { ETCD_LEASE_TTL: "7200" } slurm: { time_limit: "8:00:00" } health_check: { max_attempts: 2160, interval_seconds: 10 } @@ -63,7 +64,7 @@ backend: enable-expert-parallel: true enable-ep-weight-filter: true moe-backend: deep_gemm_mega_moe - attention-config: '{"use_fp4_indexer_cache":true}' + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' max-model-len: 1048576 max-num-seqs: 256 max-num-batched-tokens: 8192 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml index 44da87c803..7296be0cd9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml @@ -19,6 +19,9 @@ dynamo: setup_script: vllm-container-deps.sh +environment: + ETCD_LEASE_TTL: "7200" + slurm: time_limit: "8:00:00" @@ -67,7 +70,7 @@ backend: tensor-parallel-size: 8 pipeline-parallel-size: 1 disable-custom-all-reduce: true - attention-config: '{"use_fp4_indexer_cache":true}' + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' max-model-len: 1048576 max-num-seqs: 128 max-num-batched-tokens: 8192 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml index 75568eb019..9f7c3aeec8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml @@ -15,6 +15,7 @@ dynamo: install: true setup_script: vllm-container-deps.sh +environment: { ETCD_LEASE_TTL: "7200" } slurm: { time_limit: "8:00:00" } health_check: { max_attempts: 2160, interval_seconds: 10 } @@ -69,20 +70,21 @@ backend: enable-expert-parallel: true enable-ep-weight-filter: true moe-backend: deep_gemm_mega_moe - attention-config: '{"use_fp4_indexer_cache":true}' + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' max-model-len: 1048576 max-num-seqs: 16 max-num-batched-tokens: 16384 trust-remote-code: true enforce-eager: true block-size: 256 - gpu-memory-utilization: 0.90 + gpu-memory-utilization: 0.95 no-disable-hybrid-kv-cache-manager: true enable-sleep-mode: true tokenizer-mode: deepseek_v4 speculative-config: '{"method":"mtp","num_speculative_tokens":2}' decode: <<: *dep8_config + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_fp4_indexer_cache":true}' enforce-eager: false max-num-seqs: 1024 max-num-batched-tokens: 1024 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml index 5e1daf5131..620957b50c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml @@ -12,6 +12,7 @@ identity: dynamo: { version: "1.3.1", install: true } setup_script: vllm-container-deps.sh +environment: { ETCD_LEASE_TTL: "7200" } slurm: { time_limit: "8:00:00" } health_check: { max_attempts: 2160, interval_seconds: 10 } @@ -64,20 +65,21 @@ backend: enable-expert-parallel: true enable-ep-weight-filter: true moe-backend: deep_gemm_mega_moe - attention-config: '{"use_fp4_indexer_cache":true}' + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' max-model-len: 1048576 max-num-seqs: 16 max-num-batched-tokens: 16384 trust-remote-code: true enforce-eager: true block-size: 256 - gpu-memory-utilization: 0.90 + gpu-memory-utilization: 0.95 no-disable-hybrid-kv-cache-manager: true enable-sleep-mode: true tokenizer-mode: deepseek_v4 speculative-config: '{"method":"mtp","num_speculative_tokens":2}' decode: <<: *dep8_config + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_fp4_indexer_cache":true}' enforce-eager: false max-num-seqs: 1024 max-num-batched-tokens: 1024 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 246bde0df6..afc4a57541 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5829,4 +5829,7 @@ - "Use released vLLM v0.27.1, Dynamo 1.3.1, and NVIDIA/srt-slurm v1.0.45." - "Collect every logical vLLM prefill/decode leader endpoint and require nonempty vllm: server metrics before publishing results." - "Use the official K=2 MTP configuration with golden synthetic acceptance length 2.49 for throughput; eval retains real MTP verification." + - "Use the established 0.95 GB200 disaggregated GPU memory budget so K=2 MTP and NIXL retain enough KV memory for the native 1,048,576-token context." + - "Select the supported FlashInfer DeepSeek V4 sparse-MLA backend, avoiding the released FlashMLA long-prefill TMA regression while preserving FP4 indexer caching." + - "Use the established 7,200-second Dynamo lease budget so the long FP4 model load and graph compile cannot expire healthy frontend and backend registrations." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2567 From 0904351259180fd843d83c8d5406e297db0e117b Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Tue, 11 Aug 2026 23:47:41 -0500 Subject: [PATCH 05/11] fix DeepSeek V4 MTP graph capture scope --- .../vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml | 2 +- .../agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml | 2 +- .../agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml | 2 +- perf-changelog.yaml | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml index 9eeb705b9e..821542a1d9 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml @@ -70,7 +70,7 @@ backend: max-num-batched-tokens: 8192 trust-remote-code: true block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' max-cudagraph-capture-size: 256 speculative-config: '{"method":"mtp","num_speculative_tokens":2}' gpu-memory-utilization: 0.90 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml index 9f7c3aeec8..c3dad304d3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml @@ -88,7 +88,7 @@ backend: enforce-eager: false max-num-seqs: 1024 max-num-batched-tokens: 1024 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' max-cudagraph-capture-size: 1024 stream-interval: 10 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml index 620957b50c..c035923efa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml @@ -83,7 +83,7 @@ backend: enforce-eager: false max-num-seqs: 1024 max-num-batched-tokens: 1024 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY"}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' max-cudagraph-capture-size: 1024 stream-interval: 10 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index afc4a57541..b9092c8437 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5831,5 +5831,6 @@ - "Use the official K=2 MTP configuration with golden synthetic acceptance length 2.49 for throughput; eval retains real MTP verification." - "Use the established 0.95 GB200 disaggregated GPU memory budget so K=2 MTP and NIXL retain enough KV memory for the native 1,048,576-token context." - "Select the supported FlashInfer DeepSeek V4 sparse-MLA backend, avoiding the released FlashMLA long-prefill TMA regression while preserving FP4 indexer caching." + - "Use the public DeepSeek V4 MTP full-decode-only graph mode so aggregate and disaggregated DEP do not capture incompatible sliding-window cache views." - "Use the established 7,200-second Dynamo lease budget so the long FP4 model load and graph compile cannot expire healthy frontend and backend registrations." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2567 From 033d9286e8221a6ace2885c6249aaab360c48939 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 02:08:08 -0500 Subject: [PATCH 06/11] fix(gb200): use current vLLM for DeepSeek V4 MTP --- .../vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml | 4 ++-- .../vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml | 4 ++-- .../agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml | 4 ++-- .../agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml | 4 ++-- configs/nvidia-master.yaml | 4 ++-- perf-changelog.yaml | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml index 821542a1d9..5fa2a163c4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml @@ -2,12 +2,12 @@ name: dsv4-gb200-vllm-agentic-mtp-agg-dep8 model: path: deepseek-v4-pro - container: vllm/vllm-openai:v0.27.1 + container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 precision: fp4 identity: model: { repo: deepseek-ai/DeepSeek-V4-Pro } - container: { image: vllm/vllm-openai:v0.27.1 } + container: { image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 } frameworks: { dynamo: "1.3.1" } dynamo: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml index 7296be0cd9..657b0c8e09 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml @@ -2,14 +2,14 @@ name: dsv4-gb200-vllm-agentic-mtp-agg-tp8 model: path: deepseek-v4-pro - container: vllm/vllm-openai:v0.27.1 + container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 precision: fp4 identity: model: repo: deepseek-ai/DeepSeek-V4-Pro container: - image: vllm/vllm-openai:v0.27.1 + image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 frameworks: dynamo: "1.3.1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml index c3dad304d3..8a0e3254dc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml @@ -2,12 +2,12 @@ name: dsv4-gb200-vllm-agentic-mtp-disagg-1p1d-dep8-dep8 model: path: deepseek-v4-pro - container: vllm/vllm-openai:v0.27.1 + container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 precision: fp4 identity: model: { repo: deepseek-ai/DeepSeek-V4-Pro } - container: { image: vllm/vllm-openai:v0.27.1 } + container: { image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 } frameworks: { dynamo: "1.3.1" } dynamo: diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml index c035923efa..5e40d50fdf 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml @@ -2,12 +2,12 @@ name: dsv4-gb200-vllm-agentic-mtp-disagg-2p1d-dep8-dep8 model: path: deepseek-v4-pro - container: vllm/vllm-openai:v0.27.1 + container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 precision: fp4 identity: model: { repo: deepseek-ai/DeepSeek-V4-Pro } - container: { image: vllm/vllm-openai:v0.27.1 } + container: { image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 } frameworks: { dynamo: "1.3.1" } dynamo: { version: "1.3.1", install: true } diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index a0a0a829d5..6b3047f956 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7366,7 +7366,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep8: dp-attn: true dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: - image: vllm/vllm-openai:v0.27.1 + image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:gb200-nv @@ -7414,7 +7414,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: dp-attn: true dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: - image: vllm/vllm-openai:v0.27.1 + image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:gb200-nv diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b9092c8437..fbe85842ef 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5826,7 +5826,7 @@ - agentic-coding description: - "Add GB200 DeepSeek-V4-Pro FP4 Dynamo-vLLM MTP AgentX coverage across TP8, DEP8, 1P1D, and 2P1D discovery arms." - - "Use released vLLM v0.27.1, Dynamo 1.3.1, and NVIDIA/srt-slurm v1.0.45." + - "Use immutable vLLM nightly commit 3ee2df3, Dynamo 1.3.1, and NVIDIA/srt-slurm v1.0.45; released v0.27.1 fails DeepSeek-V4 sparse-cache warmup on GB200." - "Collect every logical vLLM prefill/decode leader endpoint and require nonempty vllm: server metrics before publishing results." - "Use the official K=2 MTP configuration with golden synthetic acceptance length 2.49 for throughput; eval retains real MTP verification." - "Use the established 0.95 GB200 disaggregated GPU memory budget so K=2 MTP and NIXL retain enough KV memory for the native 1,048,576-token context." From 824b8571ca4987a7bd47cc8d825a1143822bde37 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 02:39:28 -0500 Subject: [PATCH 07/11] fix(gb200): reserve decode memory for MTP graphs --- .../agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml | 1 + .../agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml | 1 + perf-changelog.yaml | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml index 8a0e3254dc..6d2354b80b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml @@ -86,6 +86,7 @@ backend: <<: *dep8_config attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_fp4_indexer_cache":true}' enforce-eager: false + gpu-memory-utilization: 0.90 max-num-seqs: 1024 max-num-batched-tokens: 1024 compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml index 5e40d50fdf..067bc5c8f8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml @@ -81,6 +81,7 @@ backend: <<: *dep8_config attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_fp4_indexer_cache":true}' enforce-eager: false + gpu-memory-utilization: 0.90 max-num-seqs: 1024 max-num-batched-tokens: 1024 compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' diff --git a/perf-changelog.yaml b/perf-changelog.yaml index fbe85842ef..8863715f10 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5829,7 +5829,7 @@ - "Use immutable vLLM nightly commit 3ee2df3, Dynamo 1.3.1, and NVIDIA/srt-slurm v1.0.45; released v0.27.1 fails DeepSeek-V4 sparse-cache warmup on GB200." - "Collect every logical vLLM prefill/decode leader endpoint and require nonempty vllm: server metrics before publishing results." - "Use the official K=2 MTP configuration with golden synthetic acceptance length 2.49 for throughput; eval retains real MTP verification." - - "Use the established 0.95 GB200 disaggregated GPU memory budget so K=2 MTP and NIXL retain enough KV memory for the native 1,048,576-token context." + - "Use a 0.95 prefill and 0.90 decode GPU-memory budget: prefill retains long-context KV capacity while decode leaves measured headroom for K=2 MTP CUDA-graph capture." - "Select the supported FlashInfer DeepSeek V4 sparse-MLA backend, avoiding the released FlashMLA long-prefill TMA regression while preserving FP4 indexer caching." - "Use the public DeepSeek V4 MTP full-decode-only graph mode so aggregate and disaggregated DEP do not capture incompatible sliding-window cache views." - "Use the established 7,200-second Dynamo lease budget so the long FP4 model load and graph compile cannot expire healthy frontend and backend registrations." From bac45888f39d0831b8e48bdf97866d162ce20d59 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 03:03:32 -0500 Subject: [PATCH 08/11] fix(gb200): report vLLM prompt cache usage --- .../vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml | 1 + .../vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml | 1 + .../agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml | 1 + .../agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml | 1 + perf-changelog.yaml | 1 + 5 files changed, 5 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml index 5fa2a163c4..eb68daa1af 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml @@ -57,6 +57,7 @@ backend: vllm_config: aggregated: served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-prompt-tokens-details: true kv-cache-dtype: fp8 tensor-parallel-size: 1 pipeline-parallel-size: 1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml index 657b0c8e09..32990051e1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml @@ -66,6 +66,7 @@ backend: vllm_config: aggregated: served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-prompt-tokens-details: true kv-cache-dtype: fp8 tensor-parallel-size: 8 pipeline-parallel-size: 1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml index 6d2354b80b..a8fbbf1b67 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml @@ -63,6 +63,7 @@ backend: prefill: &dep8_config kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-prompt-tokens-details: true kv-cache-dtype: fp8 tensor-parallel-size: 1 pipeline-parallel-size: 1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml index 067bc5c8f8..bfca0449ab 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml @@ -58,6 +58,7 @@ backend: prefill: &dep8_config kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-prompt-tokens-details: true kv-cache-dtype: fp8 tensor-parallel-size: 1 pipeline-parallel-size: 1 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 8863715f10..d93e781281 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5828,6 +5828,7 @@ - "Add GB200 DeepSeek-V4-Pro FP4 Dynamo-vLLM MTP AgentX coverage across TP8, DEP8, 1P1D, and 2P1D discovery arms." - "Use immutable vLLM nightly commit 3ee2df3, Dynamo 1.3.1, and NVIDIA/srt-slurm v1.0.45; released v0.27.1 fails DeepSeek-V4 sparse-cache warmup on GB200." - "Collect every logical vLLM prefill/decode leader endpoint and require nonempty vllm: server metrics before publishing results." + - "Enable vLLM prompt-token details so AIPerf records per-request prefix-cache token usage in addition to backend Prometheus metrics." - "Use the official K=2 MTP configuration with golden synthetic acceptance length 2.49 for throughput; eval retains real MTP verification." - "Use a 0.95 prefill and 0.90 decode GPU-memory budget: prefill retains long-context KV capacity while decode leaves measured headroom for K=2 MTP CUDA-graph capture." - "Select the supported FlashInfer DeepSeek V4 sparse-MLA backend, avoiding the released FlashMLA long-prefill TMA regression while preserving FP4 indexer caching." From 1c225fec8a56367e88ad2c95f5c74d5e40d461e3 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 03:11:05 -0500 Subject: [PATCH 09/11] revert(gb200): drop unsupported vLLM cache flag --- .../vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml | 1 - .../vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml | 1 - .../agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml | 1 - .../agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml | 1 - perf-changelog.yaml | 1 - 5 files changed, 5 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml index eb68daa1af..5fa2a163c4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml @@ -57,7 +57,6 @@ backend: vllm_config: aggregated: served-model-name: deepseek-ai/DeepSeek-V4-Pro - enable-prompt-tokens-details: true kv-cache-dtype: fp8 tensor-parallel-size: 1 pipeline-parallel-size: 1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml index 32990051e1..657b0c8e09 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml @@ -66,7 +66,6 @@ backend: vllm_config: aggregated: served-model-name: deepseek-ai/DeepSeek-V4-Pro - enable-prompt-tokens-details: true kv-cache-dtype: fp8 tensor-parallel-size: 8 pipeline-parallel-size: 1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml index a8fbbf1b67..6d2354b80b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml @@ -63,7 +63,6 @@ backend: prefill: &dep8_config kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' served-model-name: deepseek-ai/DeepSeek-V4-Pro - enable-prompt-tokens-details: true kv-cache-dtype: fp8 tensor-parallel-size: 1 pipeline-parallel-size: 1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml index bfca0449ab..067bc5c8f8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml @@ -58,7 +58,6 @@ backend: prefill: &dep8_config kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' served-model-name: deepseek-ai/DeepSeek-V4-Pro - enable-prompt-tokens-details: true kv-cache-dtype: fp8 tensor-parallel-size: 1 pipeline-parallel-size: 1 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index d93e781281..8863715f10 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5828,7 +5828,6 @@ - "Add GB200 DeepSeek-V4-Pro FP4 Dynamo-vLLM MTP AgentX coverage across TP8, DEP8, 1P1D, and 2P1D discovery arms." - "Use immutable vLLM nightly commit 3ee2df3, Dynamo 1.3.1, and NVIDIA/srt-slurm v1.0.45; released v0.27.1 fails DeepSeek-V4 sparse-cache warmup on GB200." - "Collect every logical vLLM prefill/decode leader endpoint and require nonempty vllm: server metrics before publishing results." - - "Enable vLLM prompt-token details so AIPerf records per-request prefix-cache token usage in addition to backend Prometheus metrics." - "Use the official K=2 MTP configuration with golden synthetic acceptance length 2.49 for throughput; eval retains real MTP verification." - "Use a 0.95 prefill and 0.90 decode GPU-memory budget: prefill retains long-context KV capacity while decode leaves measured headroom for K=2 MTP CUDA-graph capture." - "Select the supported FlashInfer DeepSeek V4 sparse-MLA backend, avoiding the released FlashMLA long-prefill TMA regression while preserving FP4 indexer caching." From 50c170020637abf2cdcc3515970b013f49705d9f Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 04:19:45 -0500 Subject: [PATCH 10/11] fix(gb200): allow long DeepSeek prefill responses --- .../agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml | 4 ++++ .../agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml | 4 ++++ perf-changelog.yaml | 1 + 3 files changed, 9 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml index 6d2354b80b..6bd221ec10 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml @@ -36,6 +36,10 @@ infra: frontend: type: dynamo enable_multiple_frontends: false + env: + # Long AgentX prefills can exceed Dynamo's request-plane default while + # the healthy DEP8 worker is still computing the first response. + DYN_TCP_REQUEST_TIMEOUT: "60" args: router-mode: random router-session-affinity-ttl-secs: 900 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml index 067bc5c8f8..df18448148 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml @@ -31,6 +31,10 @@ infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } frontend: type: dynamo enable_multiple_frontends: false + env: + # Long AgentX prefills can exceed Dynamo's request-plane default while + # the healthy DEP8 workers are still computing their first responses. + DYN_TCP_REQUEST_TIMEOUT: "60" args: router-mode: random router-session-affinity-ttl-secs: 900 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 8863715f10..647cc960e3 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5833,4 +5833,5 @@ - "Select the supported FlashInfer DeepSeek V4 sparse-MLA backend, avoiding the released FlashMLA long-prefill TMA regression while preserving FP4 indexer caching." - "Use the public DeepSeek V4 MTP full-decode-only graph mode so aggregate and disaggregated DEP do not capture incompatible sliding-window cache views." - "Use the established 7,200-second Dynamo lease budget so the long FP4 model load and graph compile cannot expire healthy frontend and backend registrations." + - "Allow 60 seconds for Dynamo TCP request-plane acknowledgements so healthy long AgentX prefills are not cut off by the transport default." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2567 From b1f2928367e9fd2b4431816b25f06daed97a9833 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 07:01:17 -0500 Subject: [PATCH 11/11] perf(gb200): broaden DeepSeek V4 AgentX search --- configs/nvidia-master.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 6b3047f956..f0d0f4bc94 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7381,7 +7381,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: # The checkpoint needs two GB200 NVL4 trays. TP8 anchors latency; DEP8 # follows the public vLLM recommendation and probes the throughput curve. - spec-decoding: mtp - conc-list: [1] + conc-list: [1, 2, 4, 8] prefill: num-worker: 1 tp: 8 @@ -7397,7 +7397,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: ep: 1 dp-attn: false - spec-decoding: mtp - conc-list: [16] + conc-list: [4, 8, 16, 24, 32, 48, 64, 96] prefill: num-worker: 1 tp: 8 @@ -7430,7 +7430,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: # Public-recipe baseline: one two-tray DEP8 prefill worker and one # two-tray DEP8 decode worker. Dense sampling finds the MTP knee. - spec-decoding: mtp - conc-list: [128] + conc-list: [32, 64, 96, 128, 192, 256, 384] prefill: num-worker: 1 tp: 8 @@ -7448,7 +7448,7 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: # Add prefill capacity only after the balanced baseline reaches its # high-concurrency region; this separates prefill from decode scaling. - spec-decoding: mtp - conc-list: [384] + conc-list: [192, 256, 384, 512] prefill: num-worker: 2 tp: 8