From 8cb7f86d0f2a4a9fe7e640540675c151ecb5fa38 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Mon, 10 Aug 2026 20:22:24 +0000 Subject: [PATCH 1/2] perf(agentx): stabilize DSV4 GB300 MTP sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consolidate the GB300 AgentX MTP bring-up, runner selection, Dynamo transport hardening, AIPerf integration, and six-point aggregate/P-D sweep. Keep DeepGEMM AMX FP4 Mega MoE on expert-parallel P/D engines while restoring automatic MoE backend selection for pure aggregate TP. Disable the vLLM cumem allocator for aggregate TP, retain the 8192-token budget, size max-num-seqs as max(16, 4*concurrency) with matching MTP CUDA graph coverage, and use gpu-memory-utilization 0.94. 中文：整合 GB300 AgentX MTP 的启动流程、运行节点选择、Dynamo 传输加固、AIPerf 集成和六点聚合/P-D 扫描。在专家并行 P/D 引擎上保留 DeepGEMM AMX FP4 Mega MoE，并为纯聚合 TP 恢复自动 MoE 后端选择；聚合 TP 禁用 vLLM cumem 分配器并保留 8192-token 预算，同时按 max(16, 4*并发) 设置 max-num-seqs、配置对应的 MTP CUDA Graph 覆盖范围，并将 gpu-memory-utilization 设为 0.94。 --- .../agentic/agg-gb300-tp4-mtp-agentic.yaml | 15 +- .../agentic/agg-gb300-tp8-mtp-agentic.yaml | 15 +- ...b300-1p1d-dep4-dep8-c256-mtp-agentic.yaml} | 21 +- ...00-2p1d-dep8-dep12-c1152-mtp-agentic.yaml} | 58 ++--- ...300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml | 210 ++++++++++++++++++ configs/nvidia-master.yaml | 49 +++- perf-changelog.yaml | 9 + 7 files changed, 319 insertions(+), 58 deletions(-) rename benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/{disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml => disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml} (85%) rename benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/{disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml => disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml} (80%) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml index 192e0f5b95..46647f6e17 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml @@ -1,8 +1,8 @@ name: "svf-vllm-agg-gb300-tp4-mtp-agentic" # GB300 AgentX aggregate topology: one TP4 worker occupies one four-GPU node -# and serves both prefill and decode at concurrency 4. Scheduler, CUDA-graph, -# and memory settings match the B300 vLLM TP4 MTP agentic configuration. +# and serves both prefill and decode at concurrency 8. Size max-num-seqs at +# 4x concurrency and expand the MTP CUDA-graph envelope to match. model: path: "deepseek-v4-pro" @@ -77,7 +77,7 @@ backend: VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "0" TORCH_SYMMMEM: "NVSHMEM" @@ -110,17 +110,16 @@ backend: tensor-parallel-size: 4 pipeline-parallel-size: 1 disable-custom-all-reduce: true - enable-cumem-allocator: true attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' max-model-len: 1048576 - max-num-seqs: 16 + max-num-seqs: 32 max-num-batched-tokens: 8192 trust-remote-code: true no-enable-flashinfer-autotune: true block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' - gpu-memory-utilization: 0.93 + gpu-memory-utilization: 0.94 stream-interval: 10 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" @@ -141,6 +140,8 @@ benchmark: PORT: "8000" IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml index 6a950787fe..bc297d2c52 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml @@ -1,7 +1,8 @@ name: "svf-vllm-agg-gb300-tp8-mtp-agentic" -# Validated GB300 AgentX aggregate topology: one TP8 worker spans two -# four-GPU nodes and serves both prefill and decode at concurrency 1. +# GB300 AgentX aggregate topology: one TP8 worker spans two four-GPU +# nodes and serves both prefill and decode at concurrency 4. Keep at least +# 16 sequence slots and otherwise size the scheduler at 4x concurrency. model: path: "deepseek-v4-pro" @@ -77,7 +78,7 @@ backend: VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_RPC_TIMEOUT: "600000" VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" TILELANG_CLEANUP_TEMP_FILES: "1" VLLM_USE_NCCL_SYMM_MEM: "0" TORCH_SYMMMEM: "NVSHMEM" @@ -110,18 +111,16 @@ backend: tensor-parallel-size: 8 pipeline-parallel-size: 1 disable-custom-all-reduce: true - enable-cumem-allocator: true attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' max-model-len: 1048576 - max-num-seqs: 32 + max-num-seqs: 16 max-num-batched-tokens: 8192 trust-remote-code: true no-enable-flashinfer-autotune: true block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 128 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' - gpu-memory-utilization: 0.90 + gpu-memory-utilization: 0.94 stream-interval: 10 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml similarity index 85% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml index dd58dde28f..c7d95ecbeb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml @@ -1,7 +1,7 @@ -name: "svf-vllm-disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic" +name: "svf-vllm-disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic" -# Validated GB300 AgentX MTP3 low-latency topology: one DEP4 prefill worker -# feeds one DEP8 decode worker at concurrency 128. +# GB300 AgentX MTP3 topology: one DEP4 prefill worker feeds one +# DEP8 decode worker at concurrency 256. model: path: "deepseek-v4-pro" @@ -17,7 +17,7 @@ identity: dynamo: "1.3.0.dev20260720" dynamo: - version: "1.3.0.dev20260720" + wheel: "1.3.0.dev20260720" install: true setup_script: vllm-container-deps.sh @@ -57,6 +57,8 @@ frontend: router-session-affinity-ttl-secs: 900 env: DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" backend: type: vllm @@ -65,7 +67,7 @@ backend: mooncake_kv_store: store_config: metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" + global_segment_size: "180GB" local_buffer_size: "4GB" protocol: "rdma" device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" @@ -86,6 +88,7 @@ backend: max-model-len: 1048576 max-num-seqs: 64 max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 trust-remote-code: true no-enable-flashinfer-autotune: true block-size: 256 @@ -99,7 +102,7 @@ backend: numa-bind: true numa-bind-nodes: [0, 0, 1, 1] decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "deepseek-ai/DeepSeek-V4-Pro" kv-cache-dtype: "fp8" tensor-parallel-size: 1 @@ -149,7 +152,7 @@ backend: VLLM_USE_BREAKABLE_CUDAGRAPH: "0" VLLM_CONNECTOR_PREFETCH_DEPTH: "8" VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-c128-mtp-{job_id}" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep4-dep8-c256-{job_id}" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" MC_STORE_CLIENT_METRIC: "1" MC_STORE_CLIENT_METRIC_INTERVAL: "5" @@ -176,7 +179,7 @@ backend: VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-c128-mtp-{job_id}" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-dep4-dep8-c256-{job_id}" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" MC_STORE_CLIENT_METRIC: "1" MC_STORE_CLIENT_METRIC_INTERVAL: "5" @@ -198,6 +201,8 @@ benchmark: IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml similarity index 80% rename from benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml rename to benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml index 00a876ebe8..2e1fc4e2c3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml @@ -1,7 +1,8 @@ -name: "svf-vllm-disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic" +name: "svf-vllm-disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic" -# Validated GB300 AgentX MTP3 efficiency topology: one DEP8 prefill worker and -# one DEP8 decode worker at concurrency 384. +# GB300 AgentX MTP3 topology: two DEP8 prefill workers feed one +# DEP12 decode worker at concurrency 1152. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. model: path: "deepseek-v4-pro" @@ -17,9 +18,13 @@ identity: dynamo: "1.3.0.dev20260720" dynamo: - version: "1.3.0.dev20260720" + wheel: "1.3.0.dev20260720" install: true +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + setup_script: vllm-container-deps.sh slurm: @@ -34,21 +39,17 @@ resources: gpus_per_node: 4 het_jobs: false spread_workers: false - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 1 + prefill_nodes: 4 + decode_nodes: 3 + prefill_workers: 2 decode_workers: 1 gpus_per_prefill: 8 - gpus_per_decode: 8 + gpus_per_decode: 12 infra: etcd_nats_dedicated_node: false nats_max_payload_mb: 32 -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - frontend: type: dynamo enable_multiple_frontends: false @@ -57,6 +58,8 @@ frontend: router-session-affinity-ttl-secs: 900 env: DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" backend: type: vllm @@ -65,7 +68,7 @@ backend: mooncake_kv_store: store_config: metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" + global_segment_size: "180GB" local_buffer_size: "4GB" protocol: "rdma" device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" @@ -84,13 +87,13 @@ backend: enable-expert-parallel: true enable-ep-weight-filter: true max-model-len: 1048576 - max-num-seqs: 32 + max-num-seqs: 64 max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 trust-remote-code: true no-enable-flashinfer-autotune: true block-size: 256 - max-cudagraph-capture-size: 128 - gpu-memory-utilization: 0.92 + gpu-memory-utilization: 0.90 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' @@ -99,25 +102,25 @@ backend: numa-bind: true numa-bind-nodes: [0, 0, 1, 1] decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' served-model-name: "deepseek-ai/DeepSeek-V4-Pro" kv-cache-dtype: "fp8" tensor-parallel-size: 1 pipeline-parallel-size: 1 - data-parallel-size: 8 + data-parallel-size: 12 data-parallel-rpc-port: 13345 enable-cumem-allocator: true enable-expert-parallel: true enable-ep-weight-filter: true max-model-len: 1048576 - max-num-seqs: 256 - max-num-batched-tokens: 1024 + max-num-seqs: 64 + max-num-batched-tokens: 256 trust-remote-code: true no-enable-flashinfer-autotune: true block-size: 256 compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 1024 - gpu-memory-utilization: 0.92 + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.90 no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' @@ -136,6 +139,7 @@ backend: VLLM_SERVER_DEV_MODE: "1" VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" VLLM_ALLREDUCE_USE_SYMM_MEM: "0" UCX_MEMTYPE_CACHE: "n" UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" @@ -143,13 +147,13 @@ backend: NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" VLLM_USE_BREAKABLE_CUDAGRAPH: "0" VLLM_CONNECTOR_PREFETCH_DEPTH: "8" VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-mtp-{job_id}" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-prefill-{job_id}" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" MC_STORE_CLIENT_METRIC: "1" MC_STORE_CLIENT_METRIC_INTERVAL: "5" @@ -172,11 +176,11 @@ backend: NCCL_CUMEM_ENABLE: "1" NCCL_MNNVL_ENABLE: "1" NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-mtp-{job_id}" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-dep12-decode-{job_id}" MC_ENABLE_DEST_DEVICE_AFFINITY: "1" MC_STORE_CLIENT_METRIC: "1" MC_STORE_CLIENT_METRIC_INTERVAL: "5" @@ -199,6 +203,8 @@ benchmark: IS_MULTINODE: "true" AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml new file mode 100644 index 0000000000..925dcdfcca --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml @@ -0,0 +1,210 @@ +name: "svf-vllm-disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic" + +# GB300 AgentX MTP3 topology: two DEP8 prefill workers feed one +# DEP16 decode worker at concurrency 1024. Decode consumes P/D KV through NIXL +# and MooncakeStore but skips Mooncake prefix lookup to avoid CPU overhead. + +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" + frameworks: + dynamo: "1.3.0.dev20260720" + +dynamo: + wheel: "1.3.0.dev20260720" + install: true + +environment: + # Mooncake prefix-block hashes must match across processes and nodes. + PYTHONHASHSEED: "0" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: "gb300" + gpus_per_node: 4 + het_jobs: false + spread_workers: false + prefill_nodes: 4 + decode_nodes: 4 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 16 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: "random" + router-session-affinity-ttl-secs: 900 + env: + DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" + DYN_TCP_CHANNEL_BUFFER: "128" + DYN_TCP_REQUEST_TIMEOUT: "60" + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + mooncake_kv_store: + store_config: + metadata_server: "P2PHANDSHAKE" + global_segment_size: "180GB" + local_buffer_size: "4GB" + protocol: "rdma" + device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + mode: "embedded" + enable_offload: false + vllm_config: + prefill: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 8192 + long-prefill-token-threshold: 1024 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + decode: + kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_consumer","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + kv-cache-dtype: "fp8" + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 16 + data-parallel-rpc-port: 13345 + enable-cumem-allocator: true + enable-expert-parallel: true + enable-ep-weight-filter: true + max-model-len: 1048576 + max-num-seqs: 64 + max-num-batched-tokens: 256 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 256 + gpu-memory-utilization: 0.90 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: "deepseek_v4" + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + moe-backend: "deep_gemm_amxf4_mega_moe" + numa-bind: true + numa-bind-nodes: [0, 0, 1, 1] + prefill_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_MOONCAKE_STORE_SEND_THREADS: "8" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_BREAKABLE_CUDAGRAPH: "0" + VLLM_CONNECTOR_PREFETCH_DEPTH: "8" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-prefill-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + decode_environment: + HF_HUB_CACHE: "/hf_hub_cache" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" + TRANSFORMERS_CACHE: "/hf_hub_cache" + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" + VLLM_SERVER_DEV_MODE: "1" + VLLM_USE_V2_MODEL_RUNNER: "1" + VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" + VLLM_ALLREDUCE_USE_SYMM_MEM: "0" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_DSV4_MEGA_FP8_COMBINE: "1" + DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-2p1d-dep16-decode-{job_id}" + MC_ENABLE_DEST_DEVICE_AFFINITY: "1" + MC_STORE_CLIENT_METRIC: "1" + MC_STORE_CLIENT_METRIC_INTERVAL: "5" + MC_TE_METRIC: "0" + +sbatch_directives: + cpus-per-task: "72" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + # Avoid concurrent readers observing a mismatched mmap data/index pair. + AIPERF_DATASET_MMAP_CACHE_ENABLED: "false" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" + WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 5c049d9f0c..2557c4feb2 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7576,6 +7576,22 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: dp-attn: false - spec-decoding: mtp conc-list: [4] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml" + # The aggregate worker also performs decode; keep the decode worker + # count at zero so result aggregation counts eight GPUs only once. + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [8] prefill: num-worker: 1 tp: 4 @@ -7605,30 +7621,45 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: agentic-coding: - search-space: - spec-decoding: mtp - conc-list: [128] + conc-list: [1152] router: { name: dynamo-router, version: "1.3.0.dev20260720" } prefill: - num-worker: 1 - tp: 4 - ep: 4 + num-worker: 2 + tp: 8 + ep: 8 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml" decode: num-worker: 1 + tp: 12 + ep: 12 + dp-attn: true + - spec-decoding: mtp + conc-list: [1024] + router: { name: dynamo-router, version: "1.3.0.dev20260720" } + prefill: + num-worker: 2 tp: 8 ep: 8 dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true - spec-decoding: mtp - conc-list: [384] + conc-list: [256] router: { name: dynamo-router, version: "1.3.0.dev20260720" } prefill: num-worker: 1 - tp: 8 - ep: 8 + tp: 4 + ep: 4 dp-attn: true additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml" decode: num-worker: 1 tp: 8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 676fc73e7f..5d5554941b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5902,3 +5902,12 @@ - "Use NIXL for KV transfer with EAGLE speculative decoding and chat-formatted benchmark inputs." - "Image: lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2554 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Run all GB300 DeepSeek-V4-Pro AgentX MTP configs." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2571 From 9926c051931ec3ffd3d82386cddea1e440d075c8 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 12 Aug 2026 14:14:59 -0500 Subject: [PATCH 2/2] fix(agentx): keep GB300 evals on real MTP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Route GB300 multi-node AgentX recipes through the shared throughput-only synthetic acceptance injector and add regression coverage for eval-only behavior.\n\n中文：让 GB300 多节点 AgentX 配方使用仅吞吐量启用的共享合成接受率注入器，并添加仅评估模式的回归测试。 --- .../agentic/agg-gb300-tp4-mtp-agentic.yaml | 2 +- .../agentic/agg-gb300-tp8-mtp-agentic.yaml | 2 +- ...gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml | 4 +- ...300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml | 4 +- ...300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml | 4 +- configs/nvidia-master.yaml | 16 +++ perf-changelog.yaml | 9 ++ runners/launch_gb300-nv.sh | 8 ++ runners/test_synthetic_acceptance.py | 113 ++++++++++++++++++ 9 files changed, 154 insertions(+), 8 deletions(-) create mode 100644 runners/test_synthetic_acceptance.py diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml index 46647f6e17..9559ffb75b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml @@ -118,7 +118,7 @@ backend: no-enable-flashinfer-autotune: true block-size: 256 compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68,72,76,80,84,88,92,96,100,104,108,112,116,120,124,128],"mode":0}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' gpu-memory-utilization: 0.94 stream-interval: 10 no-disable-hybrid-kv-cache-manager: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml index bc297d2c52..4eebce98fa 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml @@ -119,7 +119,7 @@ backend: no-enable-flashinfer-autotune: true block-size: 256 compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' gpu-memory-utilization: 0.94 stream-interval: 10 no-disable-hybrid-kv-cache-manager: true diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml index c7d95ecbeb..e5cc1c76ea 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml @@ -97,7 +97,7 @@ backend: no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' moe-backend: "deep_gemm_amxf4_mega_moe" numa-bind: true numa-bind-nodes: [0, 0, 1, 1] @@ -124,7 +124,7 @@ backend: no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' moe-backend: "deep_gemm_amxf4_mega_moe" numa-bind: true numa-bind-nodes: [0, 0, 1, 1] diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml index 2e1fc4e2c3..d5a0b44161 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml @@ -97,7 +97,7 @@ backend: no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' moe-backend: "deep_gemm_amxf4_mega_moe" numa-bind: true numa-bind-nodes: [0, 0, 1, 1] @@ -124,7 +124,7 @@ backend: no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' moe-backend: "deep_gemm_amxf4_mega_moe" numa-bind: true numa-bind-nodes: [0, 0, 1, 1] diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml index 925dcdfcca..f770241932 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml @@ -97,7 +97,7 @@ backend: no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' moe-backend: "deep_gemm_amxf4_mega_moe" numa-bind: true numa-bind-nodes: [0, 0, 1, 1] @@ -124,7 +124,7 @@ backend: no-disable-hybrid-kv-cache-manager: true tokenizer-mode: "deepseek_v4" attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' + speculative-config: '{"method":"mtp","num_speculative_tokens":3}' moe-backend: "deep_gemm_amxf4_mega_moe" numa-bind: true numa-bind-nodes: [0, 0, 1, 1] diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 2557c4feb2..5da4749ebe 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7558,6 +7558,8 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: scenarios: agentic-coding: - search-space: + # Keep the checked-in recipes on real MTP verification. Throughput jobs + # inject the committed golden AL at launch; EVAL_ONLY leaves them real. - spec-decoding: mtp conc-list: [1] prefill: @@ -7566,6 +7568,8 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: ep: 1 dp-attn: false additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml" # The aggregate worker also performs decode; keep the decode worker # count at zero so result aggregation counts eight GPUs only once. @@ -7582,6 +7586,8 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: ep: 1 dp-attn: false additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml" # The aggregate worker also performs decode; keep the decode worker # count at zero so result aggregation counts eight GPUs only once. @@ -7598,6 +7604,8 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: ep: 1 dp-attn: false additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml" # The aggregate worker also performs decode; keep the decode worker # count at zero so result aggregation counts four GPUs only once. @@ -7620,6 +7628,8 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: scenarios: agentic-coding: - search-space: + # Keep the checked-in recipes on real MTP verification. Throughput jobs + # inject the committed golden AL at launch; EVAL_ONLY leaves them real. - spec-decoding: mtp conc-list: [1152] router: { name: dynamo-router, version: "1.3.0.dev20260720" } @@ -7629,6 +7639,8 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 8 dp-attn: true additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep12-c1152-mtp-agentic.yaml" decode: num-worker: 1 @@ -7644,6 +7656,8 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 8 dp-attn: true additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-dep16-c1024-mtp-agentic.yaml" decode: num-worker: 1 @@ -7659,6 +7673,8 @@ dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: ep: 4 dp-attn: true additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c256-mtp-agentic.yaml" decode: num-worker: 1 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5d5554941b..d6240c4d6a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5911,3 +5911,12 @@ description: - "Run all GB300 DeepSeek-V4-Pro AgentX MTP configs." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2571 + +- config-keys: + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg + - dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Inject the committed synthetic MTP acceptance length only for GB300 AgentX throughput; keep eval-only jobs on real target verification." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2571 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 27154c404e..54d208bc7f 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -4,6 +4,9 @@ set -exo pipefail +# shellcheck source=runners/slurm_utils.sh +source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh" + export SLURM_PARTITION="batch_1" export SLURM_ACCOUNT="benchmark" export ENROOT_ROOTFS_WRITABLE=1 @@ -425,6 +428,11 @@ fi CONFIG_PATH="${CONFIG_FILE%%:*}" sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" +# Throughput recipes opt into synthetic acceptance through the master config. +# Eval-only jobs leave the checked-in real-MTP recipe unchanged so generated +# tokens still pass target-model verification. +inject_synthetic_acceptance "$CONFIG_PATH" "$FRAMEWORK" || exit 1 + # --no-preflight skips srtctl's pre-submit model-path stat, which runs on # the GHA runner host (im-gb300-login-02, an x86 login node). It's required # whenever model.path resolves to the node-local /scratch NVMe that the login diff --git a/runners/test_synthetic_acceptance.py b/runners/test_synthetic_acceptance.py new file mode 100644 index 0000000000..8c91e314d3 --- /dev/null +++ b/runners/test_synthetic_acceptance.py @@ -0,0 +1,113 @@ +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[1] +INJECTOR = REPO_ROOT / "runners" / "inject_synthetic_acceptance.py" +GB300_LAUNCHER = REPO_ROOT / "runners" / "launch_gb300-nv.sh" +NVIDIA_MASTER = REPO_ROOT / "configs" / "nvidia-master.yaml" +CHECKED_IN_RECIPES = REPO_ROOT / "benchmarks" / "multi_node" / "srt-slurm-recipes" +SPEC_CONFIG_RE = re.compile(r"speculative-config:\s*'([^']+)'") +GB300_DSV4_KEYS = ( + "dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg", + "dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg", +) + + +def run_injector(recipe: Path, *, eval_only: bool) -> subprocess.CompletedProcess[str]: + env = { + **os.environ, + "EVAL_ONLY": "true" if eval_only else "false", + "SYNTHETIC_ACCEPTANCE": "true", + "SYNTHETIC_ACCEPTANCE_LENGTH": "2.49", + } + return subprocess.run( + [sys.executable, str(INJECTOR), str(recipe), "dynamo-vllm"], + check=False, + capture_output=True, + text=True, + env=env, + ) + + +def test_throughput_injects_synthetic_acceptance(tmp_path: Path) -> None: + recipe = tmp_path / "recipe.yaml" + recipe.write_text( + ' speculative-config: \'{"method":"mtp","num_speculative_tokens":3}\'\n' + ) + + result = run_injector(recipe, eval_only=False) + + assert result.returncode == 0, result.stderr + match = SPEC_CONFIG_RE.search(recipe.read_text()) + assert match is not None + assert json.loads(match.group(1)) == { + "method": "mtp", + "num_speculative_tokens": 3, + "rejection_sample_method": "synthetic", + "synthetic_acceptance_length": 2.49, + } + + +def test_eval_only_keeps_real_mtp_recipe(tmp_path: Path) -> None: + recipe = tmp_path / "recipe.yaml" + original = ( + ' speculative-config: \'{"method":"mtp","num_speculative_tokens":3}\'\n' + ) + recipe.write_text(original) + + result = run_injector(recipe, eval_only=True) + + assert result.returncode == 0, result.stderr + assert recipe.read_text() == original + assert "EVAL_ONLY=true: keeping real MTP recipe" in result.stdout + + +def test_gb300_agentic_configs_inject_synthetic_only_at_launch() -> None: + master = yaml.safe_load(NVIDIA_MASTER.read_text()) + recipe_paths: set[Path] = set() + + for config_key in GB300_DSV4_KEYS: + search_space = master[config_key]["scenarios"]["agentic-coding"][0][ + "search-space" + ] + for point in search_space: + settings = point["prefill"]["additional-settings"] + assert "SYNTHETIC_ACCEPTANCE=true" in settings + assert "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" in settings + config_setting = next( + setting for setting in settings if setting.startswith("CONFIG_FILE=") + ) + config_path = config_setting.removeprefix("CONFIG_FILE=") + recipe_paths.add(CHECKED_IN_RECIPES / config_path.removeprefix("recipes/")) + + assert len(recipe_paths) == 5 + for recipe_path in recipe_paths: + content = recipe_path.read_text() + matches = SPEC_CONFIG_RE.findall(content) + assert matches, recipe_path + for raw_config in matches: + spec_config = json.loads(raw_config) + assert spec_config["method"] == "mtp" + assert spec_config["num_speculative_tokens"] == 3 + assert "rejection_sample_method" not in spec_config + assert "synthetic_acceptance_length" not in spec_config + + +def test_gb300_launcher_runs_acceptance_normalizer_before_srtctl() -> None: + launcher = GB300_LAUNCHER.read_text() + + source_offset = launcher.index( + 'source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh"' + ) + inject_offset = launcher.index( + 'inject_synthetic_acceptance "$CONFIG_PATH" "$FRAMEWORK" || exit 1' + ) + apply_offset = launcher.index("SRTCTL_OUTPUT=$(srtctl apply") + + assert source_offset < inject_offset < apply_offset