diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml new file mode 100644 index 0000000000..9548372f23 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml @@ -0,0 +1,98 @@ +name: dsv4-gb200-vllm-agentic-mtp-agg-dep8 + +model: + path: deepseek-v4-pro + container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 + precision: fp4 + +identity: + model: { repo: deepseek-ai/DeepSeek-V4-Pro } + container: { image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 } + frameworks: { dynamo: "1.3.1" } + +dynamo: + version: "1.3.1" + install: true + +setup_script: vllm-container-deps.sh +environment: { ETCD_LEASE_TTL: "7200" } +slurm: { time_limit: "8:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } + +resources: + gpu_type: gb200 + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: random + router-session-affinity-ttl-secs: 900 + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + aggregated_environment: + HF_HUB_CACHE: /hf_hub_cache + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + vllm_config: + aggregated: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_mega_moe + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 256 + max-num-batched-tokens: 8192 + trust-remote-code: true + block-size: 256 + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' + gpu-memory-utilization: 0.90 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml new file mode 100644 index 0000000000..bcbcd22059 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml @@ -0,0 +1,108 @@ +name: dsv4-gb200-vllm-agentic-mtp-agg-tp8 + +model: + path: deepseek-v4-pro + container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 + precision: fp4 + +identity: + model: + repo: deepseek-ai/DeepSeek-V4-Pro + container: + image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 + frameworks: + dynamo: "1.3.1" + +dynamo: + version: "1.3.1" + install: true + +setup_script: vllm-container-deps.sh + +environment: + ETCD_LEASE_TTL: "7200" + +slurm: + time_limit: "8:00:00" + +health_check: + max_attempts: 2160 + interval_seconds: 10 + +resources: + gpu_type: gb200 + gpus_per_node: 4 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: random + router-session-affinity-ttl-secs: 900 + +backend: + type: vllm + connector: null + aggregated_environment: + HF_HUB_CACHE: /hf_hub_cache + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + vllm_config: + aggregated: + served-model-name: deepseek-ai/DeepSeek-V4-Pro + kv-cache-dtype: fp8 + tensor-parallel-size: 8 + pipeline-parallel-size: 1 + disable-custom-all-reduce: true + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 128 + max-num-batched-tokens: 8192 + trust-remote-code: true + no-enable-flashinfer-autotune: true + block-size: 256 + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' + gpu-memory-utilization: 0.90 + stream-interval: 10 + no-disable-hybrid-kv-cache-manager: true + tokenizer-mode: deepseek_v4 + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + +srun_options: + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml new file mode 100644 index 0000000000..5226a8dd02 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml @@ -0,0 +1,118 @@ +name: dsv4-gb200-vllm-agentic-mtp-disagg-1p1d-dep8-dep8 + +model: + path: deepseek-v4-pro + container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 + precision: fp4 + +identity: + model: { repo: deepseek-ai/DeepSeek-V4-Pro } + container: { image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 } + frameworks: { dynamo: "1.3.1" } + +dynamo: + version: "1.3.1" + install: true + +setup_script: vllm-container-deps.sh +environment: { ETCD_LEASE_TTL: "7200" } +slurm: { time_limit: "8:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } + +resources: + gpu_type: gb200 + gpus_per_node: 4 + prefill_nodes: 2 + decode_nodes: 2 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: + etcd_nats_dedicated_node: false + nats_max_payload_mb: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + # Long AgentX prefills can exceed Dynamo's request-plane default while + # the healthy DEP8 worker is still computing the first response. + DYN_TCP_REQUEST_TIMEOUT: "60" + args: + router-mode: random + router-session-affinity-ttl-secs: 900 + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + prefill_environment: &worker_environment + HF_HUB_CACHE: /hf_hub_cache + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: *worker_environment + vllm_config: + prefill: &dep8_config + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + served-model-name: deepseek-ai/DeepSeek-V4-Pro + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_mega_moe + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + enforce-eager: true + block-size: 256 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' + decode: + <<: *dep8_config + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_fp4_indexer_cache":true}' + enforce-eager: false + gpu-memory-utilization: 0.90 + max-num-seqs: 1024 + max-num-batched-tokens: 1024 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 1024 + stream-interval: 10 + +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml new file mode 100644 index 0000000000..c07d458126 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml @@ -0,0 +1,113 @@ +name: dsv4-gb200-vllm-agentic-mtp-disagg-2p1d-dep8-dep8 + +model: + path: deepseek-v4-pro + container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 + precision: fp4 + +identity: + model: { repo: deepseek-ai/DeepSeek-V4-Pro } + container: { image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 } + frameworks: { dynamo: "1.3.1" } + +dynamo: { version: "1.3.1", install: true } +setup_script: vllm-container-deps.sh +environment: { ETCD_LEASE_TTL: "7200" } +slurm: { time_limit: "8:00:00" } +health_check: { max_attempts: 2160, interval_seconds: 10 } + +resources: + gpu_type: gb200 + gpus_per_node: 4 + prefill_nodes: 4 + decode_nodes: 2 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +infra: { etcd_nats_dedicated_node: false, nats_max_payload_mb: 32 } + +frontend: + type: dynamo + enable_multiple_frontends: false + env: + # Long AgentX prefills can exceed Dynamo's request-plane default while + # the healthy DEP8 workers are still computing their first responses. + DYN_TCP_REQUEST_TIMEOUT: "60" + args: + router-mode: random + router-session-affinity-ttl-secs: 900 + +backend: + type: vllm + connector: null + dp_launch_mode: per_node + prefill_environment: &worker_environment + HF_HUB_CACHE: /hf_hub_cache + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + VLLM_RPC_TIMEOUT: "600000" + VLLM_LOG_STATS_INTERVAL: "1" + VLLM_SERVER_DEV_MODE: "1" + VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" + VLLM_USE_NCCL_SYMM_MEM: "1" + NCCL_CUMEM_ENABLE: "1" + NCCL_MNNVL_ENABLE: "1" + NCCL_NVLS_ENABLE: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" + UCX_TLS: "rc,cuda_copy" + NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" + decode_environment: *worker_environment + vllm_config: + prefill: &dep8_config + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + served-model-name: deepseek-ai/DeepSeek-V4-Pro + kv-cache-dtype: fp8 + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 8 + enable-expert-parallel: true + enable-ep-weight-filter: true + moe-backend: deep_gemm_mega_moe + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' + max-model-len: 1048576 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + trust-remote-code: true + enforce-eager: true + block-size: 256 + gpu-memory-utilization: 0.95 + no-disable-hybrid-kv-cache-manager: true + enable-sleep-mode: true + tokenizer-mode: deepseek_v4 + speculative-config: '{"method":"mtp","num_speculative_tokens":2}' + decode: + <<: *dep8_config + attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_fp4_indexer_cache":true}' + enforce-eager: false + gpu-memory-utilization: 0.90 + max-num-seqs: 1024 + max-num-batched-tokens: 1024 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + max-cudagraph-capture-size: 1024 + stream-interval: 10 + +sbatch_directives: { cpus-per-task: "144", mem: "0" } +srun_options: { container-remap-root: "" } + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 5c049d9f0c..10c79ea22a 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -7545,6 +7545,103 @@ dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep8: ep: 8 dp-attn: true +dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg: + image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.1" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - search-space: + # Match the B200 TP8 latency points for direct normalized-interactivity + # comparison, then use the public vLLM GB200 DEP8 topology for throughput. + - spec-decoding: mtp + conc-list: [1, 2, 4, 6, 8] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 1 + dp-attn: false + - spec-decoding: mtp + conc-list: [8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml" + decode: + num-worker: 0 + tp: 8 + ep: 8 + dp-attn: true + +dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg: + image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:gb200-nv + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.3.1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + # One two-tray DEP8 prefill and decode worker covers the balanced region. + - spec-decoding: mtp + conc-list: [32, 64, 96, 128, 192, 256] + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # A second DEP8 prefill worker extends the high-concurrency region. + - spec-decoding: mtp + conc-list: [192, 256, 384, 512] + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "SYNTHETIC_ACCEPTANCE=true" + - "SYNTHETIC_ACCEPTANCE_LENGTH=2.49" + - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-mtp-agentic.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f model: deepseek-ai/DeepSeek-V4-Pro diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 676fc73e7f..d082ced48f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5902,3 +5902,14 @@ - "Use NIXL for KV transfer with EAGLE speculative decoding and chat-formatted benchmark inputs." - "Image: lmsysorg/sglang:nightly-dev-cu13-20260710-cfc66e05" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2554 + +- config-keys: + - dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-agg + - dsv4-fp4-gb200-dynamo-vllm-agentic-mtp-disagg + scenario-type: + - agentic-coding + description: + - "Add GB200 DeepSeek-V4-Pro FP4 Dynamo-vLLM AgentX with K=2 MTP and golden synthetic acceptance length 2.49." + - "Compare the B200 TP8 and DEP8 concurrency range and extend throughput with 1P1D and 2P1D DEP8 disaggregation." + - "Retain prefix-cache blocks for 32,768 scheduling steps and export every logical vLLM server-metrics endpoint." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2567 diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index 660ae078d5..ed4cbed78b 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -391,18 +391,24 @@ if [ -d "$SRT_REPO_DIR" ]; then rm -rf "$SRT_REPO_DIR" fi -# Qwen3.5 FP4 AgentX uses the latest released srt-slurm. v1.0.45 injects -# the aggregate logical-worker Prometheus endpoint into custom benchmarks. -if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-sglang" ]]; then +# These AgentX submissions use released srt-slurm custom-benchmark metrics +# discovery so AIPerf receives every logical worker endpoint. +if [[ "$IS_AGENTIC" == "1" && (( "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-sglang" ) || ( "$MODEL_PREFIX" == "dsv4" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-vllm" )) ]]; then git clone --branch v1.0.45 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" test "$(git rev-parse HEAD)" = "9d8d92b20c350a5d42f0709f5a0b64e30eb37d33" || { echo "Error: NVIDIA/srt-slurm v1.0.45 resolved to an unexpected commit" >&2 exit 1 } - mkdir -p recipes/sglang/qwen3.5/gb200-fp4/agentic - cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic" \ - recipes/sglang/qwen3.5/gb200-fp4/agentic + if [[ "$MODEL_PREFIX" == "qwen3.5" ]]; then + mkdir -p recipes/sglang/qwen3.5/gb200-fp4/agentic + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb200-fp4/agentic" \ + recipes/sglang/qwen3.5/gb200-fp4/agentic + else + mkdir -p recipes/vllm/deepseek-v4/agentic + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic" \ + recipes/vllm/deepseek-v4/agentic + fi # TODO(CJQ): migrate the remaining Agentic model paths to released srt-slurm. elif [[ "$IS_AGENTIC" == "1" ]]; then # Agentic multi-node pins cquil11/srt-slurm-nv revisions that provide: