perf: add GB200 DeepSeek V4 FP4 vLLM MTP AgentX

benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-dep8-mtp-agentic.yaml

@@ -0,0 +1,98 @@
+name: dsv4-gb200-vllm-agentic-mtp-agg-dep8
+
+model:
+  path: deepseek-v4-pro
+  container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106
+  precision: fp4
+
+identity:
+  model: { repo: deepseek-ai/DeepSeek-V4-Pro }
+  container: { image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 }
+  frameworks: { dynamo: "1.3.1" }
+
+dynamo:
+  version: "1.3.1"
+  install: true
+
+setup_script: vllm-container-deps.sh
+environment: { ETCD_LEASE_TTL: "7200" }
+slurm: { time_limit: "8:00:00" }
+health_check: { max_attempts: 2160, interval_seconds: 10 }
+
+resources:
+  gpu_type: gb200
+  gpus_per_node: 4
+  agg_nodes: 2
+  agg_workers: 1
+  gpus_per_agg: 8
+
+infra:
+  etcd_nats_dedicated_node: false
+  nats_max_payload_mb: 32
+
+frontend:
+  type: dynamo
+  enable_multiple_frontends: false
+  args:
+    router-mode: random
+    router-session-affinity-ttl-secs: 900
+
+backend:
+  type: vllm
+  connector: null
+  dp_launch_mode: per_node
+  aggregated_environment:
+    HF_HUB_CACHE: /hf_hub_cache
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_RPC_TIMEOUT: "600000"
+    VLLM_LOG_STATS_INTERVAL: "1"
+    VLLM_SERVER_DEV_MODE: "1"
+    VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
+    VLLM_USE_NCCL_SYMM_MEM: "1"
+    NCCL_CUMEM_ENABLE: "1"
+    NCCL_MNNVL_ENABLE: "1"
+    NCCL_NVLS_ENABLE: "1"
+    UCX_MEMTYPE_CACHE: "n"
+    UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1"
+    UCX_TLS: "rc,cuda_copy"
+    NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
+  vllm_config:
+    aggregated:
+      served-model-name: deepseek-ai/DeepSeek-V4-Pro
+      kv-cache-dtype: fp8
+      tensor-parallel-size: 1
+      pipeline-parallel-size: 1
+      data-parallel-size: 8
+      enable-expert-parallel: true
+      enable-ep-weight-filter: true
+      moe-backend: deep_gemm_mega_moe
+      attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}'
+      max-model-len: 1048576
+      max-num-seqs: 256
+      max-num-batched-tokens: 8192
+      trust-remote-code: true
+      block-size: 256
+      compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'
+      speculative-config: '{"method":"mtp","num_speculative_tokens":2}'
+      gpu-memory-utilization: 0.90
+      stream-interval: 10
+      no-disable-hybrid-kv-cache-manager: true
+      tokenizer-mode: deepseek_v4
+
+sbatch_directives: { cpus-per-task: "144", mem: "0" }
+srun_options: { container-remap-root: "" }
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
+    AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true"
+    AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
+    HF_HUB_CACHE: /hf_hub_cache
+    WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:"

benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb200-tp8-mtp-agentic.yaml

@@ -0,0 +1,108 @@
+name: dsv4-gb200-vllm-agentic-mtp-agg-tp8
+
+model:
+  path: deepseek-v4-pro
+  container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106
+  precision: fp4
+
+identity:
+  model:
+    repo: deepseek-ai/DeepSeek-V4-Pro
+  container:
+    image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106
+  frameworks:
+    dynamo: "1.3.1"
+
+dynamo:
+  version: "1.3.1"
+  install: true
+
+setup_script: vllm-container-deps.sh
+
+environment:
+  ETCD_LEASE_TTL: "7200"
+
+slurm:
+  time_limit: "8:00:00"
+
+health_check:
+  max_attempts: 2160
+  interval_seconds: 10
+
+resources:
+  gpu_type: gb200
+  gpus_per_node: 4
+  agg_nodes: 2
+  agg_workers: 1
+  gpus_per_agg: 8
+
+infra:
+  etcd_nats_dedicated_node: false
+  nats_max_payload_mb: 32
+
+frontend:
+  type: dynamo
+  enable_multiple_frontends: false
+  args:
+    router-mode: random
+    router-session-affinity-ttl-secs: 900
+
+backend:
+  type: vllm
+  connector: null
+  aggregated_environment:
+    HF_HUB_CACHE: /hf_hub_cache
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_RPC_TIMEOUT: "600000"
+    VLLM_LOG_STATS_INTERVAL: "1"
+    VLLM_SERVER_DEV_MODE: "1"
+    VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
+    VLLM_USE_NCCL_SYMM_MEM: "1"
+    NCCL_CUMEM_ENABLE: "1"
+    NCCL_MNNVL_ENABLE: "1"
+    NCCL_NVLS_ENABLE: "1"
+    UCX_MEMTYPE_CACHE: "n"
+    UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1"
+    UCX_TLS: "rc,cuda_copy"
+    NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
+  vllm_config:
+    aggregated:
+      served-model-name: deepseek-ai/DeepSeek-V4-Pro
+      kv-cache-dtype: fp8
+      tensor-parallel-size: 8
+      pipeline-parallel-size: 1
+      disable-custom-all-reduce: true
+      attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}'
+      max-model-len: 1048576
+      max-num-seqs: 128
+      max-num-batched-tokens: 8192
+      trust-remote-code: true
+      no-enable-flashinfer-autotune: true
+      block-size: 256
+      speculative-config: '{"method":"mtp","num_speculative_tokens":2}'
+      gpu-memory-utilization: 0.90
+      stream-interval: 10
+      no-disable-hybrid-kv-cache-manager: true
+      tokenizer-mode: deepseek_v4
+
+sbatch_directives:
+  cpus-per-task: "144"
+  mem: "0"
+
+srun_options:
+  container-remap-root: ""
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
+    AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true"
+    AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
+    HF_HUB_CACHE: /hf_hub_cache
+    WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:"

benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-1p1d-dep8-dep8-mtp-agentic.yaml

@@ -0,0 +1,118 @@
+name: dsv4-gb200-vllm-agentic-mtp-disagg-1p1d-dep8-dep8
+
+model:
+  path: deepseek-v4-pro
+  container: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106
+  precision: fp4
+
+identity:
+  model: { repo: deepseek-ai/DeepSeek-V4-Pro }
+  container: { image: vllm/vllm-openai:nightly-3ee2df30337a301164c46ae444b76ee67e71c106 }
+  frameworks: { dynamo: "1.3.1" }
+
+dynamo:
+  version: "1.3.1"
+  install: true
+
+setup_script: vllm-container-deps.sh
+environment: { ETCD_LEASE_TTL: "7200" }
+slurm: { time_limit: "8:00:00" }
+health_check: { max_attempts: 2160, interval_seconds: 10 }
+
+resources:
+  gpu_type: gb200
+  gpus_per_node: 4
+  prefill_nodes: 2
+  decode_nodes: 2
+  prefill_workers: 1
+  decode_workers: 1
+  gpus_per_prefill: 8
+  gpus_per_decode: 8
+
+infra:
+  etcd_nats_dedicated_node: false
+  nats_max_payload_mb: 32
+
+frontend:
+  type: dynamo
+  enable_multiple_frontends: false
+  env:
+    # Long AgentX prefills can exceed Dynamo's request-plane default while
+    # the healthy DEP8 worker is still computing the first response.
+    DYN_TCP_REQUEST_TIMEOUT: "60"
+  args:
+    router-mode: random
+    router-session-affinity-ttl-secs: 900
+
+backend:
+  type: vllm
+  connector: null
+  dp_launch_mode: per_node
+  prefill_environment: &worker_environment
+    HF_HUB_CACHE: /hf_hub_cache
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    VLLM_RPC_TIMEOUT: "600000"
+    VLLM_LOG_STATS_INTERVAL: "1"
+    VLLM_SERVER_DEV_MODE: "1"
+    VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
+    VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
+    VLLM_USE_NCCL_SYMM_MEM: "1"
+    NCCL_CUMEM_ENABLE: "1"
+    NCCL_MNNVL_ENABLE: "1"
+    NCCL_NVLS_ENABLE: "1"
+    UCX_MEMTYPE_CACHE: "n"
+    UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1"
+    UCX_TLS: "rc,cuda_copy"
+    NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
+  decode_environment: *worker_environment
+  vllm_config:
+    prefill: &dep8_config
+      kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
+      served-model-name: deepseek-ai/DeepSeek-V4-Pro
+      kv-cache-dtype: fp8
+      tensor-parallel-size: 1
+      pipeline-parallel-size: 1
+      data-parallel-size: 8
+      enable-expert-parallel: true
+      enable-ep-weight-filter: true
+      moe-backend: deep_gemm_mega_moe
+      attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}'
+      max-model-len: 1048576
+      max-num-seqs: 16
+      max-num-batched-tokens: 16384
+      trust-remote-code: true
+      enforce-eager: true
+      block-size: 256
+      gpu-memory-utilization: 0.95
+      no-disable-hybrid-kv-cache-manager: true
+      enable-sleep-mode: true
+      tokenizer-mode: deepseek_v4
+      speculative-config: '{"method":"mtp","num_speculative_tokens":2}'
+    decode:
+      <<: *dep8_config
+      attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_fp4_indexer_cache":true}'
+      enforce-eager: false
+      gpu-memory-utilization: 0.90
+      max-num-seqs: 1024
+      max-num-batched-tokens: 1024
+      compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
+      max-cudagraph-capture-size: 1024
+      stream-interval: 10
+
+sbatch_directives: { cpus-per-task: "144", mem: "0" }
+srun_options: { container-remap-root: "" }
+
+benchmark:
+  type: custom
+  command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
+  env:
+    INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
+    RESULT_DIR: /logs/agentic
+    PORT: "8000"
+    IS_MULTINODE: "true"
+    AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
+    AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true"
+    AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache
+    HF_HUB_CACHE: /hf_hub_cache
+    WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126
+    AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:"