diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml
deleted file mode 100644
index 192e0f5b95..0000000000
--- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml
+++ /dev/null
@@ -1,146 +0,0 @@
-name: "svf-vllm-agg-gb300-tp4-mtp-agentic"
-
-# GB300 AgentX aggregate topology: one TP4 worker occupies one four-GPU node
-# and serves both prefill and decode at concurrency 4. Scheduler, CUDA-graph,
-# and memory settings match the B300 vLLM TP4 MTP agentic configuration.
-
-model:
- path: "deepseek-v4-pro"
- container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f"
- precision: "fp4"
-
-identity:
- model:
- repo: "deepseek-ai/DeepSeek-V4-Pro"
- container:
- image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f"
- frameworks:
- dynamo: "1.2.1"
-
-dynamo:
- wheel: "1.2.1"
- install: true
-
-environment:
- DYNAMO_WHEEL_DIRS: "/srtctl-wheels"
- ETCD_LEASE_TTL: "7200"
-
-setup_script: vllm-container-deps.sh
-
-slurm:
- time_limit: "8:00:00"
-
-health_check:
- max_attempts: 2160
- interval_seconds: 10
-
-resources:
- gpu_type: "gb300"
- gpus_per_node: 4
- agg_nodes: 1
- agg_workers: 1
- gpus_per_agg: 4
-
-infra:
- etcd_nats_dedicated_node: false
- nats_max_payload_mb: 32
-
-frontend:
- type: dynamo
- enable_multiple_frontends: false
- args:
- router-mode: "kv"
- router-reset-states: true
- router-temperature: 0.0
- router-queue-threshold: 65536
- active-decode-blocks-threshold: "None"
- active-prefill-tokens-threshold: "None"
- active-prefill-tokens-threshold-frac: "None"
- tokenizer: "fastokens"
-
-backend:
- type: vllm
- connector: null
- mooncake_kv_store:
- store_config:
- metadata_server: "P2PHANDSHAKE"
- global_segment_size: "150GB"
- local_buffer_size: "4GB"
- protocol: "rdma"
- device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
- mode: "embedded"
- enable_offload: false
- aggregated_environment:
- HF_HUB_CACHE: "/hf_hub_cache"
- HUGGINGFACE_HUB_CACHE: "/hf_hub_cache"
- TRANSFORMERS_CACHE: "/hf_hub_cache"
- VLLM_ENGINE_READY_TIMEOUT_S: "3600"
- VLLM_RPC_TIMEOUT: "600000"
- VLLM_LOG_STATS_INTERVAL: "1"
- VLLM_V2_WARMUP_MAX_NUM_SEQS: "16"
- TILELANG_CLEANUP_TEMP_FILES: "1"
- VLLM_USE_NCCL_SYMM_MEM: "0"
- TORCH_SYMMMEM: "NVSHMEM"
- NCCL_CUMEM_ENABLE: "1"
- NCCL_MNNVL_ENABLE: "1"
- NCCL_NVLS_ENABLE: "1"
- VLLM_SERVER_DEV_MODE: "1"
- VLLM_USE_V2_MODEL_RUNNER: "1"
- VLLM_USE_RUST_FRONTEND: "1"
- VLLM_ALLREDUCE_USE_FLASHINFER: "1"
- VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto"
- VLLM_MOONCAKE_LOAD_RECV_THREADS: "20"
- VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
- VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024"
- VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
- UCX_MEMTYPE_CACHE: "n"
- UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1"
- UCX_TLS: "rc,cuda_copy"
- NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
- NCCL_P2P_LEVEL: "NVL"
- MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
- MC_STORE_CLIENT_METRIC: "1"
- MC_STORE_CLIENT_METRIC_INTERVAL: "5"
- MC_TE_METRIC: "0"
- DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp4-mtp-{job_id}"
- vllm_config:
- aggregated:
- served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
- kv-cache-dtype: "fp8"
- tensor-parallel-size: 4
- pipeline-parallel-size: 1
- disable-custom-all-reduce: true
- enable-cumem-allocator: true
- attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}'
- max-model-len: 1048576
- max-num-seqs: 16
- max-num-batched-tokens: 8192
- trust-remote-code: true
- no-enable-flashinfer-autotune: true
- block-size: 256
- compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}'
- speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}'
- gpu-memory-utilization: 0.93
- stream-interval: 10
- no-disable-hybrid-kv-cache-manager: true
- tokenizer-mode: "deepseek_v4"
-
-sbatch_directives:
- cpus-per-task: "72"
- mem: "0"
-
-srun_options:
- container-remap-root: ""
-
-benchmark:
- type: custom
- command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
- env:
- INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
- RESULT_DIR: "/logs/agentic"
- PORT: "8000"
- IS_MULTINODE: "true"
- AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
- AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
- HF_HUB_CACHE: "/hf_hub_cache"
- WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"
diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml
deleted file mode 100644
index 6a950787fe..0000000000
--- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml
+++ /dev/null
@@ -1,149 +0,0 @@
-name: "svf-vllm-agg-gb300-tp8-mtp-agentic"
-
-# Validated GB300 AgentX aggregate topology: one TP8 worker spans two
-# four-GPU nodes and serves both prefill and decode at concurrency 1.
-
-model:
- path: "deepseek-v4-pro"
- container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f"
- precision: "fp4"
-
-identity:
- model:
- repo: "deepseek-ai/DeepSeek-V4-Pro"
- container:
- image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f"
- frameworks:
- dynamo: "1.2.1"
-
-dynamo:
- wheel: "1.2.1"
- install: true
-
-environment:
- DYNAMO_WHEEL_DIRS: "/srtctl-wheels"
- # The frontend shares Grace CPU capacity with the long TP8 cold start.
- ETCD_LEASE_TTL: "7200"
-
-setup_script: vllm-container-deps.sh
-
-slurm:
- time_limit: "8:00:00"
-
-health_check:
- max_attempts: 2160
- interval_seconds: 10
-
-resources:
- gpu_type: "gb300"
- gpus_per_node: 4
- agg_nodes: 2
- agg_workers: 1
- gpus_per_agg: 8
-
-infra:
- etcd_nats_dedicated_node: false
- nats_max_payload_mb: 32
-
-frontend:
- type: dynamo
- enable_multiple_frontends: false
- args:
- router-mode: "kv"
- router-reset-states: true
- router-temperature: 0.0
- router-queue-threshold: 65536
- active-decode-blocks-threshold: "None"
- active-prefill-tokens-threshold: "None"
- active-prefill-tokens-threshold-frac: "None"
- tokenizer: "fastokens"
-
-backend:
- type: vllm
- connector: null
- mooncake_kv_store:
- store_config:
- metadata_server: "P2PHANDSHAKE"
- global_segment_size: "150GB"
- local_buffer_size: "4GB"
- protocol: "rdma"
- device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
- mode: "embedded"
- enable_offload: false
- aggregated_environment:
- HF_HUB_CACHE: "/hf_hub_cache"
- HUGGINGFACE_HUB_CACHE: "/hf_hub_cache"
- TRANSFORMERS_CACHE: "/hf_hub_cache"
- VLLM_ENGINE_READY_TIMEOUT_S: "3600"
- VLLM_RPC_TIMEOUT: "600000"
- VLLM_LOG_STATS_INTERVAL: "1"
- VLLM_V2_WARMUP_MAX_NUM_SEQS: "32"
- TILELANG_CLEANUP_TEMP_FILES: "1"
- VLLM_USE_NCCL_SYMM_MEM: "0"
- TORCH_SYMMMEM: "NVSHMEM"
- NCCL_CUMEM_ENABLE: "1"
- NCCL_MNNVL_ENABLE: "1"
- NCCL_NVLS_ENABLE: "1"
- VLLM_SERVER_DEV_MODE: "1"
- VLLM_USE_V2_MODEL_RUNNER: "1"
- VLLM_USE_RUST_FRONTEND: "1"
- VLLM_ALLREDUCE_USE_FLASHINFER: "1"
- VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto"
- VLLM_MOONCAKE_LOAD_RECV_THREADS: "20"
- VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
- VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024"
- VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
- UCX_MEMTYPE_CACHE: "n"
- UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1"
- UCX_TLS: "rc,cuda_copy"
- NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
- NCCL_P2P_LEVEL: "NVL"
- MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
- MC_STORE_CLIENT_METRIC: "1"
- MC_STORE_CLIENT_METRIC_INTERVAL: "5"
- MC_TE_METRIC: "0"
- DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-mtp-{job_id}"
- vllm_config:
- aggregated:
- served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
- kv-cache-dtype: "fp8"
- tensor-parallel-size: 8
- pipeline-parallel-size: 1
- disable-custom-all-reduce: true
- enable-cumem-allocator: true
- attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}'
- max-model-len: 1048576
- max-num-seqs: 32
- max-num-batched-tokens: 8192
- trust-remote-code: true
- no-enable-flashinfer-autotune: true
- block-size: 256
- compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
- max-cudagraph-capture-size: 128
- speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}'
- gpu-memory-utilization: 0.90
- stream-interval: 10
- no-disable-hybrid-kv-cache-manager: true
- tokenizer-mode: "deepseek_v4"
-
-sbatch_directives:
- cpus-per-task: "144"
- mem: "0"
-
-srun_options:
- container-remap-root: ""
-
-benchmark:
- type: custom
- command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
- env:
- INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
- RESULT_DIR: "/logs/agentic"
- PORT: "8000"
- # Keep aggregate workers in the multinode result schema so ingestion uses
- # the zero decode-worker count instead of duplicating TP into P and D.
- IS_MULTINODE: "true"
- AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
- AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
- HF_HUB_CACHE: "/hf_hub_cache"
- WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"
diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml
deleted file mode 100644
index b647276179..0000000000
--- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml
+++ /dev/null
@@ -1,161 +0,0 @@
-name: "svf-vllm-disagg-gb200-2p1d-dep8-dep8-agentic"
-
-# High-throughput topology: two DEP8 prefill workers and one
-# DEP8 decode worker. All three workers use attention DP8 / expert EP8, for
-# 24 inference GPUs total. This makes the previously manual-only DEP/DEP
-# experiment reproducible while the June 21 AgentX frontier is retuned.
-
-model:
- path: "deepseek-v4-pro"
- container: "vllm/vllm-openai:v0.23.0"
- precision: "fp4"
-
-dynamo:
- install: true
- wheel: "1.3.0.dev20260618"
-
-setup_script: vllm-container-deps.sh
-
-slurm:
- time_limit: "8:00:00"
-
-health_check:
- max_attempts: 1440
- interval_seconds: 10
-
-resources:
- gpu_type: "gb200"
- gpus_per_node: 4
- prefill_nodes: 4
- decode_nodes: 2
- prefill_workers: 2
- decode_workers: 1
- gpus_per_prefill: 8
- gpus_per_decode: 8
-
-infra:
- etcd_nats_dedicated_node: false
- nats_max_payload_mb: 32
-
-frontend:
- type: dynamo
- enable_multiple_frontends: false
- env:
- DYN_REQUEST_PLANE: "tcp"
- # Dynamo's 10-second default can expire during brief etcd stalls while
- # many long-loading DEP ranks initialize. Slurm still detects hard exits.
- ETCD_LEASE_TTL: "120"
- args:
- router-mode: "kv"
- router-reset-states: true
-
-backend:
- type: vllm
- connector: null
- prefill_environment:
- DYN_REQUEST_PLANE: "tcp"
- ETCD_LEASE_TTL: "120"
- TILELANG_CLEANUP_TEMP_FILES: "1"
- VLLM_USE_NCCL_SYMM_MEM: "1"
- TORCH_SYMMMEM: "NVSHMEM"
- NCCL_CUMEM_ENABLE: "1"
- NCCL_MNNVL_ENABLE: "1"
- NCCL_NVLS_ENABLE: "1"
- VLLM_SERVER_DEV_MODE: "1"
- VLLM_USE_V2_MODEL_RUNNER: "1"
- # Long AgentX requests can make one DEP rank hold the EP step beyond
- # vLLM's 300-second model-execution timeout.
- VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
- VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
- VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024"
- VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048"
- UCX_MEMTYPE_CACHE: "n"
- UCX_MEMTYPE_REG_WHOLE: "n"
- UCX_RCACHE_MAX_UNRELEASED: "1024"
- UCX_TLS: "cuda_copy,rc"
- NCCL_P2P_LEVEL: NVL
- decode_environment:
- DYN_REQUEST_PLANE: "tcp"
- ETCD_LEASE_TTL: "120"
- TILELANG_CLEANUP_TEMP_FILES: "1"
- VLLM_USE_NCCL_SYMM_MEM: "1"
- TORCH_SYMMMEM: "NVSHMEM"
- NCCL_CUMEM_ENABLE: "1"
- NCCL_MNNVL_ENABLE: "1"
- NCCL_NVLS_ENABLE: "1"
- VLLM_SERVER_DEV_MODE: "1"
- VLLM_USE_V2_MODEL_RUNNER: "1"
- VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
- VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
- UCX_MEMTYPE_CACHE: "n"
- UCX_MEMTYPE_REG_WHOLE: "n"
- UCX_RCACHE_MAX_UNRELEASED: "1024"
- UCX_TLS: "cuda_copy,rc"
- NCCL_P2P_LEVEL: NVL
-
- vllm_config:
- prefill:
- kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
- kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}'
- served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
- kv-cache-dtype: "fp8"
- tensor-parallel-size: 1
- pipeline-parallel-size: 1
- data-parallel-size: 8
- data-parallel-rpc-port: 13345
- enable-expert-parallel: true
- enable-ep-weight-filter: true
- attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}'
- # Let vLLM select max-num-seqs for the long-context AgentX trace.
- # max-num-seqs: 256
- # DEP8 holds a full model replica per GPU. A 16K prefill step requires
- # a 9.84 GiB FP4 MoE intermediate and OOMs before serving traffic.
- max-num-batched-tokens: 8192
- trust-remote-code: true
- no-enable-flashinfer-autotune: true
- block-size: 256
- compilation-config: '{"cudagraph_mode":"PIECEWISE"}'
- gpu-memory-utilization: 0.9
- no-disable-hybrid-kv-cache-manager: true
- tokenizer-mode: deepseek_v4
- decode:
- kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
- served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
- kv-cache-dtype: "fp8"
- tensor-parallel-size: 1
- pipeline-parallel-size: 1
- data-parallel-size: 8
- data-parallel-rpc-port: 13345
- enable-expert-parallel: true
- enable-ep-weight-filter: true
- attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_fp4_indexer_cache": true}'
- # Let vLLM select max-num-seqs and max-num-batched-tokens.
- # max-num-seqs: 512
- trust-remote-code: true
- no-enable-flashinfer-autotune: true
- block-size: 256
- compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
- gpu-memory-utilization: 0.9
- stream-interval: 10
- no-disable-hybrid-kv-cache-manager: true
- tokenizer-mode: deepseek_v4
-
-sbatch_directives:
- cpus-per-task: "72"
-
-srun_options:
- container-remap-root: ""
-
-benchmark:
- type: custom
- aiperf_server_metrics: true
- command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
- env:
- INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
- RESULT_DIR: /logs/agentic
- PORT: "8000"
- IS_MULTINODE: "true"
- AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400"
- AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
- HF_HUB_CACHE: "/hf_hub_cache"
- WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"
diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml
deleted file mode 100644
index 9e2b665843..0000000000
--- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml
+++ /dev/null
@@ -1,155 +0,0 @@
-name: "svf-vllm-disagg-gb200-3p2d-tep8-tp8-agentic"
-
-# Middle/high-interactivity topology: three cache-affinitized TEP8 prefill
-# workers and two independent TP8 decode workers, for 40 inference GPUs.
-# The dense historical curve covered roughly 47--71 output tok/s/user through
-# c16--c80. This recipe re-establishes that curve on the June 21 AgentX corpus
-# with the current vLLM/Dynamo runtime contract.
-
-model:
- path: "deepseek-v4-pro"
- container: "vllm/vllm-openai:v0.23.0"
- precision: "fp4"
-
-dynamo:
- install: true
- wheel: "1.3.0.dev20260618"
-
-setup_script: vllm-container-deps.sh
-
-slurm:
- time_limit: "8:00:00"
-
-health_check:
- max_attempts: 1440
- interval_seconds: 10
-
-resources:
- gpu_type: "gb200"
- gpus_per_node: 4
- prefill_nodes: 6
- decode_nodes: 4
- prefill_workers: 3
- decode_workers: 2
- gpus_per_prefill: 8
- gpus_per_decode: 8
-
-infra:
- etcd_nats_dedicated_node: false
- nats_max_payload_mb: 32
-
-frontend:
- type: dynamo
- enable_multiple_frontends: false
- env:
- DYN_REQUEST_PLANE: "tcp"
- ETCD_LEASE_TTL: "120"
- args:
- router-mode: "kv"
- router-reset-states: true
-
-backend:
- type: vllm
- connector: null
- prefill_environment:
- DYN_REQUEST_PLANE: "tcp"
- ETCD_LEASE_TTL: "120"
- TILELANG_CLEANUP_TEMP_FILES: "1"
- VLLM_USE_NCCL_SYMM_MEM: "1"
- TORCH_SYMMMEM: "NVSHMEM"
- NCCL_CUMEM_ENABLE: "1"
- NCCL_MNNVL_ENABLE: "1"
- NCCL_NVLS_ENABLE: "1"
- VLLM_SERVER_DEV_MODE: "1"
- VLLM_USE_V2_MODEL_RUNNER: "1"
- VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
- VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
- VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024"
- VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048"
- UCX_MEMTYPE_CACHE: "n"
- UCX_MEMTYPE_REG_WHOLE: "n"
- UCX_RCACHE_MAX_UNRELEASED: "1024"
- UCX_TLS: "cuda_copy,rc"
- NCCL_P2P_LEVEL: NVL
- decode_environment:
- DYN_REQUEST_PLANE: "tcp"
- ETCD_LEASE_TTL: "120"
- TILELANG_CLEANUP_TEMP_FILES: "1"
- VLLM_USE_NCCL_SYMM_MEM: "1"
- TORCH_SYMMMEM: "NVSHMEM"
- NCCL_CUMEM_ENABLE: "1"
- NCCL_MNNVL_ENABLE: "1"
- NCCL_NVLS_ENABLE: "1"
- VLLM_SERVER_DEV_MODE: "1"
- VLLM_USE_V2_MODEL_RUNNER: "1"
- VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800"
- VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
- UCX_MEMTYPE_CACHE: "n"
- UCX_MEMTYPE_REG_WHOLE: "n"
- UCX_RCACHE_MAX_UNRELEASED: "1024"
- UCX_TLS: "cuda_copy,rc"
- NCCL_P2P_LEVEL: NVL
-
- vllm_config:
- prefill:
- kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
- kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}'
- served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
- kv-cache-dtype: "fp8"
- tensor-parallel-size: 8
- pipeline-parallel-size: 1
- enable-expert-parallel: true
- enable-ep-weight-filter: true
- attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}'
- # Let vLLM select max-num-seqs for the long-context AgentX trace.
- # max-num-seqs: 16
- max-num-batched-tokens: 16384
- trust-remote-code: true
- no-enable-flashinfer-autotune: true
- block-size: 256
- # FULL capture conflicts with NCCL symmetric-memory registration on the
- # TP prefill path; PIECEWISE preserves compilation and safe graph regions.
- compilation-config: '{"cudagraph_mode":"PIECEWISE"}'
- gpu-memory-utilization: 0.9
- no-disable-hybrid-kv-cache-manager: true
- tokenizer-mode: deepseek_v4
- decode:
- kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
- served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
- kv-cache-dtype: "fp8"
- tensor-parallel-size: 8
- pipeline-parallel-size: 1
- attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_fp4_indexer_cache": true}'
- # Let vLLM select max-num-seqs and max-num-batched-tokens.
- # max-num-seqs: 512
- trust-remote-code: true
- no-enable-flashinfer-autotune: true
- block-size: 256
- # TP8 all-reduce uses NCCL symmetric memory, which is incompatible with
- # the full graph-capture path. Keep graph-incompatible collectives eager.
- compilation-config: '{"cudagraph_mode":"PIECEWISE"}'
- gpu-memory-utilization: 0.9
- stream-interval: 10
- all2all-backend: "flashinfer_nvlink_one_sided"
- no-disable-hybrid-kv-cache-manager: true
- tokenizer-mode: deepseek_v4
-
-sbatch_directives:
- cpus-per-task: "72"
-
-srun_options:
- container-remap-root: ""
-
-benchmark:
- type: custom
- aiperf_server_metrics: true
- command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
- env:
- INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
- RESULT_DIR: /logs/agentic
- PORT: "8000"
- IS_MULTINODE: "true"
- AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400"
- AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
- HF_HUB_CACHE: "/hf_hub_cache"
- WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"
diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml
deleted file mode 100644
index dd58dde28f..0000000000
--- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml
+++ /dev/null
@@ -1,203 +0,0 @@
-name: "svf-vllm-disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic"
-
-# Validated GB300 AgentX MTP3 low-latency topology: one DEP4 prefill worker
-# feeds one DEP8 decode worker at concurrency 128.
-
-model:
- path: "deepseek-v4-pro"
- container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f"
- precision: "fp4"
-
-identity:
- model:
- repo: "deepseek-ai/DeepSeek-V4-Pro"
- container:
- image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f"
- frameworks:
- dynamo: "1.3.0.dev20260720"
-
-dynamo:
- version: "1.3.0.dev20260720"
- install: true
-
-setup_script: vllm-container-deps.sh
-
-slurm:
- time_limit: "8:00:00"
-
-health_check:
- max_attempts: 2160
- interval_seconds: 10
-
-resources:
- gpu_type: "gb300"
- gpus_per_node: 4
- het_jobs: false
- spread_workers: false
- prefill_nodes: 1
- decode_nodes: 2
- prefill_workers: 1
- decode_workers: 1
- gpus_per_prefill: 4
- gpus_per_decode: 8
-
-infra:
- etcd_nats_dedicated_node: false
- nats_max_payload_mb: 32
-
-environment:
- # Mooncake prefix-block hashes must match across processes and nodes.
- PYTHONHASHSEED: "0"
-
-frontend:
- type: dynamo
- enable_multiple_frontends: false
- args:
- router-mode: "random"
- router-session-affinity-ttl-secs: 900
- env:
- DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600"
-
-backend:
- type: vllm
- connector: null
- dp_launch_mode: per_node
- mooncake_kv_store:
- store_config:
- metadata_server: "P2PHANDSHAKE"
- global_segment_size: "150GB"
- local_buffer_size: "4GB"
- protocol: "rdma"
- device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
- mode: "embedded"
- enable_offload: false
- vllm_config:
- prefill:
- kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}'
- served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
- kv-cache-dtype: "fp8"
- tensor-parallel-size: 1
- pipeline-parallel-size: 1
- data-parallel-size: 4
- data-parallel-rpc-port: 13345
- enable-cumem-allocator: true
- enable-expert-parallel: true
- enable-ep-weight-filter: true
- max-model-len: 1048576
- max-num-seqs: 64
- max-num-batched-tokens: 8192
- trust-remote-code: true
- no-enable-flashinfer-autotune: true
- block-size: 256
- max-cudagraph-capture-size: 256
- gpu-memory-utilization: 0.92
- no-disable-hybrid-kv-cache-manager: true
- tokenizer-mode: "deepseek_v4"
- attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}'
- speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}'
- moe-backend: "deep_gemm_amxf4_mega_moe"
- numa-bind: true
- numa-bind-nodes: [0, 0, 1, 1]
- decode:
- kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}'
- served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
- kv-cache-dtype: "fp8"
- tensor-parallel-size: 1
- pipeline-parallel-size: 1
- data-parallel-size: 8
- data-parallel-rpc-port: 13345
- enable-cumem-allocator: true
- enable-expert-parallel: true
- enable-ep-weight-filter: true
- max-model-len: 1048576
- max-num-seqs: 256
- max-num-batched-tokens: 1024
- trust-remote-code: true
- no-enable-flashinfer-autotune: true
- block-size: 256
- compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
- max-cudagraph-capture-size: 1024
- gpu-memory-utilization: 0.92
- no-disable-hybrid-kv-cache-manager: true
- tokenizer-mode: "deepseek_v4"
- attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}'
- speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}'
- moe-backend: "deep_gemm_amxf4_mega_moe"
- numa-bind: true
- numa-bind-nodes: [0, 0, 1, 1]
- prefill_environment:
- HF_HUB_CACHE: "/hf_hub_cache"
- HUGGINGFACE_HUB_CACHE: "/hf_hub_cache"
- TRANSFORMERS_CACHE: "/hf_hub_cache"
- VLLM_ENGINE_READY_TIMEOUT_S: "3600"
- VLLM_RPC_TIMEOUT: "600000"
- VLLM_LOG_STATS_INTERVAL: "1"
- VLLM_V2_WARMUP_MAX_NUM_SEQS: "20"
- VLLM_SERVER_DEV_MODE: "1"
- VLLM_USE_V2_MODEL_RUNNER: "1"
- VLLM_MOONCAKE_LOAD_RECV_THREADS: "20"
- VLLM_ALLREDUCE_USE_SYMM_MEM: "0"
- UCX_MEMTYPE_CACHE: "n"
- UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1"
- UCX_TLS: "rc,cuda_copy"
- NCCL_CUMEM_ENABLE: "1"
- NCCL_MNNVL_ENABLE: "1"
- NCCL_NVLS_ENABLE: "1"
- NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
- VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1"
- VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
- VLLM_USE_BREAKABLE_CUDAGRAPH: "0"
- VLLM_CONNECTOR_PREFETCH_DEPTH: "8"
- VLLM_DSV4_MEGA_FP8_COMBINE: "1"
- DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-c128-mtp-{job_id}"
- MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
- MC_STORE_CLIENT_METRIC: "1"
- MC_STORE_CLIENT_METRIC_INTERVAL: "5"
- MC_TE_METRIC: "0"
- decode_environment:
- HF_HUB_CACHE: "/hf_hub_cache"
- HUGGINGFACE_HUB_CACHE: "/hf_hub_cache"
- TRANSFORMERS_CACHE: "/hf_hub_cache"
- VLLM_ENGINE_READY_TIMEOUT_S: "3600"
- VLLM_RPC_TIMEOUT: "600000"
- VLLM_LOG_STATS_INTERVAL: "1"
- VLLM_V2_WARMUP_MAX_NUM_SEQS: "20"
- VLLM_SERVER_DEV_MODE: "1"
- VLLM_USE_V2_MODEL_RUNNER: "1"
- VLLM_MOONCAKE_LOAD_RECV_THREADS: "20"
- VLLM_ALLREDUCE_USE_SYMM_MEM: "0"
- UCX_MEMTYPE_CACHE: "n"
- UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1"
- UCX_TLS: "rc,cuda_copy"
- NCCL_CUMEM_ENABLE: "1"
- NCCL_MNNVL_ENABLE: "1"
- NCCL_NVLS_ENABLE: "1"
- NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
- VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1"
- VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
- VLLM_DSV4_MEGA_FP8_COMBINE: "1"
- DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-c128-mtp-{job_id}"
- MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
- MC_STORE_CLIENT_METRIC: "1"
- MC_STORE_CLIENT_METRIC_INTERVAL: "5"
- MC_TE_METRIC: "0"
-
-sbatch_directives:
- cpus-per-task: "72"
-
-srun_options:
- container-remap-root: ""
-
-benchmark:
- type: custom
- command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
- env:
- INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
- RESULT_DIR: "/logs/agentic"
- PORT: "8000"
- IS_MULTINODE: "true"
- AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
- AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true"
- AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
- HF_HUB_CACHE: "/hf_hub_cache"
- WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"
diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml
deleted file mode 100644
index 00a876ebe8..0000000000
--- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml
+++ /dev/null
@@ -1,204 +0,0 @@
-name: "svf-vllm-disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic"
-
-# Validated GB300 AgentX MTP3 efficiency topology: one DEP8 prefill worker and
-# one DEP8 decode worker at concurrency 384.
-
-model:
- path: "deepseek-v4-pro"
- container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f"
- precision: "fp4"
-
-identity:
- model:
- repo: "deepseek-ai/DeepSeek-V4-Pro"
- container:
- image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f"
- frameworks:
- dynamo: "1.3.0.dev20260720"
-
-dynamo:
- version: "1.3.0.dev20260720"
- install: true
-
-setup_script: vllm-container-deps.sh
-
-slurm:
- time_limit: "8:00:00"
-
-health_check:
- max_attempts: 2160
- interval_seconds: 10
-
-resources:
- gpu_type: "gb300"
- gpus_per_node: 4
- het_jobs: false
- spread_workers: false
- prefill_nodes: 2
- decode_nodes: 2
- prefill_workers: 1
- decode_workers: 1
- gpus_per_prefill: 8
- gpus_per_decode: 8
-
-infra:
- etcd_nats_dedicated_node: false
- nats_max_payload_mb: 32
-
-environment:
- # Mooncake prefix-block hashes must match across processes and nodes.
- PYTHONHASHSEED: "0"
-
-frontend:
- type: dynamo
- enable_multiple_frontends: false
- args:
- router-mode: "random"
- router-session-affinity-ttl-secs: 900
- env:
- DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600"
-
-backend:
- type: vllm
- connector: null
- dp_launch_mode: per_node
- mooncake_kv_store:
- store_config:
- metadata_server: "P2PHANDSHAKE"
- global_segment_size: "150GB"
- local_buffer_size: "4GB"
- protocol: "rdma"
- device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
- mode: "embedded"
- enable_offload: false
- vllm_config:
- prefill:
- kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}'
- served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
- kv-cache-dtype: "fp8"
- tensor-parallel-size: 1
- pipeline-parallel-size: 1
- data-parallel-size: 8
- data-parallel-rpc-port: 13345
- enable-cumem-allocator: true
- enable-expert-parallel: true
- enable-ep-weight-filter: true
- max-model-len: 1048576
- max-num-seqs: 32
- max-num-batched-tokens: 8192
- trust-remote-code: true
- no-enable-flashinfer-autotune: true
- block-size: 256
- max-cudagraph-capture-size: 128
- gpu-memory-utilization: 0.92
- no-disable-hybrid-kv-cache-manager: true
- tokenizer-mode: "deepseek_v4"
- attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}'
- speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}'
- moe-backend: "deep_gemm_amxf4_mega_moe"
- numa-bind: true
- numa-bind-nodes: [0, 0, 1, 1]
- decode:
- kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}'
- served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
- kv-cache-dtype: "fp8"
- tensor-parallel-size: 1
- pipeline-parallel-size: 1
- data-parallel-size: 8
- data-parallel-rpc-port: 13345
- enable-cumem-allocator: true
- enable-expert-parallel: true
- enable-ep-weight-filter: true
- max-model-len: 1048576
- max-num-seqs: 256
- max-num-batched-tokens: 1024
- trust-remote-code: true
- no-enable-flashinfer-autotune: true
- block-size: 256
- compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
- max-cudagraph-capture-size: 1024
- gpu-memory-utilization: 0.92
- no-disable-hybrid-kv-cache-manager: true
- tokenizer-mode: "deepseek_v4"
- attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}'
- speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}'
- moe-backend: "deep_gemm_amxf4_mega_moe"
- numa-bind: true
- numa-bind-nodes: [0, 0, 1, 1]
- prefill_environment:
- HF_HUB_CACHE: "/hf_hub_cache"
- HUGGINGFACE_HUB_CACHE: "/hf_hub_cache"
- TRANSFORMERS_CACHE: "/hf_hub_cache"
- VLLM_ENGINE_READY_TIMEOUT_S: "3600"
- VLLM_RPC_TIMEOUT: "600000"
- VLLM_LOG_STATS_INTERVAL: "1"
- VLLM_V2_WARMUP_MAX_NUM_SEQS: "20"
- VLLM_SERVER_DEV_MODE: "1"
- VLLM_USE_V2_MODEL_RUNNER: "1"
- VLLM_MOONCAKE_LOAD_RECV_THREADS: "20"
- VLLM_ALLREDUCE_USE_SYMM_MEM: "0"
- UCX_MEMTYPE_CACHE: "n"
- UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1"
- UCX_TLS: "rc,cuda_copy"
- NCCL_CUMEM_ENABLE: "1"
- NCCL_MNNVL_ENABLE: "1"
- NCCL_NVLS_ENABLE: "1"
- NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
- VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1"
- VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
- VLLM_USE_BREAKABLE_CUDAGRAPH: "0"
- VLLM_CONNECTOR_PREFETCH_DEPTH: "8"
- VLLM_DSV4_MEGA_FP8_COMBINE: "1"
- DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-mtp-{job_id}"
- MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
- MC_STORE_CLIENT_METRIC: "1"
- MC_STORE_CLIENT_METRIC_INTERVAL: "5"
- MC_TE_METRIC: "0"
- decode_environment:
- HF_HUB_CACHE: "/hf_hub_cache"
- HUGGINGFACE_HUB_CACHE: "/hf_hub_cache"
- TRANSFORMERS_CACHE: "/hf_hub_cache"
- VLLM_ENGINE_READY_TIMEOUT_S: "3600"
- VLLM_RPC_TIMEOUT: "600000"
- VLLM_LOG_STATS_INTERVAL: "1"
- VLLM_V2_WARMUP_MAX_NUM_SEQS: "20"
- VLLM_SERVER_DEV_MODE: "1"
- VLLM_USE_V2_MODEL_RUNNER: "1"
- VLLM_MOONCAKE_LOAD_RECV_THREADS: "20"
- VLLM_ALLREDUCE_USE_SYMM_MEM: "0"
- UCX_MEMTYPE_CACHE: "n"
- UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1"
- UCX_TLS: "rc,cuda_copy"
- NCCL_CUMEM_ENABLE: "1"
- NCCL_MNNVL_ENABLE: "1"
- NCCL_NVLS_ENABLE: "1"
- NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3"
- VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1"
- VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768"
- VLLM_DSV4_MEGA_FP8_COMBINE: "1"
- DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-mtp-{job_id}"
- MC_ENABLE_DEST_DEVICE_AFFINITY: "1"
- MC_STORE_CLIENT_METRIC: "1"
- MC_STORE_CLIENT_METRIC_INTERVAL: "5"
- MC_TE_METRIC: "0"
-
-sbatch_directives:
- cpus-per-task: "72"
- mem: "0"
-
-srun_options:
- container-remap-root: ""
-
-benchmark:
- type: custom
- command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
- env:
- INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
- RESULT_DIR: "/logs/agentic"
- PORT: "8000"
- IS_MULTINODE: "true"
- AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0"
- AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true"
- AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
- HF_HUB_CACHE: "/hf_hub_cache"
- WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126"
diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tp4-agentic.yaml
deleted file mode 100644
index 13c8d353e3..0000000000
--- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tp4-agentic.yaml
+++ /dev/null
@@ -1,177 +0,0 @@
-name: "svf-vllm-disagg-gb300-1p6d-dep4-tp4-agentic"
-
-# Agentic-coding variant of vllm/deepseek-v4/8k1k/disagg-gb300-1p6d-dep4-tp4.yaml.
-# Topology is identical (1 prefill DEP=4 + 6 decode TP=4, 28 GPUs across 7
-# GB300 nodes + 1 dedicated NATS/etcd infra node) so we can compare against
-# the fixed-seq-len 1p6d baseline at the same concurrency point (192).
-#
-# Divergence vs the 8k1k sibling:
-# - benchmark.type: sa-bench -> custom (hands off to agentic_srt.sh)
-# - max-model-len: removed (let vLLM derive from model config; agentic
-# trajectories blow past any small explicit cap)
-# - no-enable-prefix-caching: dropped (prefix caching MUST be on for
-# trajectory reuse — entire point of agentic)
-# Note: --enable-auto-tool-choice / --tool-call-parser / --reasoning-parser
-# are NOT set on the worker. The dynamo-vllm worker entrypoint doesn't
-# accept them (different arg parser than `vllm serve`). In disagg, chat
-# parsing happens at the dynamo frontend, not at the worker.
-
-model:
- path: "deepseek-v4-pro"
- container: "vllm/vllm-openai:v0.21.0-ubuntu2404"
- precision: "fp4"
-
-dynamo:
- install: true
- wheel: "1.2.0.dev20260426"
-
-setup_script: vllm-container-deps.sh
-
-slurm:
- time_limit: "8:00:00"
-
-health_check:
- max_attempts: 1440
- interval_seconds: 10
-
-resources:
- gpu_type: "gb300"
- gpus_per_node: 4
- prefill_nodes: 1
- decode_nodes: 6
- prefill_workers: 1
- decode_workers: 6
- gpus_per_prefill: 4
- gpus_per_decode: 4
-
-infra:
- etcd_nats_dedicated_node: true
- # Raise NATS server max_payload from the 1 MiB default to 32 MiB.
- # Agentic prompts at 50k-200k DSv4 tokens serialize to JSON at ~10-15
- # bytes/token, easily clearing 1-3 MB per request. Without this, every
- # long-prompt prefill RPC gets rejected by the NATS server with
- # "maximum payload exceeded" (visible in infra.out), and the dynamo
- # frontend surfaces a misleading "NATS request ... deadline has elapsed"
- # (it never gets a reply because the publish was rejected). 32 MiB gives
- # ~10x headroom over the largest observed payload (3.2 MB) without
- # crossing NATS's 64 MiB hard cap or Dynamo's 16 MiB advisory limit.
- nats_max_payload_mb: 32
-
-frontend:
- type: dynamo
- enable_multiple_frontends: false
-
-backend:
- type: vllm
- connector: null
- prefill_environment:
- TILELANG_CLEANUP_TEMP_FILES: "1"
- VLLM_USE_NCCL_SYMM_MEM: "1"
- NCCL_CUMEM_ENABLE: "1"
- NCCL_MNNVL_ENABLE: "1"
- NCCL_NVLS_ENABLE: "1"
- TORCH_SYMMMEM: "NVSHMEM"
- decode_environment:
- TILELANG_CLEANUP_TEMP_FILES: "1"
- VLLM_USE_NCCL_SYMM_MEM: "1"
- NCCL_CUMEM_ENABLE: "1"
- NCCL_MNNVL_ENABLE: "1"
- NCCL_NVLS_ENABLE: "1"
- TORCH_SYMMMEM: "NVSHMEM"
-
- vllm_config:
- prefill:
- kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
- served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
- kv-cache-dtype: "fp8"
- tensor-parallel-size: 1
- pipeline-parallel-size: 1
- data-parallel-size: 4
- data-parallel-rpc-port: 13345
- enable-expert-parallel: true
- attention-config: '{"use_fp4_indexer_cache": true}'
- moe-backend: "deep_gemm_mega_moe"
- # enforce-eager: true
- # max-num-seqs: 256
- max-num-batched-tokens: 16384
- trust-remote-code: true
- no-enable-flashinfer-autotune: true
- block-size: 256
- gpu-memory-utilization: 0.9
- enable-ep-weight-filter: true
- no-disable-hybrid-kv-cache-manager: true
- enable-sleep-mode: true
- tokenizer-mode: deepseek_v4
- decode:
- kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
- served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
- kv-cache-dtype: "fp8"
- tensor-parallel-size: 4
- pipeline-parallel-size: 1
- # max-num-seqs: 512
- trust-remote-code: true
- block-size: 256
- compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
- gpu-memory-utilization: 0.9
- stream-interval: 10
- no-disable-hybrid-kv-cache-manager: true
- enable-ep-weight-filter: true
- all2all-backend: "flashinfer_nvlink_one_sided"
- no-enable-flashinfer-autotune: true
- enable-sleep-mode: true
- tokenizer-mode: deepseek_v4
-
-# sbatch + srun resource grants for clusters without per-GPU defaults.
-#
-# mem=0: allocate all available node memory (~868 GB on CW gb300). Without
-# this, sbatch only requests ntasks × DefMemPerCPU = 8 × 4 GB = 32 GB for
-# the whole job and worker cgroups OOM-kill mid model load (R7-R11 hit
-# this; sacct showed AllocTRES mem=4G per step).
-#
-# cpus-per-task=72: give each task one CW gb300 NUMA socket (144 cores
-# split 2 × 72). Critical for the *infra step* (etcd + nats) which
-# srtctl spawns without --gres=gpu — on CW that means DefMemPerCPU
-# applies and the step gets 1 CPU by default. With 24 dynamo DP ranks
-# all hammering etcd for lease keep-alives, single-CPU etcd can't keep
-# up and dies (R12 hit this; etcd reported max-cpu-set=1, leases
-# deadline-exceeded, infra SIGKILL'd at 16:35:49). 72 CPUs is plenty
-# for both etcd + nats AND for vLLM worker auxiliary threads.
-#
-# nv gb300 doesn't need this because cluster default DefCpuPerGPU=35
-# auto-allocates 4*35=140 CPUs per GPU-bearing task; cw has no per-GPU
-# default. Setting it here is safe on both because the value is ≤ node
-# CPU count.
-#
-# srun_options.mem=0 forces each srun step to use the full node memory
-# (without it, srun steps default back to cpus_per_task × DefMemPerCPU).
-# Docs: docs/config-reference.md#sbatch_directives + #srun_options.
-sbatch_directives:
- mem: "0"
- cpus-per-task: "72"
-srun_options:
- mem: "0"
- # gb300-nv: pyxis maps the calling user (sa-shared) into the container as
- # uid 345200007. dpkg refuses to run without EUID 0 even though
- # ENROOT_ROOTFS_WRITABLE=1 makes the rootfs writable, so the agentic_srt
- # apt-get install git step fails. --container-remap-root asks pyxis to
- # remap us to uid 0 inside the container. srt-slurm renders empty-string
- # values as flag-only srun args (see core/slurm.py:250).
- container-remap-root: ""
-
-benchmark:
- type: custom
- command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
- env:
- INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
- RESULT_DIR: /logs/agentic
- PORT: "8000"
- IS_MULTINODE: "true"
- # Container-side path of the aiperf mmap dataset cache; the host-side
- # mount is wired via launch_gb300-*.sh's srtslurm.yaml default_mounts.
- # Without this, aiperf re-tokenizes + re-writes ~65 GB of mmap files
- # per dataset on every run.
- AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
- # Persistent HF hub cache (also wired via default_mounts) so the trace
- # dataset isn't re-downloaded on every run. Overrides the workflow-level
- # HF_HUB_CACHE=/mnt/hf_hub_cache, which doesn't exist on these nodes.
- HF_HUB_CACHE: "/hf_hub_cache"
diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-4p1d-dep4-dep8-24-c4096-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-4p1d-dep4-dep8-24-c4096-agentic.yaml
deleted file mode 100644
index 05b779d541..0000000000
--- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-4p1d-dep4-dep8-24-c4096-agentic.yaml
+++ /dev/null
@@ -1,179 +0,0 @@
-name: "svf-vllm-disagg-gb300-4p1d-dep4-dep8-24-c4096-agentic"
-
-# Agentic-coding variant of vllm/deepseek-v4/8k1k/disagg-gb300-4p1d-dep4-dep8-24-c4096.yaml.
-# Max-throughput shape: 4 prefill (DEP=4 each) + 1 decode (DEP=8). 6 GB300
-# nodes (4P + 2D = 24 GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra
-# node. Sized for concurrency 4096 with deep_gemm_mega_moe on both workers.
-#
-# Divergence vs the 8k1k sibling:
-# - benchmark.type: sa-bench -> custom (hands off to agentic_srt.sh)
-# - max-model-len: removed (let vLLM derive from model config; agentic
-# trajectories blow past any small explicit cap)
-# - no-enable-prefix-caching: dropped (prefix caching MUST be on for
-# trajectory reuse — entire point of agentic)
-# Note: --enable-auto-tool-choice / --tool-call-parser / --reasoning-parser
-# are NOT set on the worker. The dynamo-vllm worker entrypoint doesn't
-# accept them (different arg parser than `vllm serve`). In disagg, chat
-# parsing happens at the dynamo frontend, not at the worker.
-
-model:
- path: "deepseek-v4-pro"
- container: "vllm/vllm-openai:v0.21.0-ubuntu2404"
- precision: "fp4"
-
-dynamo:
- install: true
- wheel: "1.2.0.dev20260426"
-
-setup_script: vllm-container-deps.sh
-
-slurm:
- time_limit: "8:00:00"
-
-health_check:
- max_attempts: 1440
- interval_seconds: 10
-
-resources:
- gpu_type: "gb300"
- gpus_per_node: 4
- prefill_nodes: 4
- decode_nodes: 2
- prefill_workers: 4
- decode_workers: 1
- gpus_per_prefill: 4
- gpus_per_decode: 8
-
-infra:
- etcd_nats_dedicated_node: true
- # See sibling 1p6d recipe for rationale — NATS 1 MiB default rejects
- # agentic prompts; 32 MiB gives ~10x headroom over observed payloads.
- nats_max_payload_mb: 32
-
-frontend:
- type: dynamo
- enable_multiple_frontends: false
-
-backend:
- type: vllm
- connector: null
-
- prefill_environment:
- TILELANG_CLEANUP_TEMP_FILES: "1"
- VLLM_LOG_STATS_INTERVAL: "1"
- VLLM_USE_NCCL_SYMM_MEM: "1"
- NCCL_CUMEM_ENABLE: "1"
- NCCL_MNNVL_ENABLE: "1"
- NCCL_NVLS_ENABLE: "1"
- TORCH_SYMMMEM: "NVSHMEM"
-
- decode_environment:
- TILELANG_CLEANUP_TEMP_FILES: "1"
- VLLM_LOG_STATS_INTERVAL: "1"
- VLLM_USE_NCCL_SYMM_MEM: "1"
- NCCL_CUMEM_ENABLE: "1"
- NCCL_MNNVL_ENABLE: "1"
- NCCL_NVLS_ENABLE: "1"
- TORCH_SYMMMEM: "NVSHMEM"
-
- vllm_config:
- prefill:
- kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
- served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
- kv-cache-dtype: "fp8"
- tensor-parallel-size: 1
- pipeline-parallel-size: 1
- data-parallel-size: 4
- data-parallel-rpc-port: 13345
- enable-expert-parallel: true
- # enforce-eager: true
- # Inherited from fixed-sequence recipes; let vLLM select the scheduler
- # sequence limit until this is tuned explicitly for the agentic trace.
- # max-num-seqs: 16
- max-num-batched-tokens: 16384
- trust-remote-code: true
- no-enable-flashinfer-autotune: true
- safetensors-load-strategy: "prefetch"
- block-size: 256
- gpu-memory-utilization: 0.9
- no-disable-hybrid-kv-cache-manager: true
- tokenizer-mode: deepseek_v4
- enable-ep-weight-filter: true
- enable-sleep-mode: true
- moe-backend: "deep_gemm_mega_moe"
-
- decode:
- kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}'
- served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
- kv-cache-dtype: "fp8"
- tensor-parallel-size: 1
- pipeline-parallel-size: 1
- data-parallel-size: 8
- data-parallel-rpc-port: 13345
- enable-expert-parallel: true
- # max-num-seqs: 512
- trust-remote-code: true
- block-size: 256
- compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
- gpu-memory-utilization: 0.9
- stream-interval: 10
- no-disable-hybrid-kv-cache-manager: true
- tokenizer-mode: deepseek_v4
- enable-ep-weight-filter: true
- enable-sleep-mode: true
- moe-backend: "deep_gemm_mega_moe"
-
-# sbatch + srun resource grants for clusters without per-GPU defaults.
-#
-# mem=0: allocate all available node memory (~868 GB on CW gb300). Without
-# this, sbatch only requests ntasks × DefMemPerCPU = 8 × 4 GB = 32 GB for
-# the whole job and worker cgroups OOM-kill mid model load (R7-R11 hit
-# this; sacct showed AllocTRES mem=4G per step).
-#
-# cpus-per-task=72: give each task one CW gb300 NUMA socket (144 cores
-# split 2 × 72). Critical for the *infra step* (etcd + nats) which
-# srtctl spawns without --gres=gpu — on CW that means DefMemPerCPU
-# applies and the step gets 1 CPU by default. With 24 dynamo DP ranks
-# all hammering etcd for lease keep-alives, single-CPU etcd can't keep
-# up and dies (R12 hit this; etcd reported max-cpu-set=1, leases
-# deadline-exceeded, infra SIGKILL'd at 16:35:49). 72 CPUs is plenty
-# for both etcd + nats AND for vLLM worker auxiliary threads.
-#
-# nv gb300 doesn't need this because cluster default DefCpuPerGPU=35
-# auto-allocates 4*35=140 CPUs per GPU-bearing task; cw has no per-GPU
-# default. Setting it here is safe on both because the value is ≤ node
-# CPU count.
-#
-# srun_options.mem=0 forces each srun step to use the full node memory
-# (without it, srun steps default back to cpus_per_task × DefMemPerCPU).
-# Docs: docs/config-reference.md#sbatch_directives + #srun_options.
-sbatch_directives:
- mem: "0"
- cpus-per-task: "72"
-srun_options:
- mem: "0"
- # gb300-nv: pyxis maps the calling user (sa-shared) into the container as
- # uid 345200007. dpkg refuses to run without EUID 0 even though
- # ENROOT_ROOTFS_WRITABLE=1 makes the rootfs writable, so the agentic_srt
- # apt-get install git step fails. --container-remap-root asks pyxis to
- # remap us to uid 0 inside the container. srt-slurm renders empty-string
- # values as flag-only srun args (see core/slurm.py:250).
- container-remap-root: ""
-
-benchmark:
- type: custom
- command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
- env:
- INFMAX_CONTAINER_WORKSPACE: /infmax-workspace
- RESULT_DIR: /logs/agentic
- PORT: "8000"
- IS_MULTINODE: "true"
- # Container-side path of the aiperf mmap dataset cache; the host-side
- # mount is wired via launch_gb300-*.sh's srtslurm.yaml default_mounts.
- # Without this, aiperf re-tokenizes + re-writes ~65 GB of mmap files
- # per dataset on every run.
- AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache"
- # Persistent HF hub cache (also wired via default_mounts) so the trace
- # dataset isn't re-downloaded on every run. Overrides the workflow-level
- # HF_HUB_CACHE=/mnt/hf_hub_cache, which doesn't exist on these nodes.
- HF_HUB_CACHE: "/hf_hub_cache"
diff --git a/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_b200.sh b/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_b200.sh
deleted file mode 100755
index e75e3bff2f..0000000000
--- a/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_b200.sh
+++ /dev/null
@@ -1,198 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-# Agentic trace replay benchmark for Kimi-K2.5 NVFP4 on B200 using vLLM.
-#
-# Required env vars:
-# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
-#
-# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=lmcache.
-
-source "$(dirname "$0")/../../../benchmark_lib.sh"
-
-check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION
-
-
-if [[ -n "${SLURM_JOB_ID:-}" ]]; then
- echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
-fi
-
-# `hf download` creates the target dir if missing and is itself idempotent.
-# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
-# Either way, MODEL_PATH is what the server is launched with.
-if [[ -n "${MODEL_PATH:-}" ]]; then
- if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
- hf download "$MODEL" --local-dir "$MODEL_PATH"
- fi
-else
- hf download "$MODEL"
- export MODEL_PATH="$MODEL"
-fi
-nvidia-smi
-
-# ---- Resolve traces and install deps ----------------------------------------
-resolve_trace_source
-install_agentic_deps
-
-# ---- Server config ----------------------------------------------------------
-SERVER_LOG="$RESULT_DIR/server.log"
-LMCACHE_LOG="$RESULT_DIR/lmcache_server.log"
-mkdir -p "$RESULT_DIR"
-
-OFFLOAD_ARGS=()
-PREFIX_CACHE_ARGS=()
-LMCACHE_PID=""
-
-cleanup_lmcache_server() {
- if [[ -n "$LMCACHE_PID" ]] && kill -0 "$LMCACHE_PID" 2>/dev/null; then
- kill "$LMCACHE_PID" 2>/dev/null || true
- wait "$LMCACHE_PID" 2>/dev/null || true
- fi
-}
-
-trap cleanup_lmcache_server EXIT
-
-wait_for_lmcache_ready() {
- { set +x; } 2>/dev/null
- local attempts="${LMCACHE_READY_ATTEMPTS:-120}"
- local tail_pid=""
-
- while [ ! -f "$LMCACHE_LOG" ]; do
- if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then
- echo "LMCache server died before creating log file. Exiting." >&2
- exit 1
- fi
- sleep 1
- done
-
- tail -f -n +1 "$LMCACHE_LOG" &
- tail_pid=$!
-
- for ((i = 1; i <= attempts; i++)); do
- if curl --output /dev/null --silent --fail "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck"; then
- kill "$tail_pid" 2>/dev/null || true
- wait "$tail_pid" 2>/dev/null || true
- return 0
- fi
- if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then
- echo "LMCache server died before becoming healthy. Log follows:" >&2
- kill "$tail_pid" 2>/dev/null || true
- wait "$tail_pid" 2>/dev/null || true
- cat "$LMCACHE_LOG" >&2 || true
- exit 1
- fi
- sleep 1
- done
-
- echo "Timed out waiting for LMCache server healthcheck. Log follows:" >&2
- kill "$tail_pid" 2>/dev/null || true
- wait "$tail_pid" 2>/dev/null || true
- cat "$LMCACHE_LOG" >&2 || true
- exit 1
-}
-
-if require_agentic_kv_offload_backend lmcache; then
- { set +x; } 2>/dev/null
- unset VLLM_USE_SIMPLE_KV_OFFLOAD
-
- agentic_pip_install --quiet --no-cache-dir 'lmcache==0.5.1'
- python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null
-
- # MP mode owns the configured CPU pool in the external LMCache
- # server instead of passing
- # --kv-offloading-size through vLLM's integrated LMCache convenience
- # path, which divides the value by TP and then hits a large single-shot
- # cudaHostAlloc in LMCache 0.4.5's single-process local CPU backend.
- LMCACHE_HOST="${LMCACHE_HOST:-127.0.0.1}"
- LMCACHE_PORT="${LMCACHE_PORT:-5555}"
- LMCACHE_HTTP_PORT="${LMCACHE_HTTP_PORT:-8080}"
- # LMCacheMPConnector builds its ZMQ endpoint by concatenating
- # lmcache.mp.host and lmcache.mp.port, and its default host already
- # includes the tcp:// scheme. Keep the server bind host raw, but pass
- # a ZMQ-style host string to the connector.
- LMCACHE_CONNECT_HOST="${LMCACHE_CONNECT_HOST:-tcp://$LMCACHE_HOST}"
- LMCACHE_L1_SIZE_GB="${LMCACHE_L1_SIZE_GB:-$TOTAL_CPU_DRAM_GB}"
- if [ "$LMCACHE_L1_SIZE_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then
- echo "Error: LMCACHE_L1_SIZE_GB=$LMCACHE_L1_SIZE_GB exceeds configured capacity $TOTAL_CPU_DRAM_GB" >&2
- exit 1
- fi
- # Initial allocation is deliberately small; --l1-size-gb above is the
- # actual pool capacity and grows lazily as the run fills the cache.
- LMCACHE_L1_INIT_SIZE_GB="${LMCACHE_L1_INIT_SIZE_GB:-20}"
- LMCACHE_CHUNK_SIZE="${LMCACHE_CHUNK_SIZE:-256}"
- LMCACHE_MAX_WORKERS="${LMCACHE_MAX_WORKERS:-$TP}"
- export PYTHONHASHSEED="${PYTHONHASHSEED:-0}"
-
- echo "Starting LMCache MP server..."
- LMCACHE_CMD=(
- lmcache server
- --host "$LMCACHE_HOST"
- --port "$LMCACHE_PORT"
- --http-host "$LMCACHE_HOST"
- --http-port "$LMCACHE_HTTP_PORT"
- --l1-size-gb "$LMCACHE_L1_SIZE_GB"
- --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB"
- --chunk-size "$LMCACHE_CHUNK_SIZE"
- --max-workers "$LMCACHE_MAX_WORKERS"
- --eviction-policy LRU
- )
- printf '%q ' "${LMCACHE_CMD[@]}" > "$RESULT_DIR/lmcache_command.txt"
- printf '\n' >> "$RESULT_DIR/lmcache_command.txt"
- "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 &
- LMCACHE_PID=$!
- echo "LMCache server PID: $LMCACHE_PID"
- wait_for_lmcache_ready
-
- PREFIX_CACHE_ARGS=(--enable-prefix-caching)
- OFFLOAD_ARGS=(
- --kv-transfer-config
- "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT}}"
- --disable-hybrid-kv-cache-manager
- )
-fi
-
-echo "Starting vllm server..."
-export TORCH_CUDA_ARCH_LIST="10.0"
-export PYTHONNOUSERSITE=1
-# Disable vLLM v0.21+ CUDA-graph memory estimator. Its pre-reservation
-# eats ~32% of HBM upfront which, combined with FP4 weights at TP=4
-# (~62 GB/GPU), leaves no room for KV blocks -- _check_enough_kv_cache_memory
-# trips before the engine starts. Our --gpu-memory-utilization=0.90 already
-# leaves ~18 GB/GPU slack outside vLLM's budget, which is the same safety
-# net the estimator provides, so disabling it is redundant rather than
-# unsafe.
-export VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0
-
-{ set +x; } 2>/dev/null
-VLLM_CMD=(
- vllm serve "$MODEL_PATH" --served-model-name "$MODEL"
- --host 0.0.0.0
- --port "$PORT"
- --tensor-parallel-size="$TP"
- --gpu-memory-utilization 0.90
- --max-num-seqs "$CONC"
- --reasoning-parser kimi_k2
- --tool-call-parser kimi_k2
- --compilation_config.pass_config.fuse_allreduce_rms true
- --kv-cache-dtype fp8
- --max-cudagraph-capture-size 2048
- --stream-interval 20
- --trust-remote-code
- "${PREFIX_CACHE_ARGS[@]}"
- "${OFFLOAD_ARGS[@]}"
-)
-printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt"
-printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt"
-"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 &
-SERVER_PID=$!
-echo "Server PID: $SERVER_PID"
-
-wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
-
-if [ "${EVAL_ONLY}" = "true" ]; then
- run_eval --port "$PORT"
-else
- build_replay_cmd "$RESULT_DIR"
- run_agentic_replay_and_write_outputs "$RESULT_DIR"
-fi
diff --git a/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_b300.sh b/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_b300.sh
deleted file mode 100755
index 94636fad41..0000000000
--- a/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_b300.sh
+++ /dev/null
@@ -1,88 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-# Agentic trace replay benchmark for Kimi-K2.5 NVFP4 on B300 using vLLM.
-#
-# Required env vars:
-# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
-#
-# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=vllm-simple.
-
-source "$(dirname "$0")/../../../benchmark_lib.sh"
-
-check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION
-
-
-if [[ -n "${SLURM_JOB_ID:-}" ]]; then
- echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
-fi
-
-# `hf download` creates the target dir if missing and is itself idempotent.
-# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
-# Either way, MODEL_PATH is what the server is launched with.
-if [[ -n "${MODEL_PATH:-}" ]]; then
- if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
- hf download "$MODEL" --local-dir "$MODEL_PATH"
- fi
-else
- hf download "$MODEL"
- export MODEL_PATH="$MODEL"
-fi
-nvidia-smi
-
-# ---- Resolve traces and install deps ----------------------------------------
-resolve_trace_source
-install_agentic_deps
-
-# ---- Server config ----------------------------------------------------------
-SERVER_LOG="$RESULT_DIR/server.log"
-mkdir -p "$RESULT_DIR"
-
-OFFLOAD_ARGS=()
-PREFIX_CACHE_ARGS=()
-
-if require_agentic_kv_offload_backend vllm-simple; then
- export VLLM_USE_SIMPLE_KV_OFFLOAD=1
- OFFLOAD_ARGS=(
- --kv_offloading_backend native
- --kv_offloading_size "$TOTAL_CPU_DRAM_GB"
- --disable-hybrid-kv-cache-manager
- )
-fi
-
-echo "Starting vllm server..."
-export PYTHONNOUSERSITE=1
-
-{ set +x; } 2>/dev/null
-VLLM_CMD=(
- vllm serve "$MODEL_PATH" --served-model-name "$MODEL"
- --host 0.0.0.0
- --port "$PORT"
- --tensor-parallel-size="$TP"
- --gpu-memory-utilization 0.90
- --max-num-seqs "$CONC"
- --reasoning-parser kimi_k2
- --tool-call-parser kimi_k2
- --compilation_config.pass_config.fuse_allreduce_rms true
- --kv-cache-dtype fp8
- --max-cudagraph-capture-size 2048
- --stream-interval 20
- --trust-remote-code
- "${PREFIX_CACHE_ARGS[@]}"
- "${OFFLOAD_ARGS[@]}"
-)
-printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt"
-printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt"
-"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 &
-SERVER_PID=$!
-echo "Server PID: $SERVER_PID"
-
-wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
-
-if [ "${EVAL_ONLY}" = "true" ]; then
- run_eval --port "$PORT"
-else
- build_replay_cmd "$RESULT_DIR"
- run_agentic_replay_and_write_outputs "$RESULT_DIR"
-fi
diff --git a/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_b300_mtp.sh b/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_b300_mtp.sh
deleted file mode 100755
index 20f6ad25d9..0000000000
--- a/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_b300_mtp.sh
+++ /dev/null
@@ -1,130 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-source "$(dirname "$0")/../../../benchmark_lib.sh"
-
-check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION
-
-
-if [[ -n "${SLURM_JOB_ID:-}" ]]; then
- echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
-fi
-
-DRAFT_MODEL="lightseekorg/kimi-k2.6-eagle3.1-mla"
-
-if [[ -n "${MODEL_PATH:-}" ]]; then
- if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
- hf download "$MODEL" --local-dir "$MODEL_PATH"
- fi
- DRAFT_MODEL_PATH="/data/models/${DRAFT_MODEL##*/}"
- if [[ ! -d "$DRAFT_MODEL_PATH" || -z "$(ls -A "$DRAFT_MODEL_PATH" 2>/dev/null)" ]]; then
- hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH"
- fi
-else
- hf download "$MODEL"
- export MODEL_PATH="$MODEL"
- hf download "$DRAFT_MODEL"
- DRAFT_MODEL_PATH="$DRAFT_MODEL"
-fi
-nvidia-smi
-
-resolve_trace_source
-install_agentic_deps
-
-SERVER_LOG="$RESULT_DIR/server.log"
-mkdir -p "$RESULT_DIR"
-
-SERVER_PID=""
-
-cleanup_agentic_services() {
- local exit_code=$?
- trap - EXIT INT TERM
- set +e
- stop_background_process_tree "$SERVER_PID" "vLLM server" 60
- exit "$exit_code"
-}
-trap cleanup_agentic_services EXIT
-trap 'exit 130' INT
-trap 'exit 143' TERM
-
-DCP_SIZE="${DCP_SIZE:-1}"
-DCP_ARGS=()
-if [[ "$DCP_SIZE" -gt 1 ]]; then
- DCP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE" --dcp-comm-backend a2a)
- NUM_SPEC_TOKENS=3
- SYNTHETIC_ACCEPT_LEN=2.88
- SPEC_ARGS=(--speculative-config "{\"method\":\"eagle3\",\"model\":\"$DRAFT_MODEL_PATH\",\"num_speculative_tokens\":$NUM_SPEC_TOKENS,\"rejection_sample_method\":\"synthetic\",\"synthetic_acceptance_length\":$SYNTHETIC_ACCEPT_LEN,\"attention_backend\":\"TOKENSPEED_MLA\"}")
- ATTN_CONFIG='{"mla_prefill_backend":"TOKENSPEED_MLA"}'
- COMPILATION_CONFIG='{"pass_config":{"fuse_allreduce_rms":false}}'
-else
- NUM_SPEC_TOKENS=4
- SYNTHETIC_ACCEPT_LEN=3.24
- SPEC_ARGS=(--speculative-config "{\"method\":\"eagle3\",\"model\":\"$DRAFT_MODEL_PATH\",\"num_speculative_tokens\":$NUM_SPEC_TOKENS,\"rejection_sample_method\":\"synthetic\",\"synthetic_acceptance_length\":$SYNTHETIC_ACCEPT_LEN}")
- ATTN_CONFIG='{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}'
- COMPILATION_CONFIG='{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'
-fi
-ATTN_BACKEND_ARGS=(--attention-backend TOKENSPEED_MLA)
-
-OFFLOAD_ARGS=()
-
-if agentic_kv_offload_enabled; then
- case "$KV_OFFLOAD_BACKEND" in
- native)
- export VLLM_USE_SIMPLE_KV_OFFLOAD=1
- CPU_OFFLOAD_BYTES=$((TOTAL_CPU_DRAM_GB * 1024 * 1024 * 1024))
- OFFLOAD_ARGS=(
- --disable-hybrid-kv-cache-manager
- --kv-transfer-config
- "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use\":$CPU_OFFLOAD_BYTES,\"lazy_offload\":false}}"
- )
- ;;
- *) echo "Error: unsupported KV_OFFLOAD_BACKEND value '$KV_OFFLOAD_BACKEND' with EAGLE3 (expected: native)" >&2; exit 1 ;;
- esac
-fi
-
-
-GMU=0.90
-if [[ "$DCP_SIZE" -gt 1 && "$KV_OFFLOADING" == "none" ]]; then
- GMU=0.85
-fi
-
-echo "Starting vllm server..."
-export PYTHONNOUSERSITE=1
-
-export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm
-
-{ set +x; } 2>/dev/null
-VLLM_CMD=(
- vllm serve "$MODEL_PATH" --served-model-name "$MODEL"
- --host 0.0.0.0
- --port "$PORT"
- --kv-cache-dtype fp8
- --trust-remote-code
- --block-size 64
- --language-model-only
- --gpu-memory-utilization "$GMU"
- --max-num-seqs "$CONC"
- "${ATTN_BACKEND_ARGS[@]}"
- --attention-config "$ATTN_CONFIG"
- --compilation-config "$COMPILATION_CONFIG"
- --max-cudagraph-capture-size 2048
- --max-num-batched-tokens 16384
- --stream-interval 10
- --enable-prefix-caching
- --tensor-parallel-size "$TP"
- "${SPEC_ARGS[@]}"
- "${DCP_ARGS[@]}"
- "${OFFLOAD_ARGS[@]}"
-)
-printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt"
-printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt"
-"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 &
-SERVER_PID=$!
-echo "Server PID: $SERVER_PID"
-
-wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
-
-build_replay_cmd "$RESULT_DIR"
-
-run_agentic_replay_and_write_outputs "$RESULT_DIR"
diff --git a/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_mi355x.sh b/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_mi355x.sh
deleted file mode 100755
index b0d5f801c8..0000000000
--- a/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_mi355x.sh
+++ /dev/null
@@ -1,120 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-# Agentic trace replay benchmark for Kimi-K2.5 FP4 on MI355X using vLLM.
-#
-# Required env vars:
-# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
-#
-# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=vllm-native.
-
-
-source "$(dirname "$0")/../../../benchmark_lib.sh"
-
-check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE
-
-if [[ -n "${SLURM_JOB_ID:-}" ]]; then
- echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
-fi
-
-# ROCR/HIP visibility for vLLM 0.14+
-if [ -n "${ROCR_VISIBLE_DEVICES:-}" ]; then
- export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES"
-fi
-
-# `hf download` creates the target dir if missing and is itself idempotent.
-# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
-# Either way, MODEL_PATH is what the server is launched with.
-if [[ -n "${MODEL_PATH:-}" ]]; then
- if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
- hf download "$MODEL" --local-dir "$MODEL_PATH"
- fi
-else
- hf download "$MODEL"
- export MODEL_PATH="$MODEL"
-fi
-rocm-smi || true
-amd-smi || true
-
-# ---- Resolve traces and install deps ----------------------------------------
-resolve_trace_source
-install_agentic_deps
-
-# Install amd-quark for MXFP4 (manual install due to ROCm vLLM bug)
-pip install amd-quark
-
-# Disable AITER RMSNorm for TP < 8 due to accuracy issues
-if [ "${TP}" -lt 8 ]; then
- export VLLM_ROCM_USE_AITER_RMSNORM=0
-fi
-# Workaround for MEC FW <177 RCCL memory reclaim issue
-version=$(rocm-smi --showfw 2>/dev/null | grep MEC | head -n 1 | awk '{print $NF}')
-if [[ "$version" == "" || ${version:-0} -lt 177 ]]; then
- export HSA_NO_SCRATCH_RECLAIM=1
-fi
-
-export VLLM_ROCM_USE_AITER=1
-export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4
-
-# ---- Server config ----------------------------------------------------------
-SERVER_LOG="$RESULT_DIR/server.log"
-mkdir -p "$RESULT_DIR"
-
-OFFLOAD_ARGS=()
-PREFIX_CACHE_ARGS=()
-
-if require_agentic_kv_offload_backend vllm-native; then
- unset VLLM_USE_SIMPLE_KV_OFFLOAD
- # Use vLLM's regular native KV-offload path (OffloadingConnector),
- # NOT the SimpleCPUOffloadConnector. The "vllm-native" backend resolves to
- # OffloadingConnector by default; setting VLLM_USE_SIMPLE_KV_OFFLOAD=1
- # would switch it to SimpleCPUOffloadConnector. We intentionally leave
- # that env var UNSET here so the regular OffloadingConnector path is
- # used. The shortcut --kv_offloading_backend native + --kv_offloading_size
- # form constructs the KVTransferConfig at engine startup
- # (vllm/config/vllm.py:662).
- OFFLOAD_ARGS=(
- --kv_offloading_backend native
- --kv_offloading_size "$TOTAL_CPU_DRAM_GB"
- --disable-hybrid-kv-cache-manager
- )
-fi
-
-EP_ARGS=()
-if [ "$EP_SIZE" -gt 1 ]; then
- EP_ARGS=(--enable-expert-parallel)
-fi
-
-echo "Starting vllm server..."
-export PYTHONNOUSERSITE=1
-
-{ set +x; } 2>/dev/null
-VLLM_CMD=(
- vllm serve "$MODEL_PATH" --served-model-name "$MODEL"
- --host 0.0.0.0
- --port "$PORT"
- --tensor-parallel-size="$TP"
- "${EP_ARGS[@]}"
- --gpu-memory-utilization 0.90
- --block-size=1
- --trust-remote-code
- --max-num-seqs "$CONC"
- --mm-encoder-tp-mode data
- "${PREFIX_CACHE_ARGS[@]}"
- "${OFFLOAD_ARGS[@]}"
-)
-printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt"
-printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt"
-"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 &
-SERVER_PID=$!
-echo "Server PID: $SERVER_PID"
-
-wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
-
-if [ "${EVAL_ONLY}" = "true" ]; then
- run_eval --port "$PORT"
-else
- build_replay_cmd "$RESULT_DIR"
- run_agentic_replay_and_write_outputs "$RESULT_DIR"
-fi
diff --git a/benchmarks/single_node/agentic/deprecated/kimik2.5_int4_b200.sh b/benchmarks/single_node/agentic/deprecated/kimik2.5_int4_b200.sh
deleted file mode 100755
index b216ec7b4c..0000000000
--- a/benchmarks/single_node/agentic/deprecated/kimik2.5_int4_b200.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-# Agentic trace replay benchmark for Kimi-K2.5 INT4 on B200 using vLLM.
-#
-# Required env vars:
-# MODEL, TP, CONC, RESULT_DIR
-
-source "$(dirname "$0")/../../../benchmark_lib.sh"
-
-check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION
-
-
-if [[ -n "${SLURM_JOB_ID:-}" ]]; then
- echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
-fi
-
-# `hf download` creates the target dir if missing and is itself idempotent.
-# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
-# Either way, MODEL_PATH is what the server is launched with.
-if [[ -n "${MODEL_PATH:-}" ]]; then
- if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
- hf download "$MODEL" --local-dir "$MODEL_PATH"
- fi
-else
- hf download "$MODEL"
- export MODEL_PATH="$MODEL"
-fi
-nvidia-smi
-
-# ---- Resolve traces and install deps ----------------------------------------
-resolve_trace_source
-install_agentic_deps
-
-# ---- Server config ----------------------------------------------------------
-SERVER_LOG="$RESULT_DIR/server.log"
-mkdir -p "$RESULT_DIR"
-
-OFFLOAD_ARGS=""
-if require_agentic_kv_offload_backend vllm-simple; then
- export VLLM_USE_SIMPLE_KV_OFFLOAD=1
- OFFLOAD_ARGS="--kv_offloading_backend native --kv_offloading_size $TOTAL_CPU_DRAM_GB --disable-hybrid-kv-cache-manager"
-fi
-
-echo "Starting vllm server..."
-export TORCH_CUDA_ARCH_LIST="10.0"
-export PYTHONNOUSERSITE=1
-export VLLM_USE_FLASHINFER_MOE_INT4=1
-
-vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \
---host 0.0.0.0 \
---port $PORT \
---gpu-memory-utilization 0.95 \
---tensor-parallel-size $TP \
---max-num-seqs $CONC \
---reasoning-parser kimi_k2 \
---tool-call-parser kimi_k2 \
---compilation_config.pass_config.fuse_allreduce_rms true \
---trust-remote-code \
-$OFFLOAD_ARGS > "$SERVER_LOG" 2>&1 &
-SERVER_PID=$!
-echo "Server PID: $SERVER_PID"
-
-wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
-
-if [ "${EVAL_ONLY}" = "true" ]; then
- run_eval --port "$PORT"
-else
- build_replay_cmd "$RESULT_DIR"
- run_agentic_replay_and_write_outputs "$RESULT_DIR"
-fi
diff --git a/benchmarks/single_node/agentic/deprecated/kimik2.5_int4_h100.sh b/benchmarks/single_node/agentic/deprecated/kimik2.5_int4_h100.sh
deleted file mode 100755
index 9f6ab9066b..0000000000
--- a/benchmarks/single_node/agentic/deprecated/kimik2.5_int4_h100.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-# Agentic trace replay benchmark for Kimi-K2.5 INT4 on H100 using vLLM.
-#
-# Required env vars:
-# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
-
-source "$(dirname "$0")/../../../benchmark_lib.sh"
-
-check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION
-
-
-if [[ -n "${SLURM_JOB_ID:-}" ]]; then
- echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
-fi
-
-# `hf download` creates the target dir if missing and is itself idempotent.
-# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
-# Either way, MODEL_PATH is what the server is launched with.
-if [[ -n "${MODEL_PATH:-}" ]]; then
- if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
- hf download "$MODEL" --local-dir "$MODEL_PATH"
- fi
-else
- hf download "$MODEL"
- export MODEL_PATH="$MODEL"
-fi
-nvidia-smi
-
-# ---- Resolve traces and install deps ----------------------------------------
-resolve_trace_source
-install_agentic_deps
-
-# ---- Server config ----------------------------------------------------------
-SERVER_LOG="$RESULT_DIR/server.log"
-mkdir -p "$RESULT_DIR"
-
-OFFLOAD_ARGS=""
-if require_agentic_kv_offload_backend vllm-simple; then
- export VLLM_USE_SIMPLE_KV_OFFLOAD=1
- OFFLOAD_ARGS="--kv_offloading_backend native --kv_offloading_size $TOTAL_CPU_DRAM_GB --disable-hybrid-kv-cache-manager"
-fi
-
-echo "Starting vllm server..."
-export PYTHONNOUSERSITE=1
-export VLLM_USE_FLASHINFER_MOE_INT4=1
-
-vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \
---host 0.0.0.0 \
---port $PORT \
---gpu-memory-utilization 0.95 \
---tensor-parallel-size $TP \
---max-num-seqs $CONC \
---reasoning-parser kimi_k2 \
---tool-call-parser kimi_k2 \
---compilation_config.pass_config.fuse_allreduce_rms true \
---kv-cache-dtype fp8 \
---trust-remote-code \
-$OFFLOAD_ARGS > "$SERVER_LOG" 2>&1 &
-SERVER_PID=$!
-echo "Server PID: $SERVER_PID"
-
-wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
-
-if [ "${EVAL_ONLY}" = "true" ]; then
- run_eval --port "$PORT"
-else
- build_replay_cmd "$RESULT_DIR"
- run_agentic_replay_and_write_outputs "$RESULT_DIR"
-fi
diff --git a/benchmarks/single_node/agentic/deprecated/kimik2.5_int4_h200.sh b/benchmarks/single_node/agentic/deprecated/kimik2.5_int4_h200.sh
deleted file mode 100755
index 683dcd254d..0000000000
--- a/benchmarks/single_node/agentic/deprecated/kimik2.5_int4_h200.sh
+++ /dev/null
@@ -1,79 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-# Agentic trace replay benchmark for Kimi-K2.5 INT4 on H200 using vLLM.
-#
-# Required env vars:
-# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
-
-source "$(dirname "$0")/../../../benchmark_lib.sh"
-
-check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION
-
-
-if [[ -n "${SLURM_JOB_ID:-}" ]]; then
- echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
-fi
-
-# `hf download` creates the target dir if missing and is itself idempotent.
-# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
-# Either way, MODEL_PATH is what the server is launched with.
-if [[ -n "${MODEL_PATH:-}" ]]; then
- if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
- hf download "$MODEL" --local-dir "$MODEL_PATH"
- fi
-else
- hf download "$MODEL"
- export MODEL_PATH="$MODEL"
-fi
-nvidia-smi
-
-# ---- Resolve traces and install deps ----------------------------------------
-resolve_trace_source
-install_agentic_deps
-
-# ---- Server config ----------------------------------------------------------
-SERVER_LOG="$RESULT_DIR/server.log"
-mkdir -p "$RESULT_DIR"
-
-OFFLOAD_ARGS=""
-if require_agentic_kv_offload_backend vllm-simple; then
- # Kimi K2.5 is pure TP (no DP-attn): single engine, world_size=TP.
- # SimpleCPUOffloadConnector internally divides cpu_bytes_to_use by
- # world_size, so pass the full TOTAL_CPU_DRAM_GB; TP-shared mmap
- # keeps the aggregate at TOTAL.
- PER_ENGINE_BYTES=$((TOTAL_CPU_DRAM_GB * 1024 * 1024 * 1024))
- # JSON form (rather than --kv_offloading_backend native shortcut) so we can
- # pass lazy_offload=true. Eager mode hits a popleft_n AssertionError at
- # low/mid CONC on DSv4 + SimpleCPUOffloadConnector.
- export VLLM_USE_SIMPLE_KV_OFFLOAD=1
- OFFLOAD_ARGS="--kv-transfer-config {\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use\":$PER_ENGINE_BYTES,\"lazy_offload\":true}}"
-fi
-
-echo "Starting vllm server..."
-export PYTHONNOUSERSITE=1
-export VLLM_USE_FLASHINFER_MOE_INT4=1
-
-vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \
---host 0.0.0.0 \
---port $PORT \
---gpu-memory-utilization 0.95 \
---tensor-parallel-size $TP \
---max-num-seqs $CONC \
---reasoning-parser kimi_k2 \
---tool-call-parser kimi_k2 \
---compilation_config.pass_config.fuse_allreduce_rms true \
---trust-remote-code \
-$OFFLOAD_ARGS > "$SERVER_LOG" 2>&1 &
-SERVER_PID=$!
-echo "Server PID: $SERVER_PID"
-
-wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
-
-if [ "${EVAL_ONLY}" = "true" ]; then
- run_eval --port "$PORT"
-else
- build_replay_cmd "$RESULT_DIR"
- run_agentic_replay_and_write_outputs "$RESULT_DIR"
-fi
diff --git a/benchmarks/single_node/agentic/deprecated/qwen3.5_bf16_b200.sh b/benchmarks/single_node/agentic/deprecated/qwen3.5_bf16_b200.sh
deleted file mode 100755
index 36f47e9147..0000000000
--- a/benchmarks/single_node/agentic/deprecated/qwen3.5_bf16_b200.sh
+++ /dev/null
@@ -1,79 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-# Agentic trace replay benchmark for Qwen3.5 BF16 on B200 using SGLang.
-#
-# Required env vars:
-# MODEL, TP, CONC, RESULT_DIR
-
-source "$(dirname "$0")/../../../benchmark_lib.sh"
-
-check_env_vars MODEL TP CONC RESULT_DIR DURATION EP_SIZE
-
-SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-10}
-
-if [[ -n "${SLURM_JOB_ID:-}" ]]; then
- echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
-fi
-
-# `hf download` creates the target dir if missing and is itself idempotent.
-# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
-# Either way, MODEL_PATH is what the server is launched with.
-if [[ -n "${MODEL_PATH:-}" ]]; then
- if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
- hf download "$MODEL" --local-dir "$MODEL_PATH"
- fi
-else
- hf download "$MODEL"
- export MODEL_PATH="$MODEL"
-fi
-nvidia-smi
-
-# ---- Resolve traces and install deps ----------------------------------------
-resolve_trace_source
-install_agentic_deps
-
-# ---- Start SGLang server ----------------------------------------------------
-SERVER_LOG="$RESULT_DIR/server.log"
-mkdir -p "$RESULT_DIR"
-
-echo "Starting SGLang server..."
-export TORCH_CUDA_ARCH_LIST="10.0"
-export PYTHONNOUSERSITE=1
-export NCCL_NVLS_ENABLE=1
-export SGL_ENABLE_JIT_DEEPGEMM=false
-export SGLANG_ENABLE_FLASHINFER_GEMM=true
-
-python3 -m sglang.launch_server \
---model-path=$MODEL_PATH --served-model-name=$MODEL \
---host=0.0.0.0 \
---port=$PORT \
---served-model-name "Qwen/Qwen3.5-397B-A17B" \
---trust-remote-code \
---tensor-parallel-size=$TP \
---data-parallel-size=1 \
---ep-size $EP_SIZE \
---cuda-graph-max-bs $CONC \
---max-running-requests $CONC \
---mem-fraction-static 0.82 \
---chunked-prefill-size 32768 \
---max-prefill-tokens 32768 \
---attention-backend trtllm_mha \
---moe-runner-backend flashinfer_trtllm \
---enable-flashinfer-allreduce-fusion \
---scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \
---tokenizer-worker-num 6 \
---stream-interval 30 \
---enable-metrics > "$SERVER_LOG" 2>&1 &
-SERVER_PID=$!
-echo "Server PID: $SERVER_PID"
-
-wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
-
-if [ "${EVAL_ONLY}" = "true" ]; then
- run_eval --port "$PORT"
-else
- build_replay_cmd "$RESULT_DIR"
- run_agentic_replay_and_write_outputs "$RESULT_DIR"
-fi
diff --git a/benchmarks/single_node/agentic/dsv4_fp8_h200.sh b/benchmarks/single_node/agentic/dsv4_fp8_h200.sh
deleted file mode 100755
index 155f26ddea..0000000000
--- a/benchmarks/single_node/agentic/dsv4_fp8_h200.sh
+++ /dev/null
@@ -1,76 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-# Agentic trace replay benchmark for DeepSeek-V4-Pro FP8 on H200 using vLLM.
-# Uses the cu129 image; H200 has no FP4 path so the FP4 indexer cache flag
-# is omitted. Max-model-len pinned at 800k per the recipe.
-#
-# Required env vars:
-# MODEL, TP, CONC, RESULT_DIR
-
-source "$(dirname "$0")/../../benchmark_lib.sh"
-
-check_env_vars MODEL TP CONC RESULT_DIR DURATION
-
-if [[ -n "${SLURM_JOB_ID:-}" ]]; then
- echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
-fi
-
-# `hf download` creates the target dir if missing and is itself idempotent.
-# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
-# Either way, MODEL_PATH is what the server is launched with.
-if [[ -n "${MODEL_PATH:-}" ]]; then
- if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
- hf download "$MODEL" --local-dir "$MODEL_PATH"
- fi
-else
- hf download "$MODEL"
- export MODEL_PATH="$MODEL"
-fi
-nvidia-smi
-
-# ---- Resolve traces and install deps ----------------------------------------
-resolve_trace_source
-install_agentic_deps
-
-# DeepSeek-V4-Pro weights are large; engine startup can exceed default 600s.
-export VLLM_ENGINE_READY_TIMEOUT_S=3600
-
-# ---- Start vLLM server ------------------------------------------------------
-SERVER_LOG="$RESULT_DIR/server.log"
-mkdir -p "$RESULT_DIR"
-
-echo "Starting vLLM server..."
-export PYTHONNOUSERSITE=1
-
-# Per recipe: EP + DP=8 (no --tensor-parallel-size). TP from search space is
-# used for GPU allocation by the runner and as the DP size.
-vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \
---host 0.0.0.0 \
---port $PORT \
---trust-remote-code \
---kv-cache-dtype fp8 \
---block-size 256 \
---enable-expert-parallel \
---data-parallel-size $TP \
---gpu-memory-utilization 0.95 \
---max-num-seqs $CONC \
---max-num-batched-tokens 512 \
---no-enable-flashinfer-autotune \
---compilation-config '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY"}' \
---tokenizer-mode deepseek_v4 \
---tool-call-parser deepseek_v4 \
---enable-auto-tool-choice \
---reasoning-parser deepseek_v4 > "$SERVER_LOG" 2>&1 &
-SERVER_PID=$!
-echo "Server PID: $SERVER_PID"
-
-wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
-
-if [ "${EVAL_ONLY}" = "true" ]; then
- run_eval --port "$PORT"
-else
- build_replay_cmd "$RESULT_DIR"
- run_agentic_replay_and_write_outputs "$RESULT_DIR"
-fi
diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm.sh
deleted file mode 100755
index 17eb39dec3..0000000000
--- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm.sh
+++ /dev/null
@@ -1,170 +0,0 @@
-#!/usr/bin/env bash
-set -eo pipefail
-set -x
-
-# Agentic trace replay benchmark for Kimi-K3 (MXFP4) on B300 using vLLM.
-#
-# Day-zero single-node recipe. Serve flags follow the Kimi-K3 production recipe
-# already exercised by the DSpark AL collector
-# (benchmarks/single_node/speedbench/kimik3_fp4_b300_vllm.sh) and the upstream
-# recipes.vllm.ai/moonshotai/Kimi-K3 guidance, adapted to the agentic scenario.
-#
-# Required env vars:
-# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION
-#
-# TP8 is the only single-node layout. The MXFP4 checkpoint is ~1.5 TB on disk;
-# at TP4 that is ~375 GB of weights per GPU against B300's 288 GB of HBM, so
-# only the full 8-GPU shard (~188 GB/GPU) fits. Do not add TP4/TP2 arms.
-#
-# Weights are pre-staged node-local: `Kimi-K3` is in the b300-nv launcher's
-# STAGED_MODELS, so MODEL_PATH resolves to the read-only /scratch/models/Kimi-K3
-# mount and no download happens on the runner. The download block below only
-# covers stand-alone runs.
-#
-# KV_OFFLOADING: `none` (GPU-resident) or `dram` with
-# KV_OFFLOAD_BACKEND=vllm-simple (SimpleCPUOffloadConnector).
-#
-# Note on the DRAM arm: Kimi-K3 is a KDA/MLA hybrid — its linear-attention (KDA)
-# layers carry a recurrent state rather than paged KV blocks, and
-# SimpleCPUOffloadConnector offloads uniform paged blocks with no
-# hybrid-geometry handling. The arm is expected to offload only the MLA layers'
-# paged blocks; watch the bring-up sweep for KV-geometry errors at server init.
-
-source "$(dirname "$0")/../../benchmark_lib.sh"
-
-check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION
-
-# The 2.8T MXFP4 checkpoint only fits across all 8 B300s (see header).
-if [ "$TP" -ne 8 ]; then
- echo "Error: Kimi-K3 on B300 requires TP=8 (a ~1.5 TB MXFP4 checkpoint does not fit at TP<8), got TP='$TP'" >&2
- exit 1
-fi
-
-if [[ -n "${EP_SIZE:-}" && "${EP_SIZE}" -gt 1 ]]; then
- echo "Error: this recipe ships the pure-TP8 profile; EP_SIZE='$EP_SIZE' is not wired yet" >&2
- exit 1
-fi
-
-if [[ -n "${SLURM_JOB_ID:-}" ]]; then
- echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
-fi
-
-# `hf download` creates the target dir if missing and is itself idempotent.
-# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE.
-# Either way, MODEL_PATH is what the server is launched with.
-if [[ -n "${MODEL_PATH:-}" ]]; then
- if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
- hf download "$MODEL" --local-dir "$MODEL_PATH"
- fi
-else
- hf download "$MODEL"
- export MODEL_PATH="$MODEL"
-fi
-nvidia-smi
-
-# ---- Resolve traces and install deps ----------------------------------------
-resolve_trace_source
-install_agentic_deps
-
-# ---- Kimi-K3 production serving environment ---------------------------------
-export NCCL_DMABUF_ENABLE=0
-export VLLM_ALLREDUCE_USE_FLASHINFER=1
-export VLLM_USE_RUST_FRONTEND=1
-# Loading ~1.5 TB of MXFP4 shards off the staged mount takes well past the
-# default readiness window even with fastsafetensors.
-export VLLM_ENGINE_READY_TIMEOUT_S=3600
-export PYTHONNOUSERSITE=1
-# AIPerf pins one pooled keep-alive connection per agentic session and reuses it
-# across turns, while the Rust frontend's default VLLM_HTTP_TIMEOUT_KEEP_ALIVE is
-# 5s. An inter-turn idle gap longer than that lets the client reuse a socket at
-# the moment the server closes it -> aiohttp ServerDisconnectedError -> AIPerf
-# treats it as a terminal warmup failure and aborts the whole job. This killed
-# the dram c4 arm ~15 min into run 30324907690 with a perfectly healthy server
-# (it kept serving after the client gave up). Outlast the client pool so the
-# race cannot occur. Same fix as glm5.2_fp4_b300_sglang.sh's
-# SGLANG_TIMEOUT_KEEP_ALIVE=900.
-export VLLM_HTTP_TIMEOUT_KEEP_ALIVE=900
-# Agentic warmup dispatches large prompts at once; allow up to 15 minutes of TCP
-# progress before AIPerf declares a connection dead.
-export AIPERF_HTTP_TCP_USER_TIMEOUT=900000
-
-# ---- Server config ----------------------------------------------------------
-SERVER_LOG="$RESULT_DIR/server.log"
-mkdir -p "$RESULT_DIR"
-
-# ---- KV offloading ----------------------------------------------------------
-# The generated TOTAL_CPU_DRAM_GB budget is the aggregate host-DRAM pool for the
-# node; SimpleCPUOffloadConnector is sized per rank. At dram-utilization 0.63 on
-# cluster:b300-nv this resolves to ~220 GiB per rank across the 8 TP ranks.
-OFFLOAD_ARGS=()
-case "${KV_OFFLOAD_BACKEND:-}" in
- "")
- require_agentic_kv_offload_none
- ;;
- vllm-simple)
- require_agentic_kv_offload_backend vllm-simple
- CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / TP ))
- # Identical prefixes must hash to identical block keys run-to-run.
- export PYTHONHASHSEED=42
- # lazy_offload must be a JSON boolean, not a quoted string: the
- # connector does bool(extra_config.get("lazy_offload", False)), and
- # bool("false") is True in Python — a quoted "false" would silently
- # turn lazy offload ON. Eager offload keeps block-hash behaviour
- # aligned with the other B300 vllm-simple arms.
- OFFLOAD_ARGS=(
- --kv-transfer-config
- "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use_per_rank\":${CPU_BYTES_PER_RANK},\"lazy_offload\":false}}"
- )
- ;;
- *)
- echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty or vllm-simple)" >&2
- exit 1
- ;;
-esac
-
-# Agentic fan-out: keep the scheduler headroom convention shared by the other
-# agentic recipes. Capture decode graphs only up to that batch size — a 93-layer
-# 2.8T model makes capturing vLLM's full 2048-wide ladder prohibitively slow.
-MAX_NUM_SEQS=$((2 * CONC))
-
-echo "Starting vllm server..."
-
-{ set +x; } 2>/dev/null
-VLLM_CMD=(
- vllm serve "$MODEL_PATH" --served-model-name "$MODEL"
- --host 0.0.0.0
- --port "$PORT"
- --tensor-parallel-size "$TP"
- --gpu-memory-utilization 0.90
- --max-num-seqs "$MAX_NUM_SEQS"
- # Agentic replays run at the model's native context limit.
- --max-model-len 1048576
- --trust-remote-code
- --load-format fastsafetensors
- --moe-backend auto
- --enable-prefix-caching
- --kv-cache-dtype fp8
- --reasoning-parser kimi_k3
- --tool-call-parser kimi_k3
- --enable-auto-tool-choice
- # FP8 KV cache requires the prefill query quantization flag; MLA prefill
- # runs on FlashInfer per the production recipe.
- --attention-config '{"mla_prefill_backend":"FLASHINFER","use_prefill_query_quantization":true}'
- --max-cudagraph-capture-size "$MAX_NUM_SEQS"
- --disable-uvicorn-access-log
- "${OFFLOAD_ARGS[@]}"
-)
-printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt"
-printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt"
-"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 &
-SERVER_PID=$!
-echo "Server PID: $SERVER_PID"
-
-wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
-
-if [ "${EVAL_ONLY}" = "true" ]; then
- run_eval --port "$PORT"
-else
- build_replay_cmd "$RESULT_DIR"
- run_agentic_replay_and_write_outputs "$RESULT_DIR"
-fi
diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
index faf2232726..9bb187c8a9 100755
--- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
+++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh
@@ -181,7 +181,7 @@ SYNTHETIC_ACCEPT_LEN=2.51
# real target verification instead. Synthetic acceptance commits drafted tokens
# regardless of the target's logits, so the generated text is wrong and the
# SWE-bench eval scores 0.0000 -- the same split dsv4_fp4_b300_vllm_mtp.sh makes
-# (and which kimik2.5_fp4_b300_mtp.sh omits; follow dsv4, not kimik2.5).
+# (follow the DSV4 MTP precedent for this split).
# rejection_sample_method=block does real verification, so it is what EVAL_ONLY
# uses. vLLM rejects synthetic_acceptance_length unless the method is 'synthetic'.
if [ "${EVAL_ONLY:-false}" = "true" ]; then
diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh
deleted file mode 100644
index 7faaf69a51..0000000000
--- a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh
+++ /dev/null
@@ -1,148 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-# Agentic trace replay benchmark for Minimax-M3 FP4 on MI355X using vLLM.
-#
-# Required env vars:
-# MODEL, MODEL_PATH, TP, CONC, KV_OFFLOADING, KV_OFFLOAD_BACKEND,
-# TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, EP_SIZE, DP_ATTENTION
-
-source "$(dirname "$0")/../../benchmark_lib.sh"
-
-# Force the eval framework to lm-eval for this recipe. run_eval derives its
-# default as swebench for agentic scenarios (scenario_default=swebench when
-# IS_AGENTIC/SCENARIO_TYPE=agentic-coding), but EVAL_FRAMEWORK takes precedence
-# over that default (benchmark_lib.sh: framework=${EVAL_FRAMEWORK:-...}), so
-# setting it here makes the effective framework always lm-eval, never swebench.
-export EVAL_FRAMEWORK="lm-eval"
-
-check_env_vars MODEL TP CONC KV_OFFLOADING KV_OFFLOAD_BACKEND TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION
-
-echo "MODEL=$MODEL TP=$TP CONC=$CONC KV_OFFLOADING=$KV_OFFLOADING TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB RESULT_DIR=$RESULT_DIR DURATION=$DURATION EP_SIZE=$EP_SIZE DP_ATTENTION=$DP_ATTENTION"
-
-PORT=8888
-
-if [[ -n "${SLURM_JOB_ID+x}" ]]; then
- echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME"
-fi
-
-# ROCR/HIP visibility for vLLM 0.14+
-if [[ -n "${ROCR_VISIBLE_DEVICES+x}" ]]; then
- export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES"
-fi
-
-if [[ -n "${MODEL_PATH:-}" ]]; then
- if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
- hf download "$MODEL" --local-dir "$MODEL_PATH"
- fi
-else
- hf download "$MODEL"
- export MODEL_PATH="$MODEL"
-fi
-
-rocm-smi || true
-amd-smi || true
-
-resolve_trace_source
-install_agentic_deps
-
-# ---- Server config ----------------------------------------------------------
-SERVER_LOG="$RESULT_DIR/server.log"
-LMCACHE_LOG="$RESULT_DIR/lmcache_server.log"
-mkdir -p "$RESULT_DIR"
-
-OFFLOAD_ARGS=(--no-enable-prefix-caching)
-
-case "$KV_OFFLOAD_BACKEND" in
- vllm-simple)
- unset VLLM_USE_SIMPLE_KV_OFFLOAD
- # Use vLLM's regular native KV-offload path (OffloadingConnector),
- # NOT the SimpleCPUOffloadConnector. The "native" backend resolves to
- # OffloadingConnector by default; setting VLLM_USE_SIMPLE_KV_OFFLOAD=1
- # would switch it to SimpleCPUOffloadConnector. We intentionally leave
- # that env var UNSET here so the regular OffloadingConnector path is
- # used. The shortcut --kv_offloading_backend native + --kv_offloading_size
- # form constructs the KVTransferConfig at engine startup
- # (vllm/config/vllm.py:662).
-
- # Remove --disable-hybrid-kv-cache-manager and enable hybrid kv cache manager (default)
- # This gives extra cache hit than disabling hybrid kv cache manager
- OFFLOAD_ARGS=(
- --kv_offloading_backend native
- --kv_offloading_size "$TOTAL_CPU_DRAM_GB"
- )
- ;;
-esac
-
-# ---- LLM server config ----------------------------------------------------------
-PARALLEL_ARGS=(--tensor-parallel-size "$TP")
-if [ "${DP_ATTENTION}" = "true" ]; then
- PARALLEL_ARGS=(
- --tensor-parallel-size 1
- --data-parallel-size "$TP"
- --enable-expert-parallel
- )
-elif [ "$EP_SIZE" -gt 1 ]; then
- PARALLEL_ARGS+=(--enable-expert-parallel)
-fi
-
-echo "Starting vllm server..."
-export PYTHONNOUSERSITE=1
-
-export VLLM_ENGINE_READY_TIMEOUT_S=3600
-export VLLM_USE_BREAKABLE_CUDAGRAPH=0
-export VLLM_ROCM_USE_AITER=1
-export VLLM_ROCM_USE_AITER_MOE=1
-export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1
-# INT4 quantized all-reduce for the (~1.5 MB) decode all-reduces, which are the
-# single biggest decode kernel at high concurrency. The MIN_SIZE_KB override is
-# required: vLLM's default INT4 quick-reduce size gate for (bf16, TP4) is 16 MB,
-# so it never fires for decode-sized tensors without it.
-export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4
-export VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=0
-export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB=256
-
-VLLM_CMD=(
- vllm serve "$MODEL_PATH"
- --served-model-name "$MODEL"
- --host 0.0.0.0
- --port "$PORT"
- "${PARALLEL_ARGS[@]}"
- --trust-remote-code
- --block-size 128
- --gpu-memory-utilization 0.85
- --language-model-only
- --attention-backend TRITON_ATTN
- --moe-backend aiter
- --kv-cache-dtype fp8
- --tool-call-parser minimax_m3
- --enable-auto-tool-choice
- # NOTE: --reasoning-parser minimax_m3 is intentionally OMITTED.
- # MiniMax-M3 is an interleaved-thinking model: its ...
- # block MUST be round-tripped back into the conversation history every turn
- # or multi-turn quality collapses (the model loses its plan and degenerates
- # into repeating the same command until the step limit -> empty patch).
- # The reasoning parser moves out of message.content into the
- # response-only reasoning_content field, which the mini-swe-agent/litellm
- # OpenAI client does NOT resend. Leaving the parser off keeps the think block
- # inline in message.content, so the client preserves it across turns. The
- # tool-call parser above still extracts tool calls from the full output.
- --max-num-seqs "$CONC"
- "${OFFLOAD_ARGS[@]}"
-)
-printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt"
-printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt"
-"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 &
-SERVER_PID=$!
-echo "Server PID: $SERVER_PID"
-
-wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
-
-# ---- Run benchmark ----------------------------------------------------------
-if [ "${EVAL_ONLY}" = "true" ]; then
- run_eval --port "$PORT"
-else
- build_replay_cmd "$RESULT_DIR"
- run_agentic_replay_and_write_outputs "$RESULT_DIR"
-fi
diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_h100.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_h100.sh
deleted file mode 100755
index c5ab50b37f..0000000000
--- a/benchmarks/single_node/agentic/minimaxm3_fp8_h100.sh
+++ /dev/null
@@ -1,139 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-source "$(dirname "$0")/../../benchmark_lib.sh"
-
-check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION
-
-if [[ -n "${SLURM_JOB_ID:-}" ]]; then
- echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
-fi
-
-if [[ -n "${MODEL_PATH:-}" ]]; then
- if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
- hf download "$MODEL" --local-dir "$MODEL_PATH"
- fi
-else
- hf download "$MODEL"
- export MODEL_PATH="$MODEL"
-fi
-nvidia-smi
-
-export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126
-resolve_trace_source
-install_agentic_deps
-
-export VLLM_ENGINE_READY_TIMEOUT_S=3600
-export PYTHONNOUSERSITE=1
-
-SERVER_LOG="$RESULT_DIR/server.log"
-ROUTER_LOG="$RESULT_DIR/router.log"
-MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log"
-mkdir -p "$RESULT_DIR"
-
-OFFLOAD_ARGS=()
-MODEL_CPU_OFFLOAD_GB=26
-MODEL_CHECKPOINT_PAGE_CACHE_GIB=414
-MOONCAKE_LOCAL_BUFFER_GIB=4
-if require_agentic_kv_offload_backend mooncake; then
- TOTAL_CPU_DRAM_GIB=$((TOTAL_CPU_DRAM_GB * 1000000000 / 1073741824))
- PER_RANK_GIB=$(((TOTAL_CPU_DRAM_GIB - MODEL_CHECKPOINT_PAGE_CACHE_GIB) / TP - MODEL_CPU_OFFLOAD_GB - MOONCAKE_LOCAL_BUFFER_GIB))
- if (( PER_RANK_GIB <= 0 )); then
- echo "Error: CPU DRAM budget is too small for checkpoint cache, model, and KV offload" >&2
- exit 1
- fi
- MOONCAKE_VERSION=0.3.11.post1
- agentic_pip_install --quiet --no-cache-dir --no-deps \
- --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION"
- python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null
- MOONCAKE_MASTER_PORT=$((PORT + 12000))
- MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json"
- cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 &
- MOONCAKE_MASTER_PID=$!
- sleep 2
- kill -0 "$MOONCAKE_MASTER_PID"
- OFFLOAD_ARGS=(
- --kv-transfer-config
- '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}'
- )
-fi
-
-PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1)
-if [[ "$DP_ATTENTION" == "true" ]]; then
- PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP")
-fi
-
-EP_ARGS=()
-if (( EP_SIZE > 1 )); then
- EP_ARGS=(--enable-expert-parallel)
-fi
-
-VLLM_BACKEND_PORT="$PORT"
-if [[ "$DP_ATTENTION" == "true" ]]; then
- VLLM_BACKEND_PORT=$((PORT + 1))
- export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1
- agentic_pip_install --quiet 'vllm-router==0.1.14'
-fi
-
-MAX_NUM_SEQS=$((2 * CONC))
-vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \
- --host 0.0.0.0 \
- --port "$VLLM_BACKEND_PORT" \
- "${PARALLEL_ARGS[@]}" \
- "${EP_ARGS[@]}" \
- --gpu-memory-utilization 0.95 \
- --cpu-offload-gb "$MODEL_CPU_OFFLOAD_GB" \
- --kv-cache-dtype fp8 \
- --attention-backend TRITON_ATTN \
- --block-size 128 \
- --language-model-only \
- --enable-prefix-caching \
- --max-num-seqs "$MAX_NUM_SEQS" \
- --tool-call-parser minimax_m3 \
- --reasoning-parser minimax_m3 \
- --enable-auto-tool-choice \
- --safetensors-load-strategy lazy \
- --trust-remote-code \
- "${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 &
-SERVER_PID=$!
-
-wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
-
-if [[ "$DP_ATTENTION" == "true" ]]; then
- vllm-router \
- --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \
- --policy consistent_hash \
- --intra-node-data-parallel-size "$TP" \
- --host 0.0.0.0 \
- --port "$PORT" \
- --prometheus-host 127.0.0.1 \
- --prometheus-port "$((PORT + 10000))" \
- --request-timeout-secs 14400 \
- --disable-retries > "$ROUTER_LOG" 2>&1 &
- ROUTER_PID=$!
- wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID"
-fi
-
-if [ "${EVAL_ONLY}" = "true" ]; then
- run_eval --port "$PORT"
-else
- build_replay_cmd "$RESULT_DIR"
- run_agentic_replay_and_write_outputs "$RESULT_DIR"
-fi
diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_h200.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_h200.sh
deleted file mode 100755
index 6b984cee03..0000000000
--- a/benchmarks/single_node/agentic/minimaxm3_fp8_h200.sh
+++ /dev/null
@@ -1,181 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-source "$(dirname "$0")/../../benchmark_lib.sh"
-
-check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION
-
-if [[ -n "${SLURM_JOB_ID:-}" ]]; then
- echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
-fi
-
-resolve_complete_model_snapshot() {
- python3 - "$1" <<'PY'
-import json
-import sys
-from pathlib import Path
-
-model_cache_dir = Path(sys.argv[1])
-try:
- revision = model_cache_dir.joinpath("refs/main").read_text().strip()
-except OSError:
- raise SystemExit
-
-if not revision or Path(revision).name != revision:
- raise SystemExit
-
-snapshot = model_cache_dir / "snapshots" / revision
-index_path = snapshot / "model.safetensors.index.json"
-required_files = (
- snapshot / "config.json",
- snapshot / "tokenizer_config.json",
- index_path,
-)
-if not all(path.is_file() for path in required_files):
- raise SystemExit
-try:
- weight_map = json.loads(index_path.read_text())["weight_map"]
-except (KeyError, json.JSONDecodeError, OSError):
- raise SystemExit
-shards = {snapshot / filename for filename in weight_map.values()}
-if shards and all(path.is_file() for path in shards):
- print(snapshot)
-PY
-}
-
-if [[ -n "${MODEL_PATH:-}" ]]; then
- if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
- hf download "$MODEL" --local-dir "$MODEL_PATH"
- fi
-else
- MODEL_CACHE_ROOT="${HF_HUB_CACHE:-${HF_HOME:-$HOME/.cache/huggingface/hub}}"
- MODEL_CACHE_DIR="$MODEL_CACHE_ROOT/models--${MODEL//\//--}"
- mkdir -p "$MODEL_CACHE_ROOT"
- MODEL_PATH=$(resolve_complete_model_snapshot "$MODEL_CACHE_DIR")
- if [[ -z "$MODEL_PATH" ]]; then
- exec 9>"$MODEL_CACHE_ROOT/.minimaxm3-download.lock"
- flock -w 3600 9
- MODEL_PATH=$(resolve_complete_model_snapshot "$MODEL_CACHE_DIR")
- if [[ -z "$MODEL_PATH" ]]; then
- DOWNLOADED_MODEL_PATH=$(hf download "$MODEL")
- MODEL_PATH=$(resolve_complete_model_snapshot "$MODEL_CACHE_DIR")
- if [[ -z "$MODEL_PATH" ]]; then
- echo "Downloaded model snapshot is incomplete: $DOWNLOADED_MODEL_PATH" >&2
- exit 1
- fi
- fi
- flock -u 9
- fi
- echo "Using complete cached model snapshot: $MODEL_PATH"
- export MODEL_PATH
-fi
-nvidia-smi
-
-export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126
-resolve_trace_source
-install_agentic_deps
-
-export VLLM_ENGINE_READY_TIMEOUT_S=3600
-export PYTHONNOUSERSITE=1
-
-SERVER_LOG="$RESULT_DIR/server.log"
-ROUTER_LOG="$RESULT_DIR/router.log"
-MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log"
-mkdir -p "$RESULT_DIR"
-
-OFFLOAD_ARGS=()
-if require_agentic_kv_offload_backend mooncake; then
- PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / TP))
- MOONCAKE_VERSION=0.3.11.post1
- agentic_pip_install --quiet --no-cache-dir --no-deps \
- --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION"
- python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null
- MOONCAKE_MASTER_PORT=$((PORT + 12000))
- MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json"
- cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 &
- MOONCAKE_MASTER_PID=$!
- sleep 2
- kill -0 "$MOONCAKE_MASTER_PID"
- OFFLOAD_ARGS=(
- --kv-transfer-config
- '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}'
- )
-fi
-
-PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1)
-if [[ "$DP_ATTENTION" == "true" ]]; then
- PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP")
-fi
-
-EP_ARGS=()
-if (( EP_SIZE > 1 )); then
- EP_ARGS=(--enable-expert-parallel)
-fi
-
-VLLM_BACKEND_PORT="$PORT"
-if [[ "$DP_ATTENTION" == "true" ]]; then
- VLLM_BACKEND_PORT=$((PORT + 1))
- export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1
- agentic_pip_install --quiet 'vllm-router==0.1.14'
-fi
-
-MAX_NUM_SEQS=$((2 * CONC))
-vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \
- --host 0.0.0.0 \
- --port "$VLLM_BACKEND_PORT" \
- "${PARALLEL_ARGS[@]}" \
- "${EP_ARGS[@]}" \
- --gpu-memory-utilization 0.92 \
- --kv-cache-dtype fp8 \
- --attention-backend TRITON_ATTN \
- --block-size 128 \
- --language-model-only \
- --enable-prefix-caching \
- --max-num-seqs "$MAX_NUM_SEQS" \
- --tool-call-parser minimax_m3 \
- --reasoning-parser minimax_m3 \
- --enable-auto-tool-choice \
- --trust-remote-code \
- "${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 &
-SERVER_PID=$!
-
-wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
-
-if [[ "$DP_ATTENTION" == "true" ]]; then
- vllm-router \
- --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \
- --policy consistent_hash \
- --intra-node-data-parallel-size "$TP" \
- --host 0.0.0.0 \
- --port "$PORT" \
- --prometheus-host 127.0.0.1 \
- --prometheus-port "$((PORT + 10000))" \
- --request-timeout-secs 14400 \
- --disable-retries > "$ROUTER_LOG" 2>&1 &
- ROUTER_PID=$!
- wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID"
-fi
-
-if [ "${EVAL_ONLY}" = "true" ]; then
- run_eval --port "$PORT"
-else
- build_replay_cmd "$RESULT_DIR"
- run_agentic_replay_and_write_outputs "$RESULT_DIR"
-fi
diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x.sh
deleted file mode 100755
index 4dc06f77af..0000000000
--- a/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x.sh
+++ /dev/null
@@ -1,208 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-source "$(dirname "$0")/../../benchmark_lib.sh"
-
-check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION
-
-if [[ -n "${SLURM_JOB_ID:-}" ]]; then
- echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
-fi
-
-if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then
- export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES"
-fi
-
-if [[ -n "${MODEL_PATH:-}" ]]; then
- if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
- hf download "$MODEL" --local-dir "$MODEL_PATH"
- fi
-else
- hf download "$MODEL"
- export MODEL_PATH="$MODEL"
-fi
-rocm-smi || true
-amd-smi || true
-
-export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126
-resolve_trace_source
-install_agentic_deps
-
-export VLLM_ENGINE_READY_TIMEOUT_S=3600
-export VLLM_USE_BREAKABLE_CUDAGRAPH=0
-export PYTHONNOUSERSITE=1
-
-SERVER_LOG="$RESULT_DIR/server.log"
-ROUTER_LOG="$RESULT_DIR/router.log"
-MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log"
-mkdir -p "$RESULT_DIR"
-
-install_mooncake_rocm() {
- local mooncake_tag="v0.3.11.post1"
- local mooncake_src="/tmp/Mooncake-$mooncake_tag"
- local mooncake_stage="/tmp/mooncake-stage-$mooncake_tag"
- local build_jobs
- local cache_root
- local cache_key
- local cache_archive
- local cache_tmp
- local engine_path
- local os_version
- local python_abi
- local rocm_version
-
- build_jobs=$(nproc)
- if ((build_jobs > 32)); then
- build_jobs=32
- fi
-
- os_version=$(. /etc/os-release && printf '%s-%s' "$ID" "$VERSION_ID")
- python_abi=$(python3 -c 'import sys; print(f"cp{sys.version_info.major}{sys.version_info.minor}")')
- rocm_version=$(sed -n '1p' /opt/rocm/.info/version 2>/dev/null || true)
- if [[ -z "$rocm_version" ]]; then
- rocm_version=$(hipconfig --version)
- fi
- rocm_version=${rocm_version//[^[:alnum:]._-]/_}
- cache_root="${HF_HUB_CACHE:?HF_HUB_CACHE must be set}/inferencex/mooncake"
- cache_key="${mooncake_tag}-${os_version}-${python_abi}-${rocm_version}-$(uname -m)-hip"
- cache_archive="$cache_root/$cache_key.tar.gz"
- mkdir -p "$cache_root"
-
- apt-get update
- DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
- build-essential cmake git libasio-dev libboost-dev libcurl4-openssl-dev \
- libgflags-dev libgoogle-glog-dev libibverbs-dev libjsoncpp-dev \
- libnuma-dev libpython3-dev libssl-dev libunwind-dev liburing-dev \
- libxxhash-dev libyaml-cpp-dev libzstd-dev ninja-build pybind11-dev
-
- exec 9>"$cache_archive.lock"
- flock -w 1800 9
- if [[ -f "$cache_archive" ]] && ! tar -tzf "$cache_archive" >/dev/null 2>&1; then
- rm -f "$cache_archive"
- fi
- if [[ ! -f "$cache_archive" ]]; then
- echo "Building HIP Mooncake cache artifact: $cache_archive"
- rm -rf "$mooncake_src" "$mooncake_stage"
- git clone --depth 1 --branch "$mooncake_tag" --recurse-submodules \
- --shallow-submodules https://github.com/kvcache-ai/Mooncake.git "$mooncake_src"
- cmake -S "$mooncake_src/extern/yalantinglibs" \
- -B "$mooncake_src/extern/yalantinglibs/build" \
- -DBUILD_EXAMPLES=OFF -DBUILD_BENCHMARK=OFF -DBUILD_UNIT_TESTS=OFF
- cmake --build "$mooncake_src/extern/yalantinglibs/build" -j "$build_jobs"
- cmake --install "$mooncake_src/extern/yalantinglibs/build"
- cmake -S "$mooncake_src" -B "$mooncake_src/build" -G Ninja \
- -DCMAKE_BUILD_TYPE=Release -DUSE_CUDA=OFF -DUSE_HIP=ON \
- -DWITH_EP=OFF -DWITH_STORE=ON -DWITH_STORE_RUST=OFF \
- -DWITH_RUST_EXAMPLE=OFF -DBUILD_EXAMPLES=OFF -DBUILD_UNIT_TESTS=OFF
- cmake --build "$mooncake_src/build" -j "$build_jobs"
- mkdir -p "$mooncake_stage"
- DESTDIR="$mooncake_stage" cmake --install "$mooncake_src/build"
- cache_tmp=$(mktemp "$cache_root/$cache_key.tmp.XXXXXX")
- tar -C "$mooncake_stage" -czf "$cache_tmp" .
- mv -f "$cache_tmp" "$cache_archive"
- else
- echo "Using HIP Mooncake cache artifact: $cache_archive"
- fi
- tar -C / -xzf "$cache_archive"
- engine_path=$(python3 -c 'import mooncake.engine; print(mooncake.engine.__file__)')
- ldd "$engine_path" | grep -q 'libamdhip64.so'
- exec 9>&-
-}
-
-OFFLOAD_ARGS=()
-if require_agentic_kv_offload_backend mooncake; then
- PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / TP))
- if ! python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null 2>&1; then
- install_mooncake_rocm
- fi
- python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null
- MOONCAKE_MASTER_PORT=$((PORT + 12000))
- MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json"
- cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 &
- MOONCAKE_MASTER_PID=$!
- sleep 2
- kill -0 "$MOONCAKE_MASTER_PID"
- OFFLOAD_ARGS=(
- --kv-transfer-config
- '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}'
- )
-fi
-
-PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1)
-if [[ "$DP_ATTENTION" == "true" ]]; then
- PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP")
-fi
-
-EP_ARGS=()
-if (( EP_SIZE > 1 )); then
- EP_ARGS=(--enable-expert-parallel)
-fi
-
-VLLM_BACKEND_PORT="$PORT"
-if [[ "$DP_ATTENTION" == "true" ]]; then
- VLLM_BACKEND_PORT=$((PORT + 1))
- export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1
- agentic_pip_install --quiet 'vllm-router==0.1.14'
-fi
-
-MAX_NUM_SEQS=$((2 * CONC))
-vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \
- --host 0.0.0.0 \
- --port "$VLLM_BACKEND_PORT" \
- "${PARALLEL_ARGS[@]}" \
- "${EP_ARGS[@]}" \
- --gpu-memory-utilization 0.95 \
- --block-size 128 \
- --language-model-only \
- --attention-backend TRITON_ATTN \
- --kv-cache-dtype fp8 \
- --enable-prefix-caching \
- --max-num-seqs "$MAX_NUM_SEQS" \
- --tool-call-parser minimax_m3 \
- --reasoning-parser minimax_m3 \
- --enable-auto-tool-choice \
- --trust-remote-code \
- "${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 &
-SERVER_PID=$!
-
-wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
-
-if [[ "$DP_ATTENTION" == "true" ]]; then
- vllm-router \
- --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \
- --policy consistent_hash \
- --intra-node-data-parallel-size "$TP" \
- --host 0.0.0.0 \
- --port "$PORT" \
- --prometheus-host 127.0.0.1 \
- --prometheus-port "$((PORT + 10000))" \
- --request-timeout-secs 14400 \
- --disable-retries > "$ROUTER_LOG" 2>&1 &
- ROUTER_PID=$!
- wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID"
-fi
-
-if [ "${EVAL_ONLY}" = "true" ]; then
- run_eval --port "$PORT"
-else
- build_replay_cmd "$RESULT_DIR"
- run_agentic_replay_and_write_outputs "$RESULT_DIR"
-fi
diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x.sh
deleted file mode 100755
index 7cb862ae88..0000000000
--- a/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x.sh
+++ /dev/null
@@ -1,218 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-source "$(dirname "$0")/../../benchmark_lib.sh"
-
-check_env_vars MODEL IMAGE TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION
-
-if [[ -n "${SLURM_JOB_ID:-}" ]]; then
- echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
-fi
-
-if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then
- export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES"
-fi
-
-if [[ -n "${MODEL_PATH:-}" ]]; then
- if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
- hf download "$MODEL" --local-dir "$MODEL_PATH"
- fi
-else
- hf download "$MODEL"
- export MODEL_PATH="$MODEL"
-fi
-rocm-smi || true
-amd-smi || true
-
-export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126
-resolve_trace_source
-install_agentic_deps
-
-export VLLM_ENGINE_READY_TIMEOUT_S=3600
-export VLLM_USE_BREAKABLE_CUDAGRAPH=0
-export PYTHONNOUSERSITE=1
-
-SERVER_LOG="$RESULT_DIR/server.log"
-ROUTER_LOG="$RESULT_DIR/router.log"
-MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log"
-mkdir -p "$RESULT_DIR"
-
-install_mooncake_rocm() {
- local mooncake_tag="v0.3.11.post1"
- local mooncake_src="/tmp/Mooncake-$mooncake_tag"
- local mooncake_stage="/tmp/mooncake-stage-$mooncake_tag"
- local build_jobs
- local cache_root
- local cache_key
- local cache_archive
- local cache_tmp
- local engine_path
- local os_version
- local python_abi
- local rocm_version
-
- build_jobs=$(nproc)
- if ((build_jobs > 32)); then
- build_jobs=32
- fi
-
- os_version=$(. /etc/os-release && printf '%s-%s' "$ID" "$VERSION_ID")
- python_abi=$(python3 -c 'import sys; print(f"cp{sys.version_info.major}{sys.version_info.minor}")')
- rocm_version=$(sed -n '1p' /opt/rocm/.info/version 2>/dev/null || true)
- if [[ -z "$rocm_version" ]]; then
- rocm_version=$(hipconfig --version)
- fi
- rocm_version=${rocm_version//[^[:alnum:]._-]/_}
- cache_root="${HF_HUB_CACHE:?HF_HUB_CACHE must be set}/inferencex/mooncake"
- cache_key="${mooncake_tag}-${os_version}-${python_abi}-${rocm_version}-$(uname -m)-hip"
- cache_archive="$cache_root/$cache_key.tar.gz"
- mkdir -p "$cache_root"
-
- apt-get update
- DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
- build-essential cmake git libasio-dev libboost-dev libcurl4-openssl-dev \
- libgflags-dev libgoogle-glog-dev libibverbs-dev libjsoncpp-dev \
- libnuma-dev libpython3-dev libssl-dev libunwind-dev liburing-dev \
- libxxhash-dev libyaml-cpp-dev libzstd-dev ninja-build pybind11-dev
-
- exec 9>"$cache_archive.lock"
- flock -w 1800 9
- if [[ -f "$cache_archive" ]] && ! tar -tzf "$cache_archive" >/dev/null 2>&1; then
- rm -f "$cache_archive"
- fi
- if [[ ! -f "$cache_archive" ]]; then
- echo "Building HIP Mooncake cache artifact: $cache_archive"
- rm -rf "$mooncake_src" "$mooncake_stage"
- git clone --depth 1 --branch "$mooncake_tag" --recurse-submodules \
- --shallow-submodules https://github.com/kvcache-ai/Mooncake.git "$mooncake_src"
- cmake -S "$mooncake_src/extern/yalantinglibs" \
- -B "$mooncake_src/extern/yalantinglibs/build" \
- -DBUILD_EXAMPLES=OFF -DBUILD_BENCHMARK=OFF -DBUILD_UNIT_TESTS=OFF
- cmake --build "$mooncake_src/extern/yalantinglibs/build" -j "$build_jobs"
- cmake --install "$mooncake_src/extern/yalantinglibs/build"
- cmake -S "$mooncake_src" -B "$mooncake_src/build" -G Ninja \
- -DCMAKE_BUILD_TYPE=Release -DUSE_CUDA=OFF -DUSE_HIP=ON \
- -DWITH_EP=OFF -DWITH_STORE=ON -DWITH_STORE_RUST=OFF \
- -DWITH_RUST_EXAMPLE=OFF -DBUILD_EXAMPLES=OFF -DBUILD_UNIT_TESTS=OFF
- cmake --build "$mooncake_src/build" -j "$build_jobs"
- mkdir -p "$mooncake_stage"
- DESTDIR="$mooncake_stage" cmake --install "$mooncake_src/build"
- cache_tmp=$(mktemp "$cache_root/$cache_key.tmp.XXXXXX")
- tar -C "$mooncake_stage" -czf "$cache_tmp" .
- mv -f "$cache_tmp" "$cache_archive"
- else
- echo "Using HIP Mooncake cache artifact: $cache_archive"
- fi
- tar -C / -xzf "$cache_archive"
- engine_path=$(python3 -c 'import mooncake.engine; print(mooncake.engine.__file__)')
- ldd "$engine_path" | grep -q 'libamdhip64.so'
- exec 9>&-
-}
-
-OFFLOAD_ARGS=()
-if require_agentic_kv_offload_backend mooncake; then
- PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / TP))
- if ! python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null 2>&1; then
- install_mooncake_rocm
- fi
- python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null
- MOONCAKE_MASTER_PORT=$((PORT + 12000))
- MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json"
- cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 &
- MOONCAKE_MASTER_PID=$!
- sleep 2
- kill -0 "$MOONCAKE_MASTER_PID"
- OFFLOAD_ARGS=(
- --kv-transfer-config
- '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}'
- )
-fi
-
-PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1)
-if [[ "$DP_ATTENTION" == "true" ]]; then
- PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP")
-fi
-
-EP_ARGS=()
-if (( EP_SIZE > 1 )); then
- EP_ARGS=(--enable-expert-parallel)
-fi
-
-KV_CACHE_ARGS=()
-if [[ "$IMAGE" == vllm/vllm-openai-rocm:nightly-* ]]; then
- KV_CACHE_ARGS=(--kv-cache-dtype fp8)
-fi
-
-VLLM_BACKEND_PORT="$PORT"
-if [[ "$DP_ATTENTION" == "true" ]]; then
- VLLM_BACKEND_PORT=$((PORT + 1))
- export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1
- agentic_pip_install --quiet 'vllm-router==0.1.14'
-fi
-
-MAX_NUM_SEQS=$((2 * CONC))
-GPU_MEMORY_UTILIZATION=0.95
-if (( TP == 4 )); then
- GPU_MEMORY_UTILIZATION=0.98
-fi
-
-vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \
- --host 0.0.0.0 \
- --port "$VLLM_BACKEND_PORT" \
- "${PARALLEL_ARGS[@]}" \
- "${EP_ARGS[@]}" \
- --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
- --block-size 128 \
- --language-model-only \
- --attention-backend TRITON_ATTN \
- --enable-prefix-caching \
- --max-num-seqs "$MAX_NUM_SEQS" \
- --tool-call-parser minimax_m3 \
- --reasoning-parser minimax_m3 \
- --enable-auto-tool-choice \
- --trust-remote-code \
- "${KV_CACHE_ARGS[@]}" \
- "${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 &
-SERVER_PID=$!
-
-wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
-
-if [[ "$DP_ATTENTION" == "true" ]]; then
- vllm-router \
- --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \
- --policy consistent_hash \
- --intra-node-data-parallel-size "$TP" \
- --host 0.0.0.0 \
- --port "$PORT" \
- --prometheus-host 127.0.0.1 \
- --prometheus-port "$((PORT + 10000))" \
- --request-timeout-secs 14400 \
- --disable-retries > "$ROUTER_LOG" 2>&1 &
- ROUTER_PID=$!
- wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID"
-fi
-
-if [ "${EVAL_ONLY}" = "true" ]; then
- run_eval --port "$PORT"
-else
- build_replay_cmd "$RESULT_DIR"
- run_agentic_replay_and_write_outputs "$RESULT_DIR"
-fi
diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_b200.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_b200.sh
deleted file mode 100755
index 58857e73c5..0000000000
--- a/benchmarks/single_node/agentic/qwen3.5_fp8_b200.sh
+++ /dev/null
@@ -1,79 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-# Agentic trace replay benchmark for Qwen3.5 FP8 on B200 using SGLang.
-#
-# Required env vars:
-# MODEL, TP, CONC, RESULT_DIR
-
-source "$(dirname "$0")/../../benchmark_lib.sh"
-
-check_env_vars MODEL TP CONC RESULT_DIR DURATION EP_SIZE
-
-SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-10}
-
-if [[ -n "${SLURM_JOB_ID:-}" ]]; then
- echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
-fi
-
-# `hf download` creates the target dir if missing and is itself idempotent.
-# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
-# Either way, MODEL_PATH is what the server is launched with.
-if [[ -n "${MODEL_PATH:-}" ]]; then
- if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
- hf download "$MODEL" --local-dir "$MODEL_PATH"
- fi
-else
- hf download "$MODEL"
- export MODEL_PATH="$MODEL"
-fi
-nvidia-smi
-
-# ---- Resolve traces and install deps ----------------------------------------
-resolve_trace_source
-install_agentic_deps
-
-# ---- Start SGLang server ----------------------------------------------------
-SERVER_LOG="$RESULT_DIR/server.log"
-mkdir -p "$RESULT_DIR"
-
-echo "Starting SGLang server..."
-export TORCH_CUDA_ARCH_LIST="10.0"
-export PYTHONNOUSERSITE=1
-export NCCL_NVLS_ENABLE=1
-export SGL_ENABLE_JIT_DEEPGEMM=false
-export SGLANG_ENABLE_FLASHINFER_GEMM=true
-
-python3 -m sglang.launch_server \
---model-path=$MODEL_PATH --served-model-name=$MODEL \
---host=0.0.0.0 \
---port=$PORT \
---served-model-name "Qwen/Qwen3.5-397B-A17B-FP8" \
---trust-remote-code \
---tensor-parallel-size=$TP \
---data-parallel-size=1 \
---ep-size $EP_SIZE \
---cuda-graph-max-bs $CONC \
---max-running-requests $CONC \
---mem-fraction-static 0.82 \
---chunked-prefill-size 32768 \
---max-prefill-tokens 32768 \
---attention-backend trtllm_mha \
---moe-runner-backend flashinfer_trtllm \
---enable-flashinfer-allreduce-fusion \
---scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \
---tokenizer-worker-num 6 \
---stream-interval 30 \
---enable-metrics > "$SERVER_LOG" 2>&1 &
-SERVER_PID=$!
-echo "Server PID: $SERVER_PID"
-
-wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
-
-if [ "${EVAL_ONLY}" = "true" ]; then
- run_eval --port "$PORT"
-else
- build_replay_cmd "$RESULT_DIR"
- run_agentic_replay_and_write_outputs "$RESULT_DIR"
-fi
diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang.sh
deleted file mode 100755
index 8ae038dd7e..0000000000
--- a/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang.sh
+++ /dev/null
@@ -1,129 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-# Agentic trace replay benchmark for Qwen3.5 FP8 on B300 using SGLang.
-#
-# Required env vars:
-# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
-#
-# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache.
-
-source "$(dirname "$0")/../../benchmark_lib.sh"
-
-check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE
-
-SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-10}
-
-if [[ -n "${SLURM_JOB_ID:-}" ]]; then
- echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
-fi
-
-# `hf download` creates the target dir if missing and is itself idempotent.
-# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
-# Either way, MODEL_PATH is what the server is launched with.
-if [[ -n "${MODEL_PATH:-}" ]]; then
- if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
- hf download "$MODEL" --local-dir "$MODEL_PATH"
- fi
-else
- hf download "$MODEL"
- export MODEL_PATH="$MODEL"
-fi
-nvidia-smi
-
-# ---- Resolve traces and install deps ----------------------------------------
-resolve_trace_source
-install_agentic_deps
-
-# ---- Server config ----------------------------------------------------------
-SERVER_LOG="$RESULT_DIR/server.log"
-mkdir -p "$RESULT_DIR"
-
-CACHE_ARGS=()
-if require_agentic_kv_offload_backend hicache; then
- # HiCache extends RadixAttention, so do not pass --disable-radix-cache.
- # Qwen3.5's hybrid GDN/Mamba path allocates two HiCache host pools per TP
- # rank: one for hierarchical KV cache and one for hierarchical Mamba cache.
- REQUESTED_HICACHE_TOTAL_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-$TOTAL_CPU_DRAM_GB}"
- if [ "$REQUESTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then
- echo "Error: requested HiCache pool ${REQUESTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2
- exit 1
- fi
- TOTAL_CPU_DRAM_GB="$REQUESTED_HICACHE_TOTAL_GB"
- HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-2}"
- HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}"
- # SGLang --hicache-size is per rank per host pool, while the workflow
- # input is a node-total DRAM budget. Divide by TP and the number of
- # host pools unless HICACHE_SIZE_GB is set directly for one-off tuning.
- MAX_HICACHE_SIZE_GB=$((TOTAL_CPU_DRAM_GB / TP / HICACHE_HOST_POOL_COUNT))
- HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}"
- if [ "$HICACHE_SIZE_GB" -gt "$MAX_HICACHE_SIZE_GB" ]; then
- echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB exceeds configured per-pool limit $MAX_HICACHE_SIZE_GB" >&2
- exit 1
- fi
- if [ "$HICACHE_SIZE_GB" -lt 1 ]; then
- echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB from TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB, TP=$TP, HICACHE_HOST_POOL_COUNT=$HICACHE_HOST_POOL_COUNT" >&2
- exit 1
- fi
- echo "HiCache CPU pool: ${HICACHE_SIZE_GB} GB per rank per host pool across TP=${TP}, host_pool_count=${HICACHE_HOST_POOL_COUNT}"
- CACHE_ARGS=(
- --page-size 64
- --enable-hierarchical-cache
- --hicache-size "$HICACHE_SIZE_GB"
- --hicache-io-backend kernel
- --hicache-mem-layout page_first
- --hicache-write-policy "$HICACHE_WRITE_POLICY"
- )
-fi
-
-echo "Starting SGLang server..."
-export TORCH_CUDA_ARCH_LIST="10.0"
-export PYTHONNOUSERSITE=1
-export NCCL_NVLS_ENABLE=1
-export SGL_ENABLE_JIT_DEEPGEMM=false
-export SGLANG_ENABLE_FLASHINFER_GEMM=true
-
-{ set +x; } 2>/dev/null
-SGLANG_CMD=(
- python3 -m sglang.launch_server
- --model-path="$MODEL_PATH" --served-model-name="$MODEL"
- --host=0.0.0.0
- --port="$PORT"
- --served-model-name "Qwen/Qwen3.5-397B-A17B-FP8"
- --trust-remote-code
- --tensor-parallel-size="$TP"
- --data-parallel-size=1
- --expert-parallel-size="$EP_SIZE"
- --enable-symm-mem
- --quantization fp8
- --kv-cache-dtype fp8_e4m3
- --mamba-ssm-dtype bfloat16
- --attention-backend trtllm_mha
- --moe-runner-backend flashinfer_trtllm
- --cuda-graph-max-bs "$CONC"
- --max-running-requests "$CONC"
- --max-prefill-tokens 16384
- --chunked-prefill-size 16384
- --mem-fraction-static 0.80
- --stream-interval 50
- --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL"
- --tokenizer-worker-num 6
- --tokenizer-path "$MODEL"
- --enable-metrics
- "${CACHE_ARGS[@]}"
-)
-printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt"
-printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt"
-"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 &
-SERVER_PID=$!
-echo "Server PID: $SERVER_PID"
-
-wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
-
-if [ "${EVAL_ONLY}" = "true" ]; then
- run_eval --port "$PORT"
-else
- build_replay_cmd "$RESULT_DIR"
- run_agentic_replay_and_write_outputs "$RESULT_DIR"
-fi
diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_h100.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_h100.sh
deleted file mode 100755
index f58952a434..0000000000
--- a/benchmarks/single_node/agentic/qwen3.5_fp8_h100.sh
+++ /dev/null
@@ -1,143 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-# Agentic trace replay benchmark for Qwen3.5 FP8 on H100 using SGLang.
-#
-# H100 has 80 GB HBM3 (vs B300's 192 GB), so weights + KV fit tighter.
-# Mem-fraction-static lowered to 0.75 and chunked-prefill-size halved to
-# 8192 (mirrors fixed_seq_len/qwen3.5_fp8_h100.sh). Attention backend is
-# flashinfer (sm_90); the trtllm_mha path is Blackwell-only.
-#
-# Required env vars:
-# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
-#
-# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache.
-
-source "$(dirname "$0")/../../benchmark_lib.sh"
-
-check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE
-
-SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-10}
-
-if [[ -n "${SLURM_JOB_ID:-}" ]]; then
- echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
-fi
-
-# `hf download` creates the target dir if missing and is itself idempotent.
-# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
-# Either way, MODEL_PATH is what the server is launched with.
-if [[ -n "${MODEL_PATH:-}" ]]; then
- if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
- hf download "$MODEL" --local-dir "$MODEL_PATH"
- fi
-else
- hf download "$MODEL"
- export MODEL_PATH="$MODEL"
-fi
-nvidia-smi
-
-# ---- Resolve traces and install deps ----------------------------------------
-# H100 max_model_len caps at 131k (HBM-bound). The unfiltered with-subagents
-# corpus has requests up to ~1M proxy tokens that the server would reject.
-# Switch to the 256k-capped variant (470 traces, max in+out <= 256k); even
-# at 131k context, the rejection rate is much lower than against the
-# unfiltered corpus.
-export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_with_subagents_256k
-
-resolve_trace_source
-install_agentic_deps
-
-# ---- Server config ----------------------------------------------------------
-SERVER_LOG="$RESULT_DIR/server.log"
-mkdir -p "$RESULT_DIR"
-
-CACHE_ARGS=()
-if require_agentic_kv_offload_backend hicache; then
- # HiCache extends RadixAttention, so do not pass --disable-radix-cache.
- # Hybrid GDN/Mamba allocates one KV and one Mamba host pool per rank.
- REQUESTED_HICACHE_TOTAL_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-$TOTAL_CPU_DRAM_GB}"
- if [ "$REQUESTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then
- echo "Error: requested HiCache pool ${REQUESTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2
- exit 1
- fi
- TOTAL_CPU_DRAM_GB="$REQUESTED_HICACHE_TOTAL_GB"
- HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-2}"
- HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}"
- MAX_HICACHE_SIZE_GB=$((TOTAL_CPU_DRAM_GB / TP / HICACHE_HOST_POOL_COUNT))
- HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}"
- if [ "$HICACHE_SIZE_GB" -gt "$MAX_HICACHE_SIZE_GB" ]; then
- echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB exceeds configured per-pool limit $MAX_HICACHE_SIZE_GB" >&2
- exit 1
- fi
- if [ "$HICACHE_SIZE_GB" -lt 1 ]; then
- echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB from TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB, TP=$TP, HICACHE_HOST_POOL_COUNT=$HICACHE_HOST_POOL_COUNT" >&2
- exit 1
- fi
- echo "HiCache CPU pool: ${HICACHE_SIZE_GB} GB per rank per host pool across TP=${TP}, host_pool_count=${HICACHE_HOST_POOL_COUNT}"
- CACHE_ARGS=(
- --page-size 64
- --enable-hierarchical-cache
- --hicache-size "$HICACHE_SIZE_GB"
- --hicache-io-backend kernel
- --hicache-mem-layout page_first
- --hicache-write-policy "$HICACHE_WRITE_POLICY"
- )
-fi
-
-echo "Starting SGLang server..."
-export PYTHONNOUSERSITE=1
-
-SGLANG_MULTI_TOKENIZER=/sgl-workspace/sglang/python/sglang/srt/managers/multi_tokenizer_mixin.py
-if ! sed -n '/elif isinstance(output, BatchStrOutput):/,/input_token_logprobs_val=_extract_field_by_index/p' "$SGLANG_MULTI_TOKENIZER" \
- | grep -q 'cached_tokens_details=_extract_field_by_index'; then
- sed -i '/elif isinstance(output, BatchStrOutput):/,/input_token_logprobs_val=_extract_field_by_index/ {
- /cached_tokens=_extract_field_by_index(output, "cached_tokens", i),/a\
- cached_tokens_details=_extract_field_by_index(\
- output, "cached_tokens_details", i\
- ),
- }' "$SGLANG_MULTI_TOKENIZER"
-fi
-
-{ set +x; } 2>/dev/null
-SGLANG_CMD=(
- python3 -m sglang.launch_server
- --model-path="$MODEL_PATH" --served-model-name="$MODEL"
- --host=0.0.0.0
- --port="$PORT"
- --served-model-name "Qwen/Qwen3.5-397B-A17B-FP8"
- --trust-remote-code
- --tensor-parallel-size="$TP"
- --data-parallel-size=1
- --expert-parallel-size="$EP_SIZE"
- --quantization fp8
- --kv-cache-dtype fp8_e4m3
- --mamba-ssm-dtype bfloat16
- --attention-backend flashinfer
- --enable-flashinfer-allreduce-fusion
- # --cuda-graph-max-bs "$CONC"
- # --max-running-requests "$CONC"
- # --max-prefill-tokens 8192
- # --chunked-prefill-size 8192
- --mem-fraction-static 0.75
- --stream-interval 50
- --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL"
- --tokenizer-worker-num 6
- --tokenizer-path "$MODEL"
- --enable-metrics
- "${CACHE_ARGS[@]}"
-)
-printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt"
-printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt"
-"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 &
-SERVER_PID=$!
-echo "Server PID: $SERVER_PID"
-
-wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
-
-if [ "${EVAL_ONLY}" = "true" ]; then
- run_eval --port "$PORT"
-else
- build_replay_cmd "$RESULT_DIR"
- run_agentic_replay_and_write_outputs "$RESULT_DIR"
-fi
diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_h100_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_h100_mtp.sh
index 95dae3834f..41527c775d 100755
--- a/benchmarks/single_node/agentic/qwen3.5_fp8_h100_mtp.sh
+++ b/benchmarks/single_node/agentic/qwen3.5_fp8_h100_mtp.sh
@@ -3,8 +3,7 @@ set -euo pipefail
set -x
# Agentic trace replay benchmark for Qwen3.5 FP8 on H100 using SGLang with MTP
-# speculative decoding -- the spec-decoding=mtp variant of
-# agentic/qwen3.5_fp8_h100.sh.
+# speculative decoding.
#
# H100 has 80 GB HBM3 (vs B300's 192 GB), so weights + KV fit tighter.
# Mem-fraction-static lowered to 0.75 and chunked-prefill-size halved to
diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh
index 62982a3974..518cd7e17e 100755
--- a/benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh
+++ b/benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh
@@ -7,7 +7,7 @@ set -x
# only, per the AgentX policy that new agentic arms ship with speculative
# decoding enabled rather than as an STP/MTP A/B (MODELS.md).
#
-# Structure follows the proven agentic/qwen3.5_fp8_h100.sh replay path
+# Structure follows the proven H100 MTP AgentX replay path
# (HiCache host-DRAM offload, the multi_tokenizer cached_tokens_details patch,
# aiperf-driven trace replay). H200's 141 GB HBM3e is roomier than H100's 80 GB,
# so --mem-fraction-static is 0.8 rather than 0.75, matching
diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_mi355x.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_mi355x.sh
deleted file mode 100755
index 0bcf3cf163..0000000000
--- a/benchmarks/single_node/agentic/qwen3.5_fp8_mi355x.sh
+++ /dev/null
@@ -1,69 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-# Agentic trace replay benchmark for Qwen3.5 FP8 on MI355X using SGLang.
-#
-# Required env vars:
-# MODEL, TP, CONC, RESULT_DIR
-
-source "$(dirname "$0")/../../benchmark_lib.sh"
-
-check_env_vars MODEL TP CONC RESULT_DIR DURATION EP_SIZE
-
-if [[ -n "${SLURM_JOB_ID:-}" ]]; then
- echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
-fi
-
-# `hf download` creates the target dir if missing and is itself idempotent.
-# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
-# Either way, MODEL_PATH is what the server is launched with.
-if [[ -n "${MODEL_PATH:-}" ]]; then
- if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
- hf download "$MODEL" --local-dir "$MODEL_PATH"
- fi
-else
- hf download "$MODEL"
- export MODEL_PATH="$MODEL"
-fi
-rocm-smi || true
-amd-smi || true
-
-# ---- Resolve traces and install deps ----------------------------------------
-resolve_trace_source
-install_agentic_deps
-
-# ---- Start SGLang server ----------------------------------------------------
-SERVER_LOG="$RESULT_DIR/server.log"
-mkdir -p "$RESULT_DIR"
-
-echo "Starting SGLang server..."
-export PYTHONNOUSERSITE=1
-
-python3 -m sglang.launch_server \
- --attention-backend triton \
- --model-path "$MODEL_PATH" --served-model-name "$MODEL" \
- --host=0.0.0.0 \
- --port $PORT \
- --tensor-parallel-size $TP \
- --ep-size $EP_SIZE \
- --trust-remote-code \
- --tokenizer-worker-num 6 \
- --enable-aiter-allreduce-fusion \
- --cuda-graph-max-bs $CONC \
- --max-running-requests $CONC \
- --max-prefill-tokens 32768 \
- --scheduler-recv-interval 30 \
- --mem-fraction-static 0.8 \
- --enable-metrics > "$SERVER_LOG" 2>&1 &
-SERVER_PID=$!
-echo "Server PID: $SERVER_PID"
-
-wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
-
-if [ "${EVAL_ONLY}" = "true" ]; then
- run_eval --port "$PORT"
-else
- build_replay_cmd "$RESULT_DIR"
- run_agentic_replay_and_write_outputs "$RESULT_DIR"
-fi
diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_mi355x_sglang.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_mi355x_sglang.sh
deleted file mode 100755
index 3e2a70d003..0000000000
--- a/benchmarks/single_node/agentic/qwen3.5_fp8_mi355x_sglang.sh
+++ /dev/null
@@ -1,139 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-set -x
-
-# Agentic trace replay benchmark for Qwen3.5 FP8 on MI355X using SGLang.
-#
-# Required env vars:
-# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
-#
-# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache.
-
-source "$(dirname "$0")/../../benchmark_lib.sh"
-
-check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE
-
-SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-30}
-
-if [[ -n "${SLURM_JOB_ID:-}" ]]; then
- echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}"
-fi
-
-# `hf download` creates the target dir if missing and is itself idempotent.
-# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE
-# Either way, MODEL_PATH is what the server is launched with.
-if [[ -n "${MODEL_PATH:-}" ]]; then
- if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then
- hf download "$MODEL" --local-dir "$MODEL_PATH"
- fi
-else
- hf download "$MODEL"
- export MODEL_PATH="$MODEL"
-fi
-rocm-smi || true
-amd-smi || true
-
-# ---- Resolve traces and install deps ----------------------------------------
-resolve_trace_source
-install_agentic_deps
-
-# ---- Server config ----------------------------------------------------------
-SERVER_LOG="$RESULT_DIR/server.log"
-mkdir -p "$RESULT_DIR"
-
-CACHE_ARGS=()
-WARMUP_ARGS=()
-CUDA_GRAPH_MAX_BS="$CONC"
-if require_agentic_kv_offload_backend hicache; then
- # Qwen3.5 allocates one KV and one Mamba host pool per TP rank.
- REQUESTED_HICACHE_TOTAL_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-$TOTAL_CPU_DRAM_GB}"
- if [ "$REQUESTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then
- echo "Error: requested HiCache pool ${REQUESTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2
- exit 1
- fi
- TOTAL_CPU_DRAM_GB="$REQUESTED_HICACHE_TOTAL_GB"
- HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-2}"
- HICACHE_MAX_SIZE_GB_PER_RANK_POOL="${HICACHE_MAX_SIZE_GB_PER_RANK_POOL:-${HICACHE_MAX_SIZE_GB_PER_RANK:-180}}"
- HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}"
- # Qwen3.5's hybrid Mamba path runs SGLang's no_buffer scheduler on MI355X,
- # which requires page_size=1. The kernel/page_first HiCache transfer path
- # faults on first prefill in this mode on ROCm, so keep the default on the
- # safer direct/layer_first copy path. These remain env-overridable.
- HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-1}"
- HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}"
- HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-layer_first}"
- # SGLang --hicache-size is per rank per host pool, while the workflow
- # input is a node-total DRAM budget. Divide by TP and the number of
- # host pools unless HICACHE_SIZE_GB is set directly for one-off tuning.
- MAX_HICACHE_SIZE_GB=$((TOTAL_CPU_DRAM_GB / TP / HICACHE_HOST_POOL_COUNT))
- HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}"
- if [ "$HICACHE_SIZE_GB" -gt "$MAX_HICACHE_SIZE_GB" ]; then
- echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB exceeds configured per-pool limit $MAX_HICACHE_SIZE_GB" >&2
- exit 1
- fi
- if [ "$HICACHE_SIZE_GB" -gt "$HICACHE_MAX_SIZE_GB_PER_RANK_POOL" ]; then
- HICACHE_SIZE_GB="$HICACHE_MAX_SIZE_GB_PER_RANK_POOL"
- fi
- if [ "$HICACHE_SIZE_GB" -lt 1 ]; then
- echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB from TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB, TP=$TP, HICACHE_HOST_POOL_COUNT=$HICACHE_HOST_POOL_COUNT" >&2
- exit 1
- fi
- echo "HiCache CPU pool: ${HICACHE_SIZE_GB} GB per rank per host pool across TP=${TP}, host_pool_count=${HICACHE_HOST_POOL_COUNT}"
- CACHE_ARGS=(
- --page-size "$HICACHE_PAGE_SIZE"
- --enable-hierarchical-cache
- --hicache-size "$HICACHE_SIZE_GB"
- --hicache-io-backend "$HICACHE_IO_BACKEND"
- --hicache-mem-layout "$HICACHE_MEM_LAYOUT"
- --hicache-write-policy "$HICACHE_WRITE_POLICY"
- )
- # HiCache startup reaches API readiness, but SGLang's internal warmup
- # request has timed out after 600s on this Qwen MI355X path. Let aiperf
- # own benchmark traffic instead of blocking server readiness on it.
- WARMUP_ARGS=(--skip-server-warmup)
- # Keep request concurrency as the swept variable, but do not force HiCache
- # runs to capture ROCm graphs at every high concurrency point.
- HICACHE_CUDA_GRAPH_MAX_BS="${HICACHE_CUDA_GRAPH_MAX_BS:-16}"
- if [ "$HICACHE_CUDA_GRAPH_MAX_BS" -lt "$CUDA_GRAPH_MAX_BS" ]; then
- CUDA_GRAPH_MAX_BS="$HICACHE_CUDA_GRAPH_MAX_BS"
- fi
-fi
-
-echo "Starting SGLang server..."
-export PYTHONNOUSERSITE=1
-
-{ set +x; } 2>/dev/null
-SGLANG_CMD=(
- python3 -m sglang.launch_server
- --attention-backend triton
- --model-path "$MODEL_PATH" --served-model-name "$MODEL"
- --host=0.0.0.0
- --port "$PORT"
- --tensor-parallel-size "$TP"
- --ep-size "$EP_SIZE"
- --trust-remote-code
- --tokenizer-worker-num 6
- --enable-aiter-allreduce-fusion
- --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS"
- --max-running-requests "$CONC"
- --max-prefill-tokens 32768
- --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL"
- --mem-fraction-static 0.8
- --enable-metrics
- "${CACHE_ARGS[@]}"
- "${WARMUP_ARGS[@]}"
-)
-printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt"
-printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt"
-"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 &
-SERVER_PID=$!
-echo "Server PID: $SERVER_PID"
-
-wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
-
-if [ "${EVAL_ONLY}" = "true" ]; then
- run_eval --port "$PORT"
-else
- build_replay_cmd "$RESULT_DIR"
- run_agentic_replay_and_write_outputs "$RESULT_DIR"
-fi
diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml
index 9bc224b161..3031fa0a2b 100644
--- a/configs/amd-master.yaml
+++ b/configs/amd-master.yaml
@@ -215,19 +215,6 @@ qwen3.5-fp8-mi355x-sglang-mtp:
search-space:
- { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp }
-qwen3.5-fp8-mi355x-sglang-agentic:
- image: lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260414
- model: Qwen/Qwen3.5-397B-A17B-FP8
- model-prefix: qwen3.5
- runner: cluster:mi355x-amds
- precision: fp8
- framework: sglang
- multinode: false
- scenarios:
- agentic-coding:
- - search-space:
- - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] }
-
qwen3.5-fp8-mi355x-atom:
image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511
model: Qwen/Qwen3.5-397B-A17B-FP8
@@ -1212,26 +1199,6 @@ qwen3.5-fp8-mi325x-sglang-mtp:
search-space:
- { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp }
-qwen3.5-fp8-mi355x-sglang-agentic-hicache:
- image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260521
- model: Qwen/Qwen3.5-397B-A17B-FP8
- model-prefix: qwen3.5
- runner: cluster:mi355x-amds
- precision: fp8
- framework: sglang
- multinode: false
- scenarios:
- agentic-coding:
- - dram-utilization: 0.80
- search-space:
- - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] }
- - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 32, 48, 64] }
-
-# MTP twin of dsv4-fp4-mi355x-vllm-agentic. The topologies, KV-offload backends,
-# and concurrency points are identical; speculative decoding is enabled on the
-# current immutable ROCm nightly because the base config's retired nightly tag
-# is no longer available from Docker Hub. Throughput uses the three-token golden
-# synthetic acceptance length, while eval-only runs retain real verification.
dsv4-fp4-mi355x-vllm-agentic-mtp:
image: vllm/vllm-openai-rocm:nightly-821717118fc26667dd474b9b0ab81d29259dfc5c
model: deepseek-ai/DeepSeek-V4-Pro
@@ -1433,30 +1400,6 @@ dsv4-fp4-mi355x-atom-disagg:
additional-settings:
- "DECODE_NODES=1"
# 1P1D TP8
-minimaxm3-fp8-mi300x-vllm-agentic:
- image: vllm/vllm-openai-rocm:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914
- model: MiniMaxAI/MiniMax-M3-MXFP8
- model-prefix: minimaxm3
- runner: cluster:mi300x-amds
- precision: fp8
- framework: vllm
- multinode: false
- scenarios:
- agentic-coding:
- # FP8 KV cache is 29,952 bytes/token/GPU for TP/TEP. Projecting the
- # measured BF16 cache budget gives about 2.76M active tokens on MI300X;
- # the June-21 service-time-weighted request is 269k tokens, so sample every
- # integer around the expected conc-10 cliff rather than geometric jumps.
- - dram-utilization: 0.80
- search-space:
- - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] }
- - { tp: 8, ep: 8, kv-offloading: none, conc-list: [2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] }
- - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] }
- - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] }
-
-# GLM-5.2 FP8 full-context AgentX refresh on MI325X. This preserves the TP8
-# GPU-resident-KV c1/c2/c3/c4/c5/c6/c8 curve from Actions run 29657732517
-# and enables EAGLE MTP with the committed thinking-on golden AL.
glm5.2-fp8-mi325x-sglang-agentic-mtp:
image: lmsysorg/sglang:v0.5.16-rocm720-mi30x
model: zai-org/GLM-5.2-FP8
@@ -1471,46 +1414,6 @@ glm5.2-fp8-mi325x-sglang-agentic-mtp:
search-space:
- { tp: 8, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 3, 4, 5, 6, 8] }
-minimaxm3-fp8-mi325x-vllm-agentic:
- image: vllm/vllm-openai-rocm:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914
- model: MiniMaxAI/MiniMax-M3-MXFP8
- model-prefix: minimaxm3
- runner: cluster:mi325x-amds
- precision: fp8
- framework: vllm
- multinode: false
- scenarios:
- agentic-coding:
- # Measured FP8 capacities: TP4 1.66M tokens, TP8 4.93M tokens, and DEP8
- # 12.69M aggregate tokens. With the June-21 service-time-weighted 269k
- # active request, their expected cliffs are conc 6, 18, and 47. Use dense
- # bands around each cliff; Mooncake rows overlap those bands because host
- # storage improves prefix retention but does not hold active decode KV.
- - dram-utilization: 0.80
- search-space:
- - { tp: 4, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] }
- - { tp: 4, ep: 4, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] }
- - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] }
- - { tp: 8, ep: 8, kv-offloading: none, conc-list: [2, 4, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] }
- - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80], router: { name: vllm-router, version: "0.1.14" } }
- - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] }
- - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] }
- - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [24, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 96], router: { name: vllm-router, version: "0.1.14" } }
-
-minimaxm3-fp4-mi355x-vllm-agentic:
- image: vllm/vllm-openai-rocm:nightly-dcfebf93f4eccf30f71872283331eee757915daf
- model: amd/MiniMax-M3-MXFP4
- model-prefix: minimaxm3
- runner: cluster:mi355x-amds
- precision: fp4
- framework: vllm
- multinode: false
- scenarios:
- agentic-coding:
- - dram-utilization: 0.80
- search-space:
- - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 4, 8, 16] }
-
dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp:
image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719
model: deepseek-ai/DeepSeek-V4-Pro
@@ -1633,4 +1536,3 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp:
search-space:
- { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 10, 12, 16], spec-decoding: mtp }
- { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4], spec-decoding: mtp }
-
diff --git a/configs/deprecated/amd-kimik2.5-agentic-master.yaml b/configs/deprecated/amd-kimik2.5-agentic-master.yaml
deleted file mode 100644
index 641bb41f1b..0000000000
--- a/configs/deprecated/amd-kimik2.5-agentic-master.yaml
+++ /dev/null
@@ -1,30 +0,0 @@
-# Deprecated Kimi-K2.5/2.6/2.7-Code agentic-coding entries archived from
-# amd-master.yaml. Deprecated after Monday, August 3, 2026 (see the
-# Deprecation Notice in MODELS.md). Single-turn 8k1k stays active until
-# August 6, 2026.
-# Removed from the active master config so sweep generation no longer selects them.
-
-
-kimik2.5-fp4-mi355x-vllm-agentic:
- image: vllm/vllm-openai-rocm:v0.22.0
- model: amd/Kimi-K2.5-MXFP4
- model-prefix: kimik2.5
- runner: cluster:mi355x-amds
- precision: fp4
- framework: vllm
- multinode: false
- scenarios:
- agentic-coding:
- - dram-utilization: 0.80
- search-space:
- - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48] }
- # DRAM offload only above the KV cliff. Lower concurrencies fit
- # entirely on-GPU, so paying the offload-path overhead there would
- # just slow them down without measuring anything new.
- - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-native }, conc-list: [32, 40, 48, 56] }
- # TP=4 probe: half-node layout doubles per-GPU weight footprint
- # (~62 GB on MI355X's 288 GB HBM, plenty of headroom). Restrict to
- # cliff-region concurrencies on both offload modes so we can directly
- # compare TP=4 vs TP=8 at the same conc points.
- - { tp: 4, kv-offloading: none, conc-list: [16, 24, 32, 40] }
- - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-native }, conc-list: [16, 24, 32, 40] }
diff --git a/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml b/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml
index a92bea3aa6..5a2e6d62e5 100644
--- a/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml
+++ b/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml
@@ -29,29 +29,6 @@ minimaxm2.5-fp8-mi355x-vllm:
# the original minimaxm2.5-fp8-mi355x-vllm entry is left identical to origin/main so
# its fixed-seq-len sweep is unaffected.
# - image: 'vllm/vllm-openai-rocm:v0.19.0' -> 'vllm/vllm-openai-rocm:nightly-51f22dcfd068fe8f1e3192da2a1e825b930223cf'
-minimaxm2.5-fp8-mi355x-vllm-agentic:
- # Nightly carrying vllm-project/vllm@20cac26b ("[Bug fix][KV Connector]
- # add cpu_offload_blocks > 0 check before maybe_run_layer_kv_offload"),
- # which enables SimpleCPUOffloadConnector on ROCm. Required for the
- # cpu-offload sweep points to use the same offload path as the NVIDIA
- # agentic-coding configs.
- image: vllm/vllm-openai-rocm:nightly-51f22dcfd068fe8f1e3192da2a1e825b930223cf
- model: MiniMaxAI/MiniMax-M2.5
- model-prefix: minimaxm2.5
- runner: mi355x
- precision: fp8
- framework: vllm
- multinode: false
- scenarios:
- agentic-coding:
- # MI355X tp=4 ep=4: compute ceiling ~60 (empirical), KV cliff ~91 (analytical).
- # Compute saturates first; cpu offload likely won't help, but worth confirming.
- # AMD uses native OffloadingConnector (NOT SimpleCPUOffloadConnector).
- - duration: 1800
- search-space:
- - { tp: 4, ep: 4, offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 48, 56, 64, 72, 96] }
- - { tp: 4, ep: 4, offloading: cpu, conc-list: [48, 56, 64, 72, 96] }
-
minimaxm2.5-fp8-mi355x-atom:
image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post
model: MiniMaxAI/MiniMax-M2.5
@@ -146,25 +123,6 @@ minimaxm2.5-fp8-mi300x-vllm:
# the original minimaxm2.5-fp8-mi300x-vllm entry is left identical to origin/main so
# its fixed-seq-len sweep is unaffected.
# - image: 'vllm/vllm-openai-rocm:v0.16.0' -> 'vllm/vllm-openai-rocm:nightly-51f22dcfd068fe8f1e3192da2a1e825b930223cf'
-minimaxm2.5-fp8-mi300x-vllm-agentic:
- # Nightly carrying vllm-project/vllm@20cac26b — see mi355x config above.
- image: vllm/vllm-openai-rocm:nightly-51f22dcfd068fe8f1e3192da2a1e825b930223cf
- model: MiniMaxAI/MiniMax-M2.5
- model-prefix: minimaxm2.5
- runner: mi300x
- precision: fp8
- framework: vllm
- multinode: false
- scenarios:
- agentic-coding:
- # MI300X tp=4: compute ceiling ~25 (estimated, between H100 and H200);
- # KV cliff ~52. Compute saturates first.
- # AMD uses native OffloadingConnector (NOT SimpleCPUOffloadConnector).
- - duration: 1800
- search-space:
- - { tp: 4, offloading: none, conc-list: [1, 2, 4, 8, 16, 20, 24, 28, 32, 40, 48] }
- - { tp: 4, offloading: cpu, conc-list: [16, 20, 24, 28, 32] }
-
minimaxm2.5-fp8-mi325x-vllm:
image: vllm/vllm-openai-rocm:v0.22.0
model: MiniMaxAI/MiniMax-M2.5
@@ -190,26 +148,6 @@ minimaxm2.5-fp8-mi325x-vllm:
# the original minimaxm2.5-fp8-mi325x-vllm entry is left identical to origin/main so
# its fixed-seq-len sweep is unaffected.
# - image: 'vllm/vllm-openai-rocm:v0.18.0' -> 'vllm/vllm-openai-rocm:nightly-51f22dcfd068fe8f1e3192da2a1e825b930223cf'
-minimaxm2.5-fp8-mi325x-vllm-agentic:
- # Nightly carrying vllm-project/vllm@20cac26b — see mi355x config above.
- image: vllm/vllm-openai-rocm:nightly-51f22dcfd068fe8f1e3192da2a1e825b930223cf
- model: MiniMaxAI/MiniMax-M2.5
- model-prefix: minimaxm2.5
- runner: mi325x
- precision: fp8
- framework: vllm
- multinode: false
- scenarios:
- agentic-coding:
- # MI325X tp=4: cloned from MI300X recipe (slightly faster compute,
- # similar HBM profile). Compute saturates first; cpu-offload window
- # exercises the SimpleCPUOffloadConnector path enabled by the rocm
- # nightly. Mirror MI300X conc grid for cross-vendor comparability.
- - duration: 1800
- search-space:
- - { tp: 4, offloading: none, conc-list: [1, 2, 4, 8, 16, 20, 24, 28, 32, 40, 48] }
- - { tp: 4, offloading: cpu, conc-list: [16, 20, 24, 28, 32] }
-
minimaxm2.5-fp8-mi355x-vllm-disagg:
image: vllm/vllm-openai-rocm:nightly-a6682d1d259cca69a9ae737ea5608fbbe7520031
model: MiniMaxAI/MiniMax-M2.5
diff --git a/configs/deprecated/nvidia-kimik2.5-agentic-master.yaml b/configs/deprecated/nvidia-kimik2.5-agentic-master.yaml
deleted file mode 100644
index 4be8befaa5..0000000000
--- a/configs/deprecated/nvidia-kimik2.5-agentic-master.yaml
+++ /dev/null
@@ -1,111 +0,0 @@
-# Deprecated Kimi-K2.5/2.6/2.7-Code agentic-coding entries archived from
-# nvidia-master.yaml. Deprecated after Monday, August 3, 2026 (see the
-# Deprecation Notice in MODELS.md). Single-turn 8k1k stays active until
-# August 6, 2026.
-# Removed from the active master config so sweep generation no longer selects them.
-
-
-kimik2.5-int4-b200-vllm-agentic:
- image: vllm/vllm-openai:v0.22.0
- model: moonshotai/Kimi-K2.5
- model-prefix: kimik2.5
- runner: cluster:b200-dgxc
- precision: int4
- framework: vllm
- multinode: false
- scenarios:
- agentic-coding:
- - dram-utilization: 0.80
- search-space:
- - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] }
- - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [32, 64, 96, 128] }
-
-kimik2.5-int4-h200-vllm-agentic:
- image: vllm/vllm-openai:v0.22.0
- model: moonshotai/Kimi-K2.5
- model-prefix: kimik2.5
- runner: cluster:h200-dgxc
- precision: int4
- framework: vllm
- multinode: false
- scenarios:
- agentic-coding:
- - dram-utilization: 0.80
- search-space:
- - { tp: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7] }
- - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [6, 7, 8, 9, 10, 11, 12, 13, 14] }
-
-kimik2.5-fp4-b300-vllm-agentic:
- # v0.20.2 (cu129) lacks the flashinfer kernels for B300's reported SM
- # (sm_12x); workers hit "Only SM 10.x and 11.x are supported" in the
- # trtllm_fp4_block_scale_moe path. v0.20.0-cu130 is the Blackwell-targeted
- # build that has the full sm_10x/sm_11x/sm_12x kernel set and is what the
- # INT4 B300 sister already uses successfully.
- image: vllm/vllm-openai:v0.22.0
- model: nvidia/Kimi-K2.5-NVFP4
- model-prefix: kimik2.5
- runner: cluster:b300-nv
- precision: fp4
- framework: vllm
- multinode: false
- scenarios:
- agentic-coding:
- - dram-utilization: 0.80
- search-space:
- - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 40, 48, 56, 64] }
- - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2, 4, 8, 16, 32, 40, 48, 56, 64] }
-
-kimik2.5-fp4-b300-vllm-agentic-mtp:
- image: vllm/vllm-openai:nightly-94c0ef300180f8fd1071d9cbe7270a8348155f94
- model: Kimi-K2.6-NVFP4
- model-prefix: kimik2.5
- runner: cluster:b300-nv
- precision: fp4
- framework: vllm
- multinode: false
- scenarios:
- agentic-coding:
- - dram-utilization: 0.80
- search-space:
- - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1] }
- - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [2, 4, 8] }
- - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [8, 16, 32] }
- - { tp: 4, ep: 1, spec-decoding: mtp, dcp-size: 4, kv-offloading: none, conc-list: [32, 64, 80, 96, 112, 128] }
- - { tp: 4, ep: 1, spec-decoding: mtp, dcp-size: 4, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [64, 80, 96, 112, 128, 144, 160] }
-
-kimik2.5-fp4-b200-vllm-agentic-lmcache:
- image: vllm/vllm-openai:v0.22.0
- model: nvidia/Kimi-K2.5-NVFP4
- model-prefix: kimik2.5
- runner: cluster:b200-dgxc
- precision: fp4
- framework: vllm
- multinode: false
- scenarios:
- agentic-coding:
- - dram-utilization: 0.80
- search-space:
- - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 24] }
- - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.1" }, conc-list: [16, 24, 32, 36] }
- - { tp: 4, ep: 1, kv-offloading: none, conc-list: [8, 12, 14, 16, 18, 20] }
- - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.1" }, conc-list: [12, 14, 16, 18, 20, 22, 24, 32] }
-
-# Left behind by #2493: this key stayed in nvidia-master.yaml while its script
-# moved to benchmarks/single_node/agentic/deprecated/, so the config pointed at
-# a path that no longer existed. Archived here with its siblings.
-kimik2.5-int4-h100-vllm:
- image: vllm/vllm-openai:v0.22.0
- model: moonshotai/Kimi-K2.5
- model-prefix: kimik2.5
- runner: cluster:h100-dgxc
- precision: int4
- framework: vllm
- multinode: false
- scenarios:
- # H100 has 80 GB HBM per GPU (smallest in this set); the KV cliff arrives
- # early. Sweep saturates conc=20 to keep total HBM headroom.
- agentic-coding:
- - dram-utilization: 0.80
- search-space:
- - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 16, 20] }
- - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2, 4, 8, 12, 16, 20] }
diff --git a/configs/deprecated/nvidia-minimaxm2.5-m2.7-master.yaml b/configs/deprecated/nvidia-minimaxm2.5-m2.7-master.yaml
index 50be2d4e8d..cbeef276f7 100644
--- a/configs/deprecated/nvidia-minimaxm2.5-m2.7-master.yaml
+++ b/configs/deprecated/nvidia-minimaxm2.5-m2.7-master.yaml
@@ -32,28 +32,6 @@ minimaxm2.5-fp8-b200-vllm:
# its fixed-seq-len sweep is unaffected.
# - image: 'vllm/vllm-openai:v0.19.0-cu130' -> 'vllm/vllm-openai:v0.19.1'
# - runner: 'b200' -> 'b200-dgxc'
-minimaxm2.5-fp8-b200-vllm-agentic:
- image: vllm/vllm-openai:v0.19.1
- model: MiniMaxAI/MiniMax-M2.5
- model-prefix: minimaxm2.5
- runner: b200-dgxc
- precision: fp8
- framework: vllm
- multinode: false
- scenarios:
- agentic-coding:
- # B200 tp=4: compute ceiling ~50 (empirical), KV cliff ~48 (analytical).
- # Push none past the KV cliff (96, 128) to make the no-offload throughput
- # collapse visible; cpu range overlaps fully for same-conc comparison.
- - duration: 1800
- search-space:
- - { tp: 4, offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 48, 56, 64, 96, 128] }
- - { tp: 4, offloading: cpu, conc-list: [48, 56, 64, 96, 128] }
-
- # NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/MiniMax/MiniMax-M2.html
- # does not have a B300-specific recipe, so this config reuses the existing
- # MiniMax-M2.5 FP8 B200 vLLM recipe as-is until B300-specific tuning is available.
-
minimaxm2.5-fp8-b300-vllm:
image: vllm/vllm-openai:v0.21.0
model: MiniMaxAI/MiniMax-M2.5
@@ -82,27 +60,6 @@ minimaxm2.5-fp8-b300-vllm:
# the original minimaxm2.5-fp8-b300-vllm entry is left identical to origin/main so
# its fixed-seq-len sweep is unaffected.
# - image: 'vllm/vllm-openai:v0.19.0-cu130' -> 'vllm/vllm-openai:v0.19.1'
-minimaxm2.5-fp8-b300-vllm-agentic:
- image: vllm/vllm-openai:v0.19.1
- model: MiniMaxAI/MiniMax-M2.5
- model-prefix: minimaxm2.5
- runner: b300
- precision: fp8
- framework: vllm
- multinode: false
- scenarios:
- agentic-coding:
- # B300 tp=4: compute ceiling ~60 (empirical), KV cliff ~85 (analytical).
- # Push none past the KV cliff (96, 128, 192) so the no-offload throughput
- # collapse is visible; cpu range overlaps fully so each high-conc point
- # has a same-conc no-offload counterpart for direct comparison.
- # Dense sampling between 96 and 128 (step=4) to resolve the sharp dropoff
- # observed in v6 cpu data right past conc=96.
- - duration: 1800
- search-space:
- - { tp: 4, offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 48, 64, 96, 100, 104, 108, 112, 116, 120, 124, 128, 192] }
- - { tp: 4, offloading: cpu, conc-list: [48, 64, 96, 100, 104, 108, 112, 116, 120, 124, 128, 192] }
-
minimaxm2.5-fp4-b200-vllm:
image: vllm/vllm-openai:v0.22.0
model: nvidia/MiniMax-M2.5-NVFP4
@@ -140,24 +97,6 @@ minimaxm2.5-fp4-b200-vllm:
# PR adds an agentic-coding scenarios block that differs from main
# (either main had none or had a different conc/offload sweep).
# The original minimaxm2.5-fp4-b200-vllm entry stays byte-identical to origin/main.
-minimaxm2.5-fp4-b200-vllm-agentic:
- image: vllm/vllm-openai:v0.19.0-cu130
- model: nvidia/MiniMax-M2.5-NVFP4
- model-prefix: minimaxm2.5
- runner: b200
- precision: fp4
- framework: vllm
- multinode: false
- scenarios:
- agentic-coding:
- - duration: 1800
- search-space:
- - { tp: 4, offloading: none, conc-list: [1, 2, 4, 8, 16, 32] }
-
- # NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/MiniMax/MiniMax-M2.html
- # does not have a B300-specific recipe, so this config reuses the existing
- # MiniMax-M2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available.
-
minimaxm2.5-fp4-b200-trt:
image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc18
model: nvidia/MiniMax-M2.5-NVFP4
@@ -265,24 +204,6 @@ minimaxm2.5-fp8-h100-vllm:
# PR adds an agentic-coding scenarios block that differs from main
# (either main had none or had a different conc/offload sweep).
# The original minimaxm2.5-fp8-h100-vllm entry stays byte-identical to origin/main.
-minimaxm2.5-fp8-h100-vllm-agentic:
- image: vllm/vllm-openai:v0.20.2
- model: MiniMaxAI/MiniMax-M2.5
- model-prefix: minimaxm2.5
- runner: h100
- precision: fp8
- framework: vllm
- multinode: false
- scenarios:
- agentic-coding:
- # H100 tp=4 ep=4: compute ceiling ~10 (empirical), KV cliff ~6 (analytical).
- # Best cpu-offload demo SKU — 4-conc-point window between cliffs.
- # Dense sampling 4-12 covers both cliffs; conc 16 confirms compute plateau.
- - duration: 1800
- search-space:
- - { tp: 4, ep: 4, offloading: none, conc-list: [1, 2, 4, 5, 6, 7, 8, 10, 12, 16] }
- - { tp: 4, ep: 4, offloading: cpu, conc-list: [5, 6, 7, 8, 10, 12] }
-
minimaxm2.5-fp8-h200-vllm:
image: vllm/vllm-openai:v0.22.0
model: MiniMaxAI/MiniMax-M2.5
@@ -307,23 +228,6 @@ minimaxm2.5-fp8-h200-vllm:
# PR adds an agentic-coding scenarios block that differs from main
# (either main had none or had a different conc/offload sweep).
# The original minimaxm2.5-fp8-h200-vllm entry stays byte-identical to origin/main.
-minimaxm2.5-fp8-h200-vllm-agentic:
- image: vllm/vllm-openai:v0.20.2
- model: MiniMaxAI/MiniMax-M2.5
- model-prefix: minimaxm2.5
- runner: h200
- precision: fp8
- framework: vllm
- multinode: false
- scenarios:
- agentic-coding:
- # H200 tp=4: compute ceiling ~35 (empirical), KV cliff ~29 (analytical).
- # cpu offload window conc 29-35 — dense sampling 24-40 captures both cliffs.
- - duration: 1800
- search-space:
- - { tp: 4, offloading: none, conc-list: [1, 2, 4, 8, 16, 24, 28, 32, 36, 48] }
- - { tp: 4, offloading: cpu, conc-list: [24, 28, 32, 36, 40, 48] }
-
minimaxm2.5-fp4-gb300-dynamo-vllm:
image: vllm/vllm-openai:v0.20.1
model: nvidia/MiniMax-M2.5-NVFP4
diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml
index f916be18dc..da5df2ab47 100644
--- a/configs/nvidia-master.yaml
+++ b/configs/nvidia-master.yaml
@@ -1185,19 +1185,6 @@ qwen3.5-fp8-b200-sglang:
- { tp: 8, conc-start: 4, conc-end: 4 }
- { tp: 4, ep: 1, conc-start: 4, conc-end: 256 }
-qwen3.5-fp8-b200-sglang-agentic:
- image: lmsysorg/sglang:nightly-dev-20260422-de962f32
- model: Qwen/Qwen3.5-397B-A17B-FP8
- model-prefix: qwen3.5
- runner: cluster:b200-dgxc
- precision: fp8
- framework: sglang
- multinode: false
- scenarios:
- agentic-coding:
- - search-space:
- - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] }
-
qwen3.5-fp8-b200-sglang-agentic-mtp:
image: lmsysorg/sglang:v0.5.16-cu130
model: Qwen/Qwen3.5-397B-A17B-FP8
@@ -1472,37 +1459,6 @@ dsr1-fp8-b300-sglang-mtp:
search-space:
- { tp: 8, ep: 1, conc-start: 1, conc-end: 512, spec-decoding: mtp }
-kimik3-fp4-b300-vllm-agentic:
- # Day-zero Kimi-K3 recipe. `vllm/vllm-openai:kimi-k3` is the pre-release
- # Kimi-K3 build (vLLM 0.1.dev19262+gb6bbf29dd, pushed 2026-07-27); K3 ships as
- # the out-of-tree `vllm.models.kimi_k3` plugin package (DSpark MLA + KDA
- # kernels), so a generic vllm-openai release cannot serve this checkpoint.
- image: vllm/vllm-openai:kimi-k3
- model: moonshotai/Kimi-K3
- model-prefix: kimik3
- runner: cluster:b300-nv
- precision: fp4
- framework: vllm
- multinode: false
- scenarios:
- # Agentic-coding only: no fixed-seq-len (1k1k / 8k1k) arms for this recipe.
- agentic-coding:
- # 0.63 resolves to ~220 GiB of host DRAM per rank across the 8 TP ranks on
- # cluster:b300-nv (total-cpu-dram-gb 1889), which is what the
- # SimpleCPUOffloadConnector pool is sized to in the script.
- - dram-utilization: 0.63
- search-space:
- # TP8 is the only single-node layout that fits: the MXFP4 checkpoint is
- # ~1.5 TB (~188 GB/GPU across 8 B300s), and TP4 would need ~375 GB/GPU
- # against 288 GB of HBM. The conc ceiling is set by the ~70 GB/GPU left
- # for MLA KV after weights at gpu-memory-utilization 0.90.
- # Both arms share one ladder so GPU-resident and DRAM-offload are directly
- # comparable at equal concurrency (same shape as the kimik2.5 B300 sister).
- # TP8 GPU-resident
- - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 24] }
- # TP8 SimpleCPUOffload (host DRAM)
- - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2, 4, 8, 16, 24] }
-
kimik3-fp4-b300-vllm-agentic-dspark:
# DSpark speculative decoding on the Inferact/Kimi-K3-DSpark draft head, at
# level 2 with probabilistic drafting and synthetic acceptance pinned to the
@@ -1651,22 +1607,6 @@ dsv4-fp8-h200-vllm-mtp:
# Pinned to the h200-dgxc-slurm runner pool because the deepseek-v4-hopper
# image needs the /ix mount layout that only launch_h200-dgxc-slurm.sh sets up.
-dsv4-fp8-h200-vllm-agentic:
- image: vllm/vllm-openai:v0.22.0
- model: deepseek-ai/DeepSeek-V4-Pro
- model-prefix: dsv4
- runner: cluster:h200-dgxc
- precision: fp8
- framework: vllm
- multinode: false
- scenarios:
- agentic-coding:
- - search-space:
- - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [1, 2, 4, 8, 16] }
-
-# MTP variant of dsv4-fp8-h200-sglang. Mirrors the non-MTP recipe (same image,
-# runner pool, search space) and adds EAGLE speculative decoding via
-# --speculative-algorithm EAGLE with the (3,1,4) chain matching dsv4-fp4-b300-sglang-mtp.
dsv4-fp8-h200-sglang:
image: lmsysorg/sglang:deepseek-v4-hopper@sha256:1bf5d508ab110cc0fe1659a5f21d1be02a7f0d7ca8f58cea7e7f4e11f6ae208f
model: deepseek-ai/DeepSeek-V4-Pro
@@ -6863,21 +6803,6 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg:
dp-attn: false
# ---------- 1k1k high-throughput (wide-EP decode, EAGLE MTP) ----------
-qwen3.5-fp8-b300-sglang-agentic-hicache:
- image: lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd
- model: Qwen/Qwen3.5-397B-A17B-FP8
- model-prefix: qwen3.5
- runner: cluster:b300-nv
- precision: fp8
- framework: sglang
- multinode: false
- scenarios:
- agentic-coding:
- - dram-utilization: 0.80
- search-space:
- - { tp: 4, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] }
- - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 32, 48, 64] }
-
qwen3.5-fp8-b300-sglang-agentic-mtp:
image: lmsysorg/sglang:v0.5.16-cu130
model: Qwen/Qwen3.5-397B-A17B-FP8
@@ -6910,111 +6835,6 @@ qwen3.5-fp4-b300-sglang-agentic-mtp:
- { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32] }
- { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 34, 36, 38, 40, 44, 48, 52, 56] }
-# CONC range conservative for H100's 80 GB HBM3 under the long-ISL with-
-# subagents corpus. hicache arm capped at conc 16 since high-conc + hicache
-# tends to flake on first runs and conc 16 covers the cliff. The bench script
-# sets WEKA_LOADER_OVERRIDE to the 256k-capped corpus variant.
-dsv4-fp4-gb300-dynamo-vllm-agentic:
- image: vllm/vllm-openai:v0.21.0-ubuntu2404
- model: deepseek-ai/DeepSeek-V4-Pro
- model-prefix: dsv4
- runner: cluster:gb300-nv
- precision: fp4
- framework: dynamo-vllm
- router: { name: dynamo-router, version: "1.2.0.dev20260426" }
- kv-p2p-transfer: nixl
- multinode: true
- disagg: true
- scenarios:
- agentic-coding:
- - search-space:
- # Low-latency: same 1p6d shape as the mid tier but at much lower conc
- # (32 vs 192). 32/6 ≈ 5 seqs per decode worker — well below saturation,
- # so each request gets ~6× the per-request decode compute it would get
- # at conc=192. Reuses the 1p6d recipe; no separate recipe file needed.
- - spec-decoding: none
- conc-list: [32]
- prefill:
- num-worker: 1
- tp: 4
- ep: 4
- dp-attn: true
- additional-settings:
- - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tp4-agentic.yaml"
- decode:
- num-worker: 6
- tp: 4
- ep: 1
- dp-attn: false
- # Mid: 1 prefill (DEP=4) + 6 decode (TP=4). 7 nodes / 28 GPUs.
- # Mirrors fixed-seq-len conc=192 entry.
- - spec-decoding: none
- conc-list: [192]
- prefill:
- num-worker: 1
- tp: 4
- ep: 4
- dp-attn: true
- additional-settings:
- - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tp4-agentic.yaml"
- decode:
- num-worker: 6
- tp: 4
- ep: 1
- dp-attn: false
- # High-throughput: 4 prefill (DEP=4 each) + 1 decode (DEP=8). 6 nodes /
- # 24 GPUs. Smallest 4096-class shape in fixed-seq-len; deep_gemm_mega_moe
- # on both sides. Mirrors fixed-seq-len conc=4096 entry (4p1d variant).
- - spec-decoding: none
- conc-list: [4096]
- prefill:
- num-worker: 4
- tp: 4
- ep: 4
- dp-attn: true
- additional-settings:
- - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-4p1d-dep4-dep8-24-c4096-agentic.yaml"
- decode:
- num-worker: 1
- tp: 8
- ep: 8
- dp-attn: true
-
-qwen3.5-fp8-h100-sglang-agentic:
- image: lmsysorg/sglang:v0.5.12-cu130
- model: Qwen/Qwen3.5-397B-A17B-FP8
- model-prefix: qwen3.5
- runner: cluster:h100-dgxc
- precision: fp8
- framework: sglang
- multinode: false
- scenarios:
- agentic-coding:
- - dram-utilization: 0.80
- search-space:
- - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 14, 16] }
- - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [12, 14, 16, 20, 24, 28, 32, 42] }
-
-# MTP speculative-decoding (spec-decoding: mtp) variant of
-# qwen3.5-fp8-h100-sglang-agentic: same TP8/EP8 GPU-resident and HiCache arms,
-# plus SGLang EAGLE MTP (num-steps 3, eagle-topk 1, 4 draft tokens = 3
-# speculative tokens) with simulated acceptance pinned to the golden AL 3.39
-# (golden_al_distribution/qwen3.5_mtp.yaml, thinking_on, K=3) -- the same value
-# the GB300 Qwen3.5 AgentX srt-slurm recipes use. Image moves to
-# lmsysorg/sglang:v0.5.16-cu130 because SGLANG_SIMULATE_ACC_TOKEN_MODE only
-# exists from v0.5.16; v0.5.14 is the version the fixed-seq-len H100 MTP entry
-# already runs. Conc lists are trimmed at the top end for the draft head's KV.
-
-# First Qwen3.5 AgentX recipe on H200, shipped spec-decode-only per the AgentX
-# policy that new agentic arms enable speculative decoding rather than running a
-# separate STP baseline (MODELS.md). SGLang EAGLE MTP (num-steps 3, eagle-topk 1,
-# 4 draft tokens = 3 speculative tokens) with simulated acceptance pinned to the
-# golden AL 3.39 (golden_al_distribution/qwen3.5_mtp.yaml, thinking_on, K=3).
-# Image is lmsysorg/sglang:v0.5.16-cu130: SGLANG_SIMULATE_ACC_TOKEN_MODE only
-# exists from v0.5.16, and v0.5.14 is what the fixed-seq-len H200 MTP entry runs.
-# Search space is the H100 AgentX shape widened for H200's 141 GB: the
-# fixed-seq-len H200 MTP entry already runs conc 4-128 at TP8/EP8 versus H100's
-# 4-32, so the GPU-resident arm extends to 24 and HiCache to 48.
qwen3.5-fp8-h200-sglang-agentic-mtp:
image: lmsysorg/sglang:v0.5.16-cu130
model: Qwen/Qwen3.5-397B-A17B-FP8
@@ -7086,48 +6906,6 @@ qwen3.5-fp4-b200-trt-mtp:
- { tp: 8, ep: 8, spec-decoding: "mtp", conc-list: [4] }
- { tp: 8, ep: 8, dp-attn: true, spec-decoding: "mtp", conc-list: [128, 256, 1024] }
-minimaxm3-fp8-h100-vllm-agentic:
- image: vllm/vllm-openai:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914
- model: MiniMaxAI/MiniMax-M3-MXFP8
- model-prefix: minimaxm3
- runner: cluster:h100-dgxc
- precision: fp8
- framework: vllm
- multinode: false
- scenarios:
- agentic-coding:
- # FP8 KV cache is 29,952 bytes/token/GPU for TP/TEP: 60 layers of
- # sharded FP8 K/V plus 57 layers of BF16 indexer cache. The June-21 trace
- # has a 269k-token service-time-weighted active request, placing the H100
- # 1.64M-token HBM cliff near conc 6. Sample every integer through the
- # cliff; Mooncake extends prefix retention, not active-request HBM.
- - dram-utilization: 0.80
- search-space:
- - { tp: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] }
- - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] }
- - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] }
- - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] }
-
-minimaxm3-fp8-h200-vllm-agentic:
- image: vllm/vllm-openai:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914
- model: MiniMaxAI/MiniMax-M3-MXFP8
- model-prefix: minimaxm3
- runner: cluster:h200-dgxc
- precision: fp8
- framework: vllm
- multinode: false
- scenarios:
- agentic-coding:
- # The same 29,952-byte FP8 TP/TEP cache layout gives H200 about 2.56M
- # active tokens. Against the June-21 service-time-weighted 269k-token
- # request, the expected HBM cliff is near conc 10; sample densely from
- # 5-14 and retain post-cliff points through 20.
- - dram-utilization: 0.80
- search-space:
- - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] }
- - { tp: 8, ep: 8, kv-offloading: none, conc-list: [2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] }
- - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] }
-
qwen3.5-fp4-b200-sglang-agentic-mtp:
image: lmsysorg/sglang:v0.5.16-cu130
model: nvidia/Qwen3.5-397B-A17B-NVFP4
@@ -7216,221 +6994,6 @@ minimaxm3-fp4-b200-vllm-agentic-mtp:
search-space:
- { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 5, 8, 10, 12, 15, 20] }
- { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [20, 30, 40] }
-dsv4-fp4-gb200-dynamo-vllm-agentic-3p2d-tep8-tp8:
- image: vllm/vllm-openai:v0.23.0
- model: deepseek-ai/DeepSeek-V4-Pro
- model-prefix: dsv4
- runner: cluster:gb200-nv
- precision: fp4
- framework: dynamo-vllm
- router: { name: dynamo-router, version: "1.3.0.dev20260618" }
- kv-p2p-transfer: nixl
- multinode: true
- disagg: true
- scenarios:
- agentic-coding:
- - search-space:
- # Ultra-high-interactivity probes below the historical c16 endpoint.
- - spec-decoding: none
- conc-list: [4, 8]
- prefill:
- num-worker: 3
- tp: 8
- ep: 8
- dp-attn: false
- additional-settings:
- - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml"
- decode:
- num-worker: 2
- tp: 8
- ep: 1
- dp-attn: false
- - spec-decoding: none
- conc-list: [16, 24, 32, 40]
- prefill:
- num-worker: 3
- tp: 8
- ep: 8
- dp-attn: false
- additional-settings:
- - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml"
- decode:
- num-worker: 2
- tp: 8
- ep: 1
- dp-attn: false
- - spec-decoding: none
- conc-list: [48, 56, 64, 80]
- prefill:
- num-worker: 3
- tp: 8
- ep: 8
- dp-attn: false
- additional-settings:
- - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml"
- decode:
- num-worker: 2
- tp: 8
- ep: 1
- dp-attn: false
-
-dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep8:
- image: vllm/vllm-openai:v0.23.0
- model: deepseek-ai/DeepSeek-V4-Pro
- model-prefix: dsv4
- runner: cluster:gb200-nv
- precision: fp4
- framework: dynamo-vllm
- router: { name: dynamo-router, version: "1.3.0.dev20260618" }
- kv-p2p-transfer: nixl
- multinode: true
- disagg: true
- scenarios:
- agentic-coding:
- - search-space:
- - spec-decoding: none
- conc-list: [32, 48, 64, 80]
- prefill:
- num-worker: 2
- tp: 8
- ep: 8
- dp-attn: true
- additional-settings:
- - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml"
- decode:
- num-worker: 1
- tp: 8
- ep: 8
- dp-attn: true
- - spec-decoding: none
- conc-list: [96, 128, 160]
- prefill:
- num-worker: 2
- tp: 8
- ep: 8
- dp-attn: true
- additional-settings:
- - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml"
- decode:
- num-worker: 1
- tp: 8
- ep: 8
- dp-attn: true
- # Exploratory tail beyond the measured c160 normalized-throughput peak.
- - spec-decoding: none
- conc-list: [192, 224, 256]
- prefill:
- num-worker: 2
- tp: 8
- ep: 8
- dp-attn: true
- additional-settings:
- - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml"
- decode:
- num-worker: 1
- tp: 8
- ep: 8
- dp-attn: true
-
-dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg:
- image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f
- model: deepseek-ai/DeepSeek-V4-Pro
- model-prefix: dsv4
- runner: cluster:gb300-nv
- precision: fp4
- framework: dynamo-vllm
- router: { name: dynamo-router, version: "1.2.1" }
- multinode: true
- disagg: false
- scenarios:
- agentic-coding:
- - search-space:
- - spec-decoding: mtp
- conc-list: [1]
- prefill:
- num-worker: 1
- tp: 8
- ep: 1
- dp-attn: false
- additional-settings:
- - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml"
- # The aggregate worker also performs decode; keep the decode worker
- # count at zero so result aggregation counts eight GPUs only once.
- decode:
- num-worker: 0
- tp: 8
- ep: 1
- dp-attn: false
- - spec-decoding: mtp
- conc-list: [4]
- prefill:
- num-worker: 1
- tp: 4
- ep: 1
- dp-attn: false
- additional-settings:
- - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml"
- # The aggregate worker also performs decode; keep the decode worker
- # count at zero so result aggregation counts four GPUs only once.
- decode:
- num-worker: 0
- tp: 4
- ep: 1
- dp-attn: false
-
-dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg:
- image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f
- model: deepseek-ai/DeepSeek-V4-Pro
- model-prefix: dsv4
- runner: cluster:gb300-nv
- precision: fp4
- framework: dynamo-vllm
- kv-p2p-transfer: nixl
- multinode: true
- disagg: true
- scenarios:
- agentic-coding:
- - search-space:
- - spec-decoding: mtp
- conc-list: [128]
- router: { name: dynamo-router, version: "1.3.0.dev20260720" }
- prefill:
- num-worker: 1
- tp: 4
- ep: 4
- dp-attn: true
- additional-settings:
- - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml"
- decode:
- num-worker: 1
- tp: 8
- ep: 8
- dp-attn: true
- - spec-decoding: mtp
- conc-list: [384]
- router: { name: dynamo-router, version: "1.3.0.dev20260720" }
- prefill:
- num-worker: 1
- tp: 8
- ep: 8
- dp-attn: true
- additional-settings:
- - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml"
- decode:
- num-worker: 1
- tp: 8
- ep: 8
- dp-attn: true
-
-# Day-0 Kimi K3 AgentX coverage on GB200. The three aggregate srt-slurm
-# profiles mirror the serving strategies synthesized by the official recipe:
-# latency: multi-node TP16 (4 GB200 nodes / 16 GPUs)
-# balanced: multi-node TEP16 (4 GB200 nodes / 16 GPUs)
-# throughput: multi-node TP4 x DP4 (4 GB200 nodes / EP16)
-# All profiles enable Kimi K3 DSpark level 2 at the committed golden AL 2.51.
-# `spec-decoding: mtp` is the existing matrix label used for speculative
-# decoding methods; the checked-in recipes select DSpark explicitly.
-# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200
kimik3-fp4-gb200-dynamo-vllm-agentic:
image: vllm/vllm-openai:kimi-k3
model: moonshotai/Kimi-K3