diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml deleted file mode 100644 index 192e0f5b95..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml +++ /dev/null @@ -1,146 +0,0 @@ -name: "svf-vllm-agg-gb300-tp4-mtp-agentic" - -# GB300 AgentX aggregate topology: one TP4 worker occupies one four-GPU node -# and serves both prefill and decode at concurrency 4. Scheduler, CUDA-graph, -# and memory settings match the B300 vLLM TP4 MTP agentic configuration. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.2.1" - -dynamo: - wheel: "1.2.1" - install: true - -environment: - DYNAMO_WHEEL_DIRS: "/srtctl-wheels" - ETCD_LEASE_TTL: "7200" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 1 - agg_workers: 1 - gpus_per_agg: 4 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-reset-states: true - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -backend: - type: vllm - connector: null - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "16" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-tp4-mtp-{job_id}" - vllm_config: - aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - disable-custom-all-reduce: true - enable-cumem-allocator: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 16 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","cudagraph_capture_sizes":[4,8,12,16,20,24,28,32,36,40,44,48,52,56,60,64],"mode":0}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' - gpu-memory-utilization: 0.93 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml deleted file mode 100644 index 6a950787fe..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml +++ /dev/null @@ -1,149 +0,0 @@ -name: "svf-vllm-agg-gb300-tp8-mtp-agentic" - -# Validated GB300 AgentX aggregate topology: one TP8 worker spans two -# four-GPU nodes and serves both prefill and decode at concurrency 1. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.2.1" - -dynamo: - wheel: "1.2.1" - install: true - -environment: - DYNAMO_WHEEL_DIRS: "/srtctl-wheels" - # The frontend shares Grace CPU capacity with the long TP8 cold start. - ETCD_LEASE_TTL: "7200" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - agg_nodes: 2 - agg_workers: 1 - gpus_per_agg: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "kv" - router-reset-states: true - router-temperature: 0.0 - router-queue-threshold: 65536 - active-decode-blocks-threshold: "None" - active-prefill-tokens-threshold: "None" - active-prefill-tokens-threshold-frac: "None" - tokenizer: "fastokens" - -backend: - type: vllm - connector: null - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - aggregated_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "32" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "0" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_USE_RUST_FRONTEND: "1" - VLLM_ALLREDUCE_USE_FLASHINFER: "1" - VLLM_FLASHINFER_ALLREDUCE_BACKEND: "auto" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - NCCL_P2P_LEVEL: "NVL" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-agg-mtp-{job_id}" - vllm_config: - aggregated: - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - disable-custom-all-reduce: true - enable-cumem-allocator: true - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - max-model-len: 1048576 - max-num-seqs: 32 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 128 - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' - gpu-memory-utilization: 0.90 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - -sbatch_directives: - cpus-per-task: "144" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - # Keep aggregate workers in the multinode result schema so ingestion uses - # the zero decode-worker count instead of duplicating TP into P and D. - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml deleted file mode 100644 index b647276179..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml +++ /dev/null @@ -1,161 +0,0 @@ -name: "svf-vllm-disagg-gb200-2p1d-dep8-dep8-agentic" - -# High-throughput topology: two DEP8 prefill workers and one -# DEP8 decode worker. All three workers use attention DP8 / expert EP8, for -# 24 inference GPUs total. This makes the previously manual-only DEP/DEP -# experiment reproducible while the June 21 AgentX frontier is retuned. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.23.0" - precision: "fp4" - -dynamo: - install: true - wheel: "1.3.0.dev20260618" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 4 - decode_nodes: 2 - prefill_workers: 2 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_REQUEST_PLANE: "tcp" - # Dynamo's 10-second default can expire during brief etcd stalls while - # many long-loading DEP ranks initialize. Slurm still detects hard exits. - ETCD_LEASE_TTL: "120" - args: - router-mode: "kv" - router-reset-states: true - -backend: - type: vllm - connector: null - prefill_environment: - DYN_REQUEST_PLANE: "tcp" - ETCD_LEASE_TTL: "120" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - # Long AgentX requests can make one DEP rank hold the EP step beyond - # vLLM's 300-second model-execution timeout. - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_RCACHE_MAX_UNRELEASED: "1024" - UCX_TLS: "cuda_copy,rc" - NCCL_P2P_LEVEL: NVL - decode_environment: - DYN_REQUEST_PLANE: "tcp" - ETCD_LEASE_TTL: "120" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_RCACHE_MAX_UNRELEASED: "1024" - UCX_TLS: "cuda_copy,rc" - NCCL_P2P_LEVEL: NVL - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' - # Let vLLM select max-num-seqs for the long-context AgentX trace. - # max-num-seqs: 256 - # DEP8 holds a full model replica per GPU. A 16K prefill step requires - # a 9.84 GiB FP4 MoE intermediate and OOMs before serving traffic. - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"PIECEWISE"}' - gpu-memory-utilization: 0.9 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - enable-ep-weight-filter: true - attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_fp4_indexer_cache": true}' - # Let vLLM select max-num-seqs and max-num-batched-tokens. - # max-num-seqs: 512 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - -sbatch_directives: - cpus-per-task: "72" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - aiperf_server_metrics: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml deleted file mode 100644 index 9e2b665843..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml +++ /dev/null @@ -1,155 +0,0 @@ -name: "svf-vllm-disagg-gb200-3p2d-tep8-tp8-agentic" - -# Middle/high-interactivity topology: three cache-affinitized TEP8 prefill -# workers and two independent TP8 decode workers, for 40 inference GPUs. -# The dense historical curve covered roughly 47--71 output tok/s/user through -# c16--c80. This recipe re-establishes that curve on the June 21 AgentX corpus -# with the current vLLM/Dynamo runtime contract. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.23.0" - precision: "fp4" - -dynamo: - install: true - wheel: "1.3.0.dev20260618" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb200" - gpus_per_node: 4 - prefill_nodes: 6 - decode_nodes: 4 - prefill_workers: 3 - decode_workers: 2 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - env: - DYN_REQUEST_PLANE: "tcp" - ETCD_LEASE_TTL: "120" - args: - router-mode: "kv" - router-reset-states: true - -backend: - type: vllm - connector: null - prefill_environment: - DYN_REQUEST_PLANE: "tcp" - ETCD_LEASE_TTL: "120" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: "1024" - VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: "2048" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_RCACHE_MAX_UNRELEASED: "1024" - UCX_TLS: "cuda_copy,rc" - NCCL_P2P_LEVEL: NVL - decode_environment: - DYN_REQUEST_PLANE: "tcp" - ETCD_LEASE_TTL: "120" - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - TORCH_SYMMMEM: "NVSHMEM" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: "1800" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - UCX_MEMTYPE_CACHE: "n" - UCX_MEMTYPE_REG_WHOLE: "n" - UCX_RCACHE_MAX_UNRELEASED: "1024" - UCX_TLS: "cuda_copy,rc" - NCCL_P2P_LEVEL: NVL - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:20080","enable_kv_cache_events":true}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - enable-expert-parallel: true - enable-ep-weight-filter: true - attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_prefill_query_quantization": true, "use_fp4_indexer_cache": true}' - # Let vLLM select max-num-seqs for the long-context AgentX trace. - # max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - # FULL capture conflicts with NCCL symmetric-memory registration on the - # TP prefill path; PIECEWISE preserves compilation and safe graph regions. - compilation-config: '{"cudagraph_mode":"PIECEWISE"}' - gpu-memory-utilization: 0.9 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 8 - pipeline-parallel-size: 1 - attention-config: '{"backend": "FLASHINFER_MLA_SPARSE_DSV4", "use_fp4_indexer_cache": true}' - # Let vLLM select max-num-seqs and max-num-batched-tokens. - # max-num-seqs: 512 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - # TP8 all-reduce uses NCCL symmetric memory, which is incompatible with - # the full graph-capture path. Keep graph-incompatible collectives eager. - compilation-config: '{"cudagraph_mode":"PIECEWISE"}' - gpu-memory-utilization: 0.9 - stream-interval: 10 - all2all-backend: "flashinfer_nvlink_one_sided" - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - -sbatch_directives: - cpus-per-task: "72" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - aiperf_server_metrics: true - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_DYNAMO_SESSION_TIMEOUT_SECONDS: "14400" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml deleted file mode 100644 index dd58dde28f..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml +++ /dev/null @@ -1,203 +0,0 @@ -name: "svf-vllm-disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic" - -# Validated GB300 AgentX MTP3 low-latency topology: one DEP4 prefill worker -# feeds one DEP8 decode worker at concurrency 128. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - version: "1.3.0.dev20260720" - install: true - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false - prefill_nodes: 1 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 64 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - max-cudagraph-capture-size: 256 - gpu-memory-utilization: 0.92 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 256 - max-num-batched-tokens: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 1024 - gpu-memory-utilization: 0.92 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - prefill_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-c128-mtp-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - decode_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-c128-mtp-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - -sbatch_directives: - cpus-per-task: "72" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml deleted file mode 100644 index 00a876ebe8..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml +++ /dev/null @@ -1,204 +0,0 @@ -name: "svf-vllm-disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic" - -# Validated GB300 AgentX MTP3 efficiency topology: one DEP8 prefill worker and -# one DEP8 decode worker at concurrency 384. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - precision: "fp4" - -identity: - model: - repo: "deepseek-ai/DeepSeek-V4-Pro" - container: - image: "vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f" - frameworks: - dynamo: "1.3.0.dev20260720" - -dynamo: - version: "1.3.0.dev20260720" - install: true - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 2160 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - het_jobs: false - spread_workers: false - prefill_nodes: 2 - decode_nodes: 2 - prefill_workers: 1 - decode_workers: 1 - gpus_per_prefill: 8 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: false - nats_max_payload_mb: 32 - -environment: - # Mooncake prefix-block hashes must match across processes and nodes. - PYTHONHASHSEED: "0" - -frontend: - type: dynamo - enable_multiple_frontends: false - args: - router-mode: "random" - router-session-affinity-ttl-secs: 900 - env: - DYN_ROUTER_ACTIVE_REQUEST_EXPIRY_SECS: "3600" - -backend: - type: vllm - connector: null - dp_launch_mode: per_node - mooncake_kv_store: - store_config: - metadata_server: "P2PHANDSHAKE" - global_segment_size: "150GB" - local_buffer_size: "4GB" - protocol: "rdma" - device_name: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - mode: "embedded" - enable_offload: false - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true,"lookup_async":true,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 32 - max-num-batched-tokens: 8192 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - max-cudagraph-capture-size: 128 - gpu-memory-utilization: 0.92 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - decode: - kv-transfer-config: '{"kv_connector":"MultiConnector","kv_role":"kv_both","kv_connector_extra_config":{"connectors":[{"kv_connector":"NixlConnector","kv_role":"kv_both","kv_load_failure_policy":"fail","kv_buffer_device":"cuda","kv_connector_extra_config":{"enforce_handshake_compat":false,"enable_cross_layers_blocks":false}},{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"load_async":true,"lookup_async":false,"enable_lookup":false,"enable_cross_layers_blocks":false,"enable_offload":false}}]}}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-cumem-allocator: true - enable-expert-parallel: true - enable-ep-weight-filter: true - max-model-len: 1048576 - max-num-seqs: 256 - max-num-batched-tokens: 1024 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - max-cudagraph-capture-size: 1024 - gpu-memory-utilization: 0.92 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: "deepseek_v4" - attention-config: '{"backend":"FLASHINFER_MLA_SPARSE_DSV4","use_prefill_query_quantization":true,"use_fp4_indexer_cache":true}' - speculative-config: '{"method":"mtp","num_speculative_tokens":3,"rejection_sample_method":"synthetic","synthetic_acceptance_length":2.49}' - moe-backend: "deep_gemm_amxf4_mega_moe" - numa-bind: true - numa-bind-nodes: [0, 0, 1, 1] - prefill_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_USE_BREAKABLE_CUDAGRAPH: "0" - VLLM_CONNECTOR_PREFETCH_DEPTH: "8" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-mtp-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - decode_environment: - HF_HUB_CACHE: "/hf_hub_cache" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache" - TRANSFORMERS_CACHE: "/hf_hub_cache" - VLLM_ENGINE_READY_TIMEOUT_S: "3600" - VLLM_RPC_TIMEOUT: "600000" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_V2_WARMUP_MAX_NUM_SEQS: "20" - VLLM_SERVER_DEV_MODE: "1" - VLLM_USE_V2_MODEL_RUNNER: "1" - VLLM_MOONCAKE_LOAD_RECV_THREADS: "20" - VLLM_ALLREDUCE_USE_SYMM_MEM: "0" - UCX_MEMTYPE_CACHE: "n" - UCX_NET_DEVICES: "mlx5_0:1,mlx5_1:1,mlx5_2:1,mlx5_3:1" - UCX_TLS: "rc,cuda_copy" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - NCCL_IB_HCA: "mlx5_0,mlx5_1,mlx5_2,mlx5_3" - VLLM_RANDOMIZE_DP_DUMMY_INPUTS: "1" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: "32768" - VLLM_DSV4_MEGA_FP8_COMBINE: "1" - DG_JIT_CACHE_DIR: "/tmp/dg-cache-dsv4-gb300-1p1d-mtp-{job_id}" - MC_ENABLE_DEST_DEVICE_AFFINITY: "1" - MC_STORE_CLIENT_METRIC: "1" - MC_STORE_CLIENT_METRIC_INTERVAL: "5" - MC_TE_METRIC: "0" - -sbatch_directives: - cpus-per-task: "72" - mem: "0" - -srun_options: - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" - RESULT_DIR: "/logs/agentic" - PORT: "8000" - IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" - AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - HF_HUB_CACHE: "/hf_hub_cache" - WEKA_LOADER_OVERRIDE: "semianalysis_cc_traces_weka_062126" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tp4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tp4-agentic.yaml deleted file mode 100644 index 13c8d353e3..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tp4-agentic.yaml +++ /dev/null @@ -1,177 +0,0 @@ -name: "svf-vllm-disagg-gb300-1p6d-dep4-tp4-agentic" - -# Agentic-coding variant of vllm/deepseek-v4/8k1k/disagg-gb300-1p6d-dep4-tp4.yaml. -# Topology is identical (1 prefill DEP=4 + 6 decode TP=4, 28 GPUs across 7 -# GB300 nodes + 1 dedicated NATS/etcd infra node) so we can compare against -# the fixed-seq-len 1p6d baseline at the same concurrency point (192). -# -# Divergence vs the 8k1k sibling: -# - benchmark.type: sa-bench -> custom (hands off to agentic_srt.sh) -# - max-model-len: removed (let vLLM derive from model config; agentic -# trajectories blow past any small explicit cap) -# - no-enable-prefix-caching: dropped (prefix caching MUST be on for -# trajectory reuse — entire point of agentic) -# Note: --enable-auto-tool-choice / --tool-call-parser / --reasoning-parser -# are NOT set on the worker. The dynamo-vllm worker entrypoint doesn't -# accept them (different arg parser than `vllm serve`). In disagg, chat -# parsing happens at the dynamo frontend, not at the worker. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.21.0-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 1 - decode_nodes: 6 - prefill_workers: 1 - decode_workers: 6 - gpus_per_prefill: 4 - gpus_per_decode: 4 - -infra: - etcd_nats_dedicated_node: true - # Raise NATS server max_payload from the 1 MiB default to 32 MiB. - # Agentic prompts at 50k-200k DSv4 tokens serialize to JSON at ~10-15 - # bytes/token, easily clearing 1-3 MB per request. Without this, every - # long-prompt prefill RPC gets rejected by the NATS server with - # "maximum payload exceeded" (visible in infra.out), and the dynamo - # frontend surfaces a misleading "NATS request ... deadline has elapsed" - # (it never gets a reply because the publish was rejected). 32 MiB gives - # ~10x headroom over the largest observed payload (3.2 MB) without - # crossing NATS's 64 MiB hard cap or Dynamo's 16 MiB advisory limit. - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - attention-config: '{"use_fp4_indexer_cache": true}' - moe-backend: "deep_gemm_mega_moe" - # enforce-eager: true - # max-num-seqs: 256 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-flashinfer-autotune: true - block-size: 256 - gpu-memory-utilization: 0.9 - enable-ep-weight-filter: true - no-disable-hybrid-kv-cache-manager: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 4 - pipeline-parallel-size: 1 - # max-num-seqs: 512 - trust-remote-code: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - enable-ep-weight-filter: true - all2all-backend: "flashinfer_nvlink_one_sided" - no-enable-flashinfer-autotune: true - enable-sleep-mode: true - tokenizer-mode: deepseek_v4 - -# sbatch + srun resource grants for clusters without per-GPU defaults. -# -# mem=0: allocate all available node memory (~868 GB on CW gb300). Without -# this, sbatch only requests ntasks × DefMemPerCPU = 8 × 4 GB = 32 GB for -# the whole job and worker cgroups OOM-kill mid model load (R7-R11 hit -# this; sacct showed AllocTRES mem=4G per step). -# -# cpus-per-task=72: give each task one CW gb300 NUMA socket (144 cores -# split 2 × 72). Critical for the *infra step* (etcd + nats) which -# srtctl spawns without --gres=gpu — on CW that means DefMemPerCPU -# applies and the step gets 1 CPU by default. With 24 dynamo DP ranks -# all hammering etcd for lease keep-alives, single-CPU etcd can't keep -# up and dies (R12 hit this; etcd reported max-cpu-set=1, leases -# deadline-exceeded, infra SIGKILL'd at 16:35:49). 72 CPUs is plenty -# for both etcd + nats AND for vLLM worker auxiliary threads. -# -# nv gb300 doesn't need this because cluster default DefCpuPerGPU=35 -# auto-allocates 4*35=140 CPUs per GPU-bearing task; cw has no per-GPU -# default. Setting it here is safe on both because the value is ≤ node -# CPU count. -# -# srun_options.mem=0 forces each srun step to use the full node memory -# (without it, srun steps default back to cpus_per_task × DefMemPerCPU). -# Docs: docs/config-reference.md#sbatch_directives + #srun_options. -sbatch_directives: - mem: "0" - cpus-per-task: "72" -srun_options: - mem: "0" - # gb300-nv: pyxis maps the calling user (sa-shared) into the container as - # uid 345200007. dpkg refuses to run without EUID 0 even though - # ENROOT_ROOTFS_WRITABLE=1 makes the rootfs writable, so the agentic_srt - # apt-get install git step fails. --container-remap-root asks pyxis to - # remap us to uid 0 inside the container. srt-slurm renders empty-string - # values as flag-only srun args (see core/slurm.py:250). - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - # Container-side path of the aiperf mmap dataset cache; the host-side - # mount is wired via launch_gb300-*.sh's srtslurm.yaml default_mounts. - # Without this, aiperf re-tokenizes + re-writes ~65 GB of mmap files - # per dataset on every run. - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - # Persistent HF hub cache (also wired via default_mounts) so the trace - # dataset isn't re-downloaded on every run. Overrides the workflow-level - # HF_HUB_CACHE=/mnt/hf_hub_cache, which doesn't exist on these nodes. - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-4p1d-dep4-dep8-24-c4096-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-4p1d-dep4-dep8-24-c4096-agentic.yaml deleted file mode 100644 index 05b779d541..0000000000 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/agentic/disagg-gb300-4p1d-dep4-dep8-24-c4096-agentic.yaml +++ /dev/null @@ -1,179 +0,0 @@ -name: "svf-vllm-disagg-gb300-4p1d-dep4-dep8-24-c4096-agentic" - -# Agentic-coding variant of vllm/deepseek-v4/8k1k/disagg-gb300-4p1d-dep4-dep8-24-c4096.yaml. -# Max-throughput shape: 4 prefill (DEP=4 each) + 1 decode (DEP=8). 6 GB300 -# nodes (4P + 2D = 24 GPUs at 4 GPUs/node) plus a dedicated NATS/etcd infra -# node. Sized for concurrency 4096 with deep_gemm_mega_moe on both workers. -# -# Divergence vs the 8k1k sibling: -# - benchmark.type: sa-bench -> custom (hands off to agentic_srt.sh) -# - max-model-len: removed (let vLLM derive from model config; agentic -# trajectories blow past any small explicit cap) -# - no-enable-prefix-caching: dropped (prefix caching MUST be on for -# trajectory reuse — entire point of agentic) -# Note: --enable-auto-tool-choice / --tool-call-parser / --reasoning-parser -# are NOT set on the worker. The dynamo-vllm worker entrypoint doesn't -# accept them (different arg parser than `vllm serve`). In disagg, chat -# parsing happens at the dynamo frontend, not at the worker. - -model: - path: "deepseek-v4-pro" - container: "vllm/vllm-openai:v0.21.0-ubuntu2404" - precision: "fp4" - -dynamo: - install: true - wheel: "1.2.0.dev20260426" - -setup_script: vllm-container-deps.sh - -slurm: - time_limit: "8:00:00" - -health_check: - max_attempts: 1440 - interval_seconds: 10 - -resources: - gpu_type: "gb300" - gpus_per_node: 4 - prefill_nodes: 4 - decode_nodes: 2 - prefill_workers: 4 - decode_workers: 1 - gpus_per_prefill: 4 - gpus_per_decode: 8 - -infra: - etcd_nats_dedicated_node: true - # See sibling 1p6d recipe for rationale — NATS 1 MiB default rejects - # agentic prompts; 32 MiB gives ~10x headroom over observed payloads. - nats_max_payload_mb: 32 - -frontend: - type: dynamo - enable_multiple_frontends: false - -backend: - type: vllm - connector: null - - prefill_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - - decode_environment: - TILELANG_CLEANUP_TEMP_FILES: "1" - VLLM_LOG_STATS_INTERVAL: "1" - VLLM_USE_NCCL_SYMM_MEM: "1" - NCCL_CUMEM_ENABLE: "1" - NCCL_MNNVL_ENABLE: "1" - NCCL_NVLS_ENABLE: "1" - TORCH_SYMMMEM: "NVSHMEM" - - vllm_config: - prefill: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 4 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - # enforce-eager: true - # Inherited from fixed-sequence recipes; let vLLM select the scheduler - # sequence limit until this is tuned explicitly for the agentic trace. - # max-num-seqs: 16 - max-num-batched-tokens: 16384 - trust-remote-code: true - no-enable-flashinfer-autotune: true - safetensors-load-strategy: "prefetch" - block-size: 256 - gpu-memory-utilization: 0.9 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - enable-ep-weight-filter: true - enable-sleep-mode: true - moe-backend: "deep_gemm_mega_moe" - - decode: - kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' - served-model-name: "deepseek-ai/DeepSeek-V4-Pro" - kv-cache-dtype: "fp8" - tensor-parallel-size: 1 - pipeline-parallel-size: 1 - data-parallel-size: 8 - data-parallel-rpc-port: 13345 - enable-expert-parallel: true - # max-num-seqs: 512 - trust-remote-code: true - block-size: 256 - compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' - gpu-memory-utilization: 0.9 - stream-interval: 10 - no-disable-hybrid-kv-cache-manager: true - tokenizer-mode: deepseek_v4 - enable-ep-weight-filter: true - enable-sleep-mode: true - moe-backend: "deep_gemm_mega_moe" - -# sbatch + srun resource grants for clusters without per-GPU defaults. -# -# mem=0: allocate all available node memory (~868 GB on CW gb300). Without -# this, sbatch only requests ntasks × DefMemPerCPU = 8 × 4 GB = 32 GB for -# the whole job and worker cgroups OOM-kill mid model load (R7-R11 hit -# this; sacct showed AllocTRES mem=4G per step). -# -# cpus-per-task=72: give each task one CW gb300 NUMA socket (144 cores -# split 2 × 72). Critical for the *infra step* (etcd + nats) which -# srtctl spawns without --gres=gpu — on CW that means DefMemPerCPU -# applies and the step gets 1 CPU by default. With 24 dynamo DP ranks -# all hammering etcd for lease keep-alives, single-CPU etcd can't keep -# up and dies (R12 hit this; etcd reported max-cpu-set=1, leases -# deadline-exceeded, infra SIGKILL'd at 16:35:49). 72 CPUs is plenty -# for both etcd + nats AND for vLLM worker auxiliary threads. -# -# nv gb300 doesn't need this because cluster default DefCpuPerGPU=35 -# auto-allocates 4*35=140 CPUs per GPU-bearing task; cw has no per-GPU -# default. Setting it here is safe on both because the value is ≤ node -# CPU count. -# -# srun_options.mem=0 forces each srun step to use the full node memory -# (without it, srun steps default back to cpus_per_task × DefMemPerCPU). -# Docs: docs/config-reference.md#sbatch_directives + #srun_options. -sbatch_directives: - mem: "0" - cpus-per-task: "72" -srun_options: - mem: "0" - # gb300-nv: pyxis maps the calling user (sa-shared) into the container as - # uid 345200007. dpkg refuses to run without EUID 0 even though - # ENROOT_ROOTFS_WRITABLE=1 makes the rootfs writable, so the agentic_srt - # apt-get install git step fails. --container-remap-root asks pyxis to - # remap us to uid 0 inside the container. srt-slurm renders empty-string - # values as flag-only srun args (see core/slurm.py:250). - container-remap-root: "" - -benchmark: - type: custom - command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh - env: - INFMAX_CONTAINER_WORKSPACE: /infmax-workspace - RESULT_DIR: /logs/agentic - PORT: "8000" - IS_MULTINODE: "true" - # Container-side path of the aiperf mmap dataset cache; the host-side - # mount is wired via launch_gb300-*.sh's srtslurm.yaml default_mounts. - # Without this, aiperf re-tokenizes + re-writes ~65 GB of mmap files - # per dataset on every run. - AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" - # Persistent HF hub cache (also wired via default_mounts) so the trace - # dataset isn't re-downloaded on every run. Overrides the workflow-level - # HF_HUB_CACHE=/mnt/hf_hub_cache, which doesn't exist on these nodes. - HF_HUB_CACHE: "/hf_hub_cache" diff --git a/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_b200.sh b/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_b200.sh deleted file mode 100755 index e75e3bff2f..0000000000 --- a/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_b200.sh +++ /dev/null @@ -1,198 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Kimi-K2.5 NVFP4 on B200 using vLLM. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=lmcache. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION - - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" -mkdir -p "$RESULT_DIR" - -OFFLOAD_ARGS=() -PREFIX_CACHE_ARGS=() -LMCACHE_PID="" - -cleanup_lmcache_server() { - if [[ -n "$LMCACHE_PID" ]] && kill -0 "$LMCACHE_PID" 2>/dev/null; then - kill "$LMCACHE_PID" 2>/dev/null || true - wait "$LMCACHE_PID" 2>/dev/null || true - fi -} - -trap cleanup_lmcache_server EXIT - -wait_for_lmcache_ready() { - { set +x; } 2>/dev/null - local attempts="${LMCACHE_READY_ATTEMPTS:-120}" - local tail_pid="" - - while [ ! -f "$LMCACHE_LOG" ]; do - if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then - echo "LMCache server died before creating log file. Exiting." >&2 - exit 1 - fi - sleep 1 - done - - tail -f -n +1 "$LMCACHE_LOG" & - tail_pid=$! - - for ((i = 1; i <= attempts; i++)); do - if curl --output /dev/null --silent --fail "http://127.0.0.1:${LMCACHE_HTTP_PORT}/healthcheck"; then - kill "$tail_pid" 2>/dev/null || true - wait "$tail_pid" 2>/dev/null || true - return 0 - fi - if [[ -n "$LMCACHE_PID" ]] && ! kill -0 "$LMCACHE_PID" 2>/dev/null; then - echo "LMCache server died before becoming healthy. Log follows:" >&2 - kill "$tail_pid" 2>/dev/null || true - wait "$tail_pid" 2>/dev/null || true - cat "$LMCACHE_LOG" >&2 || true - exit 1 - fi - sleep 1 - done - - echo "Timed out waiting for LMCache server healthcheck. Log follows:" >&2 - kill "$tail_pid" 2>/dev/null || true - wait "$tail_pid" 2>/dev/null || true - cat "$LMCACHE_LOG" >&2 || true - exit 1 -} - -if require_agentic_kv_offload_backend lmcache; then - { set +x; } 2>/dev/null - unset VLLM_USE_SIMPLE_KV_OFFLOAD - - agentic_pip_install --quiet --no-cache-dir 'lmcache==0.5.1' - python3 -c "import lmcache.integration.vllm.lmcache_mp_connector" >/dev/null - - # MP mode owns the configured CPU pool in the external LMCache - # server instead of passing - # --kv-offloading-size through vLLM's integrated LMCache convenience - # path, which divides the value by TP and then hits a large single-shot - # cudaHostAlloc in LMCache 0.4.5's single-process local CPU backend. - LMCACHE_HOST="${LMCACHE_HOST:-127.0.0.1}" - LMCACHE_PORT="${LMCACHE_PORT:-5555}" - LMCACHE_HTTP_PORT="${LMCACHE_HTTP_PORT:-8080}" - # LMCacheMPConnector builds its ZMQ endpoint by concatenating - # lmcache.mp.host and lmcache.mp.port, and its default host already - # includes the tcp:// scheme. Keep the server bind host raw, but pass - # a ZMQ-style host string to the connector. - LMCACHE_CONNECT_HOST="${LMCACHE_CONNECT_HOST:-tcp://$LMCACHE_HOST}" - LMCACHE_L1_SIZE_GB="${LMCACHE_L1_SIZE_GB:-$TOTAL_CPU_DRAM_GB}" - if [ "$LMCACHE_L1_SIZE_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then - echo "Error: LMCACHE_L1_SIZE_GB=$LMCACHE_L1_SIZE_GB exceeds configured capacity $TOTAL_CPU_DRAM_GB" >&2 - exit 1 - fi - # Initial allocation is deliberately small; --l1-size-gb above is the - # actual pool capacity and grows lazily as the run fills the cache. - LMCACHE_L1_INIT_SIZE_GB="${LMCACHE_L1_INIT_SIZE_GB:-20}" - LMCACHE_CHUNK_SIZE="${LMCACHE_CHUNK_SIZE:-256}" - LMCACHE_MAX_WORKERS="${LMCACHE_MAX_WORKERS:-$TP}" - export PYTHONHASHSEED="${PYTHONHASHSEED:-0}" - - echo "Starting LMCache MP server..." - LMCACHE_CMD=( - lmcache server - --host "$LMCACHE_HOST" - --port "$LMCACHE_PORT" - --http-host "$LMCACHE_HOST" - --http-port "$LMCACHE_HTTP_PORT" - --l1-size-gb "$LMCACHE_L1_SIZE_GB" - --l1-init-size-gb "$LMCACHE_L1_INIT_SIZE_GB" - --chunk-size "$LMCACHE_CHUNK_SIZE" - --max-workers "$LMCACHE_MAX_WORKERS" - --eviction-policy LRU - ) - printf '%q ' "${LMCACHE_CMD[@]}" > "$RESULT_DIR/lmcache_command.txt" - printf '\n' >> "$RESULT_DIR/lmcache_command.txt" - "${LMCACHE_CMD[@]}" > "$LMCACHE_LOG" 2>&1 & - LMCACHE_PID=$! - echo "LMCache server PID: $LMCACHE_PID" - wait_for_lmcache_ready - - PREFIX_CACHE_ARGS=(--enable-prefix-caching) - OFFLOAD_ARGS=( - --kv-transfer-config - "{\"kv_connector\":\"LMCacheMPConnector\",\"kv_connector_module_path\":\"lmcache.integration.vllm.lmcache_mp_connector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"lmcache.mp.host\":\"$LMCACHE_CONNECT_HOST\",\"lmcache.mp.port\":$LMCACHE_PORT}}" - --disable-hybrid-kv-cache-manager - ) -fi - -echo "Starting vllm server..." -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -# Disable vLLM v0.21+ CUDA-graph memory estimator. Its pre-reservation -# eats ~32% of HBM upfront which, combined with FP4 weights at TP=4 -# (~62 GB/GPU), leaves no room for KV blocks -- _check_enough_kv_cache_memory -# trips before the engine starts. Our --gpu-memory-utilization=0.90 already -# leaves ~18 GB/GPU slack outside vLLM's budget, which is the same safety -# net the estimator provides, so disabling it is redundant rather than -# unsafe. -export VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 - -{ set +x; } 2>/dev/null -VLLM_CMD=( - vllm serve "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --tensor-parallel-size="$TP" - --gpu-memory-utilization 0.90 - --max-num-seqs "$CONC" - --reasoning-parser kimi_k2 - --tool-call-parser kimi_k2 - --compilation_config.pass_config.fuse_allreduce_rms true - --kv-cache-dtype fp8 - --max-cudagraph-capture-size 2048 - --stream-interval 20 - --trust-remote-code - "${PREFIX_CACHE_ARGS[@]}" - "${OFFLOAD_ARGS[@]}" -) -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_b300.sh b/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_b300.sh deleted file mode 100755 index 94636fad41..0000000000 --- a/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_b300.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Kimi-K2.5 NVFP4 on B300 using vLLM. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=vllm-simple. - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION - - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -OFFLOAD_ARGS=() -PREFIX_CACHE_ARGS=() - -if require_agentic_kv_offload_backend vllm-simple; then - export VLLM_USE_SIMPLE_KV_OFFLOAD=1 - OFFLOAD_ARGS=( - --kv_offloading_backend native - --kv_offloading_size "$TOTAL_CPU_DRAM_GB" - --disable-hybrid-kv-cache-manager - ) -fi - -echo "Starting vllm server..." -export PYTHONNOUSERSITE=1 - -{ set +x; } 2>/dev/null -VLLM_CMD=( - vllm serve "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --tensor-parallel-size="$TP" - --gpu-memory-utilization 0.90 - --max-num-seqs "$CONC" - --reasoning-parser kimi_k2 - --tool-call-parser kimi_k2 - --compilation_config.pass_config.fuse_allreduce_rms true - --kv-cache-dtype fp8 - --max-cudagraph-capture-size 2048 - --stream-interval 20 - --trust-remote-code - "${PREFIX_CACHE_ARGS[@]}" - "${OFFLOAD_ARGS[@]}" -) -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_b300_mtp.sh b/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_b300_mtp.sh deleted file mode 100755 index 20f6ad25d9..0000000000 --- a/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_b300_mtp.sh +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION - - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -DRAFT_MODEL="lightseekorg/kimi-k2.6-eagle3.1-mla" - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi - DRAFT_MODEL_PATH="/data/models/${DRAFT_MODEL##*/}" - if [[ ! -d "$DRAFT_MODEL_PATH" || -z "$(ls -A "$DRAFT_MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$DRAFT_MODEL" --local-dir "$DRAFT_MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" - hf download "$DRAFT_MODEL" - DRAFT_MODEL_PATH="$DRAFT_MODEL" -fi -nvidia-smi - -resolve_trace_source -install_agentic_deps - -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -SERVER_PID="" - -cleanup_agentic_services() { - local exit_code=$? - trap - EXIT INT TERM - set +e - stop_background_process_tree "$SERVER_PID" "vLLM server" 60 - exit "$exit_code" -} -trap cleanup_agentic_services EXIT -trap 'exit 130' INT -trap 'exit 143' TERM - -DCP_SIZE="${DCP_SIZE:-1}" -DCP_ARGS=() -if [[ "$DCP_SIZE" -gt 1 ]]; then - DCP_ARGS+=(--decode-context-parallel-size "$DCP_SIZE" --dcp-comm-backend a2a) - NUM_SPEC_TOKENS=3 - SYNTHETIC_ACCEPT_LEN=2.88 - SPEC_ARGS=(--speculative-config "{\"method\":\"eagle3\",\"model\":\"$DRAFT_MODEL_PATH\",\"num_speculative_tokens\":$NUM_SPEC_TOKENS,\"rejection_sample_method\":\"synthetic\",\"synthetic_acceptance_length\":$SYNTHETIC_ACCEPT_LEN,\"attention_backend\":\"TOKENSPEED_MLA\"}") - ATTN_CONFIG='{"mla_prefill_backend":"TOKENSPEED_MLA"}' - COMPILATION_CONFIG='{"pass_config":{"fuse_allreduce_rms":false}}' -else - NUM_SPEC_TOKENS=4 - SYNTHETIC_ACCEPT_LEN=3.24 - SPEC_ARGS=(--speculative-config "{\"method\":\"eagle3\",\"model\":\"$DRAFT_MODEL_PATH\",\"num_speculative_tokens\":$NUM_SPEC_TOKENS,\"rejection_sample_method\":\"synthetic\",\"synthetic_acceptance_length\":$SYNTHETIC_ACCEPT_LEN}") - ATTN_CONFIG='{"mla_prefill_backend":"TRTLLM_RAGGED","use_prefill_query_quantization":true}' - COMPILATION_CONFIG='{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' -fi -ATTN_BACKEND_ARGS=(--attention-backend TOKENSPEED_MLA) - -OFFLOAD_ARGS=() - -if agentic_kv_offload_enabled; then - case "$KV_OFFLOAD_BACKEND" in - native) - export VLLM_USE_SIMPLE_KV_OFFLOAD=1 - CPU_OFFLOAD_BYTES=$((TOTAL_CPU_DRAM_GB * 1024 * 1024 * 1024)) - OFFLOAD_ARGS=( - --disable-hybrid-kv-cache-manager - --kv-transfer-config - "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use\":$CPU_OFFLOAD_BYTES,\"lazy_offload\":false}}" - ) - ;; - *) echo "Error: unsupported KV_OFFLOAD_BACKEND value '$KV_OFFLOAD_BACKEND' with EAGLE3 (expected: native)" >&2; exit 1 ;; - esac -fi - - -GMU=0.90 -if [[ "$DCP_SIZE" -gt 1 && "$KV_OFFLOADING" == "none" ]]; then - GMU=0.85 -fi - -echo "Starting vllm server..." -export PYTHONNOUSERSITE=1 - -export VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm - -{ set +x; } 2>/dev/null -VLLM_CMD=( - vllm serve "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --kv-cache-dtype fp8 - --trust-remote-code - --block-size 64 - --language-model-only - --gpu-memory-utilization "$GMU" - --max-num-seqs "$CONC" - "${ATTN_BACKEND_ARGS[@]}" - --attention-config "$ATTN_CONFIG" - --compilation-config "$COMPILATION_CONFIG" - --max-cudagraph-capture-size 2048 - --max-num-batched-tokens 16384 - --stream-interval 10 - --enable-prefix-caching - --tensor-parallel-size "$TP" - "${SPEC_ARGS[@]}" - "${DCP_ARGS[@]}" - "${OFFLOAD_ARGS[@]}" -) -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -build_replay_cmd "$RESULT_DIR" - -run_agentic_replay_and_write_outputs "$RESULT_DIR" diff --git a/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_mi355x.sh b/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_mi355x.sh deleted file mode 100755 index b0d5f801c8..0000000000 --- a/benchmarks/single_node/agentic/deprecated/kimik2.5_fp4_mi355x.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Kimi-K2.5 FP4 on MI355X using vLLM. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=vllm-native. - - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# ROCR/HIP visibility for vLLM 0.14+ -if [ -n "${ROCR_VISIBLE_DEVICES:-}" ]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -rocm-smi || true -amd-smi || true - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# Install amd-quark for MXFP4 (manual install due to ROCm vLLM bug) -pip install amd-quark - -# Disable AITER RMSNorm for TP < 8 due to accuracy issues -if [ "${TP}" -lt 8 ]; then - export VLLM_ROCM_USE_AITER_RMSNORM=0 -fi -# Workaround for MEC FW <177 RCCL memory reclaim issue -version=$(rocm-smi --showfw 2>/dev/null | grep MEC | head -n 1 | awk '{print $NF}') -if [[ "$version" == "" || ${version:-0} -lt 177 ]]; then - export HSA_NO_SCRATCH_RECLAIM=1 -fi - -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -OFFLOAD_ARGS=() -PREFIX_CACHE_ARGS=() - -if require_agentic_kv_offload_backend vllm-native; then - unset VLLM_USE_SIMPLE_KV_OFFLOAD - # Use vLLM's regular native KV-offload path (OffloadingConnector), - # NOT the SimpleCPUOffloadConnector. The "vllm-native" backend resolves to - # OffloadingConnector by default; setting VLLM_USE_SIMPLE_KV_OFFLOAD=1 - # would switch it to SimpleCPUOffloadConnector. We intentionally leave - # that env var UNSET here so the regular OffloadingConnector path is - # used. The shortcut --kv_offloading_backend native + --kv_offloading_size - # form constructs the KVTransferConfig at engine startup - # (vllm/config/vllm.py:662). - OFFLOAD_ARGS=( - --kv_offloading_backend native - --kv_offloading_size "$TOTAL_CPU_DRAM_GB" - --disable-hybrid-kv-cache-manager - ) -fi - -EP_ARGS=() -if [ "$EP_SIZE" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) -fi - -echo "Starting vllm server..." -export PYTHONNOUSERSITE=1 - -{ set +x; } 2>/dev/null -VLLM_CMD=( - vllm serve "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --tensor-parallel-size="$TP" - "${EP_ARGS[@]}" - --gpu-memory-utilization 0.90 - --block-size=1 - --trust-remote-code - --max-num-seqs "$CONC" - --mm-encoder-tp-mode data - "${PREFIX_CACHE_ARGS[@]}" - "${OFFLOAD_ARGS[@]}" -) -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/deprecated/kimik2.5_int4_b200.sh b/benchmarks/single_node/agentic/deprecated/kimik2.5_int4_b200.sh deleted file mode 100755 index b216ec7b4c..0000000000 --- a/benchmarks/single_node/agentic/deprecated/kimik2.5_int4_b200.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Kimi-K2.5 INT4 on B200 using vLLM. -# -# Required env vars: -# MODEL, TP, CONC, RESULT_DIR - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION - - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -OFFLOAD_ARGS="" -if require_agentic_kv_offload_backend vllm-simple; then - export VLLM_USE_SIMPLE_KV_OFFLOAD=1 - OFFLOAD_ARGS="--kv_offloading_backend native --kv_offloading_size $TOTAL_CPU_DRAM_GB --disable-hybrid-kv-cache-manager" -fi - -echo "Starting vllm server..." -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -export VLLM_USE_FLASHINFER_MOE_INT4=1 - -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ ---host 0.0.0.0 \ ---port $PORT \ ---gpu-memory-utilization 0.95 \ ---tensor-parallel-size $TP \ ---max-num-seqs $CONC \ ---reasoning-parser kimi_k2 \ ---tool-call-parser kimi_k2 \ ---compilation_config.pass_config.fuse_allreduce_rms true \ ---trust-remote-code \ -$OFFLOAD_ARGS > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/deprecated/kimik2.5_int4_h100.sh b/benchmarks/single_node/agentic/deprecated/kimik2.5_int4_h100.sh deleted file mode 100755 index 9f6ab9066b..0000000000 --- a/benchmarks/single_node/agentic/deprecated/kimik2.5_int4_h100.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Kimi-K2.5 INT4 on H100 using vLLM. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION - - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -OFFLOAD_ARGS="" -if require_agentic_kv_offload_backend vllm-simple; then - export VLLM_USE_SIMPLE_KV_OFFLOAD=1 - OFFLOAD_ARGS="--kv_offloading_backend native --kv_offloading_size $TOTAL_CPU_DRAM_GB --disable-hybrid-kv-cache-manager" -fi - -echo "Starting vllm server..." -export PYTHONNOUSERSITE=1 -export VLLM_USE_FLASHINFER_MOE_INT4=1 - -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ ---host 0.0.0.0 \ ---port $PORT \ ---gpu-memory-utilization 0.95 \ ---tensor-parallel-size $TP \ ---max-num-seqs $CONC \ ---reasoning-parser kimi_k2 \ ---tool-call-parser kimi_k2 \ ---compilation_config.pass_config.fuse_allreduce_rms true \ ---kv-cache-dtype fp8 \ ---trust-remote-code \ -$OFFLOAD_ARGS > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/deprecated/kimik2.5_int4_h200.sh b/benchmarks/single_node/agentic/deprecated/kimik2.5_int4_h200.sh deleted file mode 100755 index 683dcd254d..0000000000 --- a/benchmarks/single_node/agentic/deprecated/kimik2.5_int4_h200.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Kimi-K2.5 INT4 on H200 using vLLM. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION - - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -OFFLOAD_ARGS="" -if require_agentic_kv_offload_backend vllm-simple; then - # Kimi K2.5 is pure TP (no DP-attn): single engine, world_size=TP. - # SimpleCPUOffloadConnector internally divides cpu_bytes_to_use by - # world_size, so pass the full TOTAL_CPU_DRAM_GB; TP-shared mmap - # keeps the aggregate at TOTAL. - PER_ENGINE_BYTES=$((TOTAL_CPU_DRAM_GB * 1024 * 1024 * 1024)) - # JSON form (rather than --kv_offloading_backend native shortcut) so we can - # pass lazy_offload=true. Eager mode hits a popleft_n AssertionError at - # low/mid CONC on DSv4 + SimpleCPUOffloadConnector. - export VLLM_USE_SIMPLE_KV_OFFLOAD=1 - OFFLOAD_ARGS="--kv-transfer-config {\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use\":$PER_ENGINE_BYTES,\"lazy_offload\":true}}" -fi - -echo "Starting vllm server..." -export PYTHONNOUSERSITE=1 -export VLLM_USE_FLASHINFER_MOE_INT4=1 - -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ ---host 0.0.0.0 \ ---port $PORT \ ---gpu-memory-utilization 0.95 \ ---tensor-parallel-size $TP \ ---max-num-seqs $CONC \ ---reasoning-parser kimi_k2 \ ---tool-call-parser kimi_k2 \ ---compilation_config.pass_config.fuse_allreduce_rms true \ ---trust-remote-code \ -$OFFLOAD_ARGS > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/deprecated/qwen3.5_bf16_b200.sh b/benchmarks/single_node/agentic/deprecated/qwen3.5_bf16_b200.sh deleted file mode 100755 index 36f47e9147..0000000000 --- a/benchmarks/single_node/agentic/deprecated/qwen3.5_bf16_b200.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Qwen3.5 BF16 on B200 using SGLang. -# -# Required env vars: -# MODEL, TP, CONC, RESULT_DIR - -source "$(dirname "$0")/../../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC RESULT_DIR DURATION EP_SIZE - -SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-10} - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# ---- Start SGLang server ---------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -echo "Starting SGLang server..." -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -export NCCL_NVLS_ENABLE=1 -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true - -python3 -m sglang.launch_server \ ---model-path=$MODEL_PATH --served-model-name=$MODEL \ ---host=0.0.0.0 \ ---port=$PORT \ ---served-model-name "Qwen/Qwen3.5-397B-A17B" \ ---trust-remote-code \ ---tensor-parallel-size=$TP \ ---data-parallel-size=1 \ ---ep-size $EP_SIZE \ ---cuda-graph-max-bs $CONC \ ---max-running-requests $CONC \ ---mem-fraction-static 0.82 \ ---chunked-prefill-size 32768 \ ---max-prefill-tokens 32768 \ ---attention-backend trtllm_mha \ ---moe-runner-backend flashinfer_trtllm \ ---enable-flashinfer-allreduce-fusion \ ---scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ ---tokenizer-worker-num 6 \ ---stream-interval 30 \ ---enable-metrics > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/dsv4_fp8_h200.sh b/benchmarks/single_node/agentic/dsv4_fp8_h200.sh deleted file mode 100755 index 155f26ddea..0000000000 --- a/benchmarks/single_node/agentic/dsv4_fp8_h200.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for DeepSeek-V4-Pro FP8 on H200 using vLLM. -# Uses the cu129 image; H200 has no FP4 path so the FP4 indexer cache flag -# is omitted. Max-model-len pinned at 800k per the recipe. -# -# Required env vars: -# MODEL, TP, CONC, RESULT_DIR - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC RESULT_DIR DURATION - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# DeepSeek-V4-Pro weights are large; engine startup can exceed default 600s. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 - -# ---- Start vLLM server ------------------------------------------------------ -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -echo "Starting vLLM server..." -export PYTHONNOUSERSITE=1 - -# Per recipe: EP + DP=8 (no --tensor-parallel-size). TP from search space is -# used for GPU allocation by the runner and as the DP size. -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ ---host 0.0.0.0 \ ---port $PORT \ ---trust-remote-code \ ---kv-cache-dtype fp8 \ ---block-size 256 \ ---enable-expert-parallel \ ---data-parallel-size $TP \ ---gpu-memory-utilization 0.95 \ ---max-num-seqs $CONC \ ---max-num-batched-tokens 512 \ ---no-enable-flashinfer-autotune \ ---compilation-config '{"mode":0,"cudagraph_mode":"FULL_DECODE_ONLY"}' \ ---tokenizer-mode deepseek_v4 \ ---tool-call-parser deepseek_v4 \ ---enable-auto-tool-choice \ ---reasoning-parser deepseek_v4 > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm.sh deleted file mode 100755 index 17eb39dec3..0000000000 --- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm.sh +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail -set -x - -# Agentic trace replay benchmark for Kimi-K3 (MXFP4) on B300 using vLLM. -# -# Day-zero single-node recipe. Serve flags follow the Kimi-K3 production recipe -# already exercised by the DSpark AL collector -# (benchmarks/single_node/speedbench/kimik3_fp4_b300_vllm.sh) and the upstream -# recipes.vllm.ai/moonshotai/Kimi-K3 guidance, adapted to the agentic scenario. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION -# -# TP8 is the only single-node layout. The MXFP4 checkpoint is ~1.5 TB on disk; -# at TP4 that is ~375 GB of weights per GPU against B300's 288 GB of HBM, so -# only the full 8-GPU shard (~188 GB/GPU) fits. Do not add TP4/TP2 arms. -# -# Weights are pre-staged node-local: `Kimi-K3` is in the b300-nv launcher's -# STAGED_MODELS, so MODEL_PATH resolves to the read-only /scratch/models/Kimi-K3 -# mount and no download happens on the runner. The download block below only -# covers stand-alone runs. -# -# KV_OFFLOADING: `none` (GPU-resident) or `dram` with -# KV_OFFLOAD_BACKEND=vllm-simple (SimpleCPUOffloadConnector). -# -# Note on the DRAM arm: Kimi-K3 is a KDA/MLA hybrid — its linear-attention (KDA) -# layers carry a recurrent state rather than paged KV blocks, and -# SimpleCPUOffloadConnector offloads uniform paged blocks with no -# hybrid-geometry handling. The arm is expected to offload only the MLA layers' -# paged blocks; watch the bring-up sweep for KV-geometry errors at server init. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION - -# The 2.8T MXFP4 checkpoint only fits across all 8 B300s (see header). -if [ "$TP" -ne 8 ]; then - echo "Error: Kimi-K3 on B300 requires TP=8 (a ~1.5 TB MXFP4 checkpoint does not fit at TP<8), got TP='$TP'" >&2 - exit 1 -fi - -if [[ -n "${EP_SIZE:-}" && "${EP_SIZE}" -gt 1 ]]; then - echo "Error: this recipe ships the pure-TP8 profile; EP_SIZE='$EP_SIZE' is not wired yet" >&2 - exit 1 -fi - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE. -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# ---- Kimi-K3 production serving environment --------------------------------- -export NCCL_DMABUF_ENABLE=0 -export VLLM_ALLREDUCE_USE_FLASHINFER=1 -export VLLM_USE_RUST_FRONTEND=1 -# Loading ~1.5 TB of MXFP4 shards off the staged mount takes well past the -# default readiness window even with fastsafetensors. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export PYTHONNOUSERSITE=1 -# AIPerf pins one pooled keep-alive connection per agentic session and reuses it -# across turns, while the Rust frontend's default VLLM_HTTP_TIMEOUT_KEEP_ALIVE is -# 5s. An inter-turn idle gap longer than that lets the client reuse a socket at -# the moment the server closes it -> aiohttp ServerDisconnectedError -> AIPerf -# treats it as a terminal warmup failure and aborts the whole job. This killed -# the dram c4 arm ~15 min into run 30324907690 with a perfectly healthy server -# (it kept serving after the client gave up). Outlast the client pool so the -# race cannot occur. Same fix as glm5.2_fp4_b300_sglang.sh's -# SGLANG_TIMEOUT_KEEP_ALIVE=900. -export VLLM_HTTP_TIMEOUT_KEEP_ALIVE=900 -# Agentic warmup dispatches large prompts at once; allow up to 15 minutes of TCP -# progress before AIPerf declares a connection dead. -export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -# ---- KV offloading ---------------------------------------------------------- -# The generated TOTAL_CPU_DRAM_GB budget is the aggregate host-DRAM pool for the -# node; SimpleCPUOffloadConnector is sized per rank. At dram-utilization 0.63 on -# cluster:b300-nv this resolves to ~220 GiB per rank across the 8 TP ranks. -OFFLOAD_ARGS=() -case "${KV_OFFLOAD_BACKEND:-}" in - "") - require_agentic_kv_offload_none - ;; - vllm-simple) - require_agentic_kv_offload_backend vllm-simple - CPU_BYTES_PER_RANK=$(( TOTAL_CPU_DRAM_GB * 1000 * 1000 * 1000 / TP )) - # Identical prefixes must hash to identical block keys run-to-run. - export PYTHONHASHSEED=42 - # lazy_offload must be a JSON boolean, not a quoted string: the - # connector does bool(extra_config.get("lazy_offload", False)), and - # bool("false") is True in Python — a quoted "false" would silently - # turn lazy offload ON. Eager offload keeps block-hash behaviour - # aligned with the other B300 vllm-simple arms. - OFFLOAD_ARGS=( - --kv-transfer-config - "{\"kv_connector\":\"SimpleCPUOffloadConnector\",\"kv_role\":\"kv_both\",\"kv_connector_extra_config\":{\"cpu_bytes_to_use_per_rank\":${CPU_BYTES_PER_RANK},\"lazy_offload\":false}}" - ) - ;; - *) - echo "Error: unsupported KV_OFFLOAD_BACKEND='$KV_OFFLOAD_BACKEND' (expected empty or vllm-simple)" >&2 - exit 1 - ;; -esac - -# Agentic fan-out: keep the scheduler headroom convention shared by the other -# agentic recipes. Capture decode graphs only up to that batch size — a 93-layer -# 2.8T model makes capturing vLLM's full 2048-wide ladder prohibitively slow. -MAX_NUM_SEQS=$((2 * CONC)) - -echo "Starting vllm server..." - -{ set +x; } 2>/dev/null -VLLM_CMD=( - vllm serve "$MODEL_PATH" --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - --tensor-parallel-size "$TP" - --gpu-memory-utilization 0.90 - --max-num-seqs "$MAX_NUM_SEQS" - # Agentic replays run at the model's native context limit. - --max-model-len 1048576 - --trust-remote-code - --load-format fastsafetensors - --moe-backend auto - --enable-prefix-caching - --kv-cache-dtype fp8 - --reasoning-parser kimi_k3 - --tool-call-parser kimi_k3 - --enable-auto-tool-choice - # FP8 KV cache requires the prefill query quantization flag; MLA prefill - # runs on FlashInfer per the production recipe. - --attention-config '{"mla_prefill_backend":"FLASHINFER","use_prefill_query_quantization":true}' - --max-cudagraph-capture-size "$MAX_NUM_SEQS" - --disable-uvicorn-access-log - "${OFFLOAD_ARGS[@]}" -) -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh index faf2232726..9bb187c8a9 100755 --- a/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/kimik3_fp4_b300_vllm_mtp.sh @@ -181,7 +181,7 @@ SYNTHETIC_ACCEPT_LEN=2.51 # real target verification instead. Synthetic acceptance commits drafted tokens # regardless of the target's logits, so the generated text is wrong and the # SWE-bench eval scores 0.0000 -- the same split dsv4_fp4_b300_vllm_mtp.sh makes -# (and which kimik2.5_fp4_b300_mtp.sh omits; follow dsv4, not kimik2.5). +# (follow the DSV4 MTP precedent for this split). # rejection_sample_method=block does real verification, so it is what EVAL_ONLY # uses. vLLM rejects synthetic_acceptance_length unless the method is 'synthetic'. if [ "${EVAL_ONLY:-false}" = "true" ]; then diff --git a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh b/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh deleted file mode 100644 index 7faaf69a51..0000000000 --- a/benchmarks/single_node/agentic/minimaxm3_fp4_mi355x.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Minimax-M3 FP4 on MI355X using vLLM. -# -# Required env vars: -# MODEL, MODEL_PATH, TP, CONC, KV_OFFLOADING, KV_OFFLOAD_BACKEND, -# TOTAL_CPU_DRAM_GB, RESULT_DIR, DURATION, EP_SIZE, DP_ATTENTION - -source "$(dirname "$0")/../../benchmark_lib.sh" - -# Force the eval framework to lm-eval for this recipe. run_eval derives its -# default as swebench for agentic scenarios (scenario_default=swebench when -# IS_AGENTIC/SCENARIO_TYPE=agentic-coding), but EVAL_FRAMEWORK takes precedence -# over that default (benchmark_lib.sh: framework=${EVAL_FRAMEWORK:-...}), so -# setting it here makes the effective framework always lm-eval, never swebench. -export EVAL_FRAMEWORK="lm-eval" - -check_env_vars MODEL TP CONC KV_OFFLOADING KV_OFFLOAD_BACKEND TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION - -echo "MODEL=$MODEL TP=$TP CONC=$CONC KV_OFFLOADING=$KV_OFFLOADING TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB RESULT_DIR=$RESULT_DIR DURATION=$DURATION EP_SIZE=$EP_SIZE DP_ATTENTION=$DP_ATTENTION" - -PORT=8888 - -if [[ -n "${SLURM_JOB_ID+x}" ]]; then - echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" -fi - -# ROCR/HIP visibility for vLLM 0.14+ -if [[ -n "${ROCR_VISIBLE_DEVICES+x}" ]]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi - -rocm-smi || true -amd-smi || true - -resolve_trace_source -install_agentic_deps - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -LMCACHE_LOG="$RESULT_DIR/lmcache_server.log" -mkdir -p "$RESULT_DIR" - -OFFLOAD_ARGS=(--no-enable-prefix-caching) - -case "$KV_OFFLOAD_BACKEND" in - vllm-simple) - unset VLLM_USE_SIMPLE_KV_OFFLOAD - # Use vLLM's regular native KV-offload path (OffloadingConnector), - # NOT the SimpleCPUOffloadConnector. The "native" backend resolves to - # OffloadingConnector by default; setting VLLM_USE_SIMPLE_KV_OFFLOAD=1 - # would switch it to SimpleCPUOffloadConnector. We intentionally leave - # that env var UNSET here so the regular OffloadingConnector path is - # used. The shortcut --kv_offloading_backend native + --kv_offloading_size - # form constructs the KVTransferConfig at engine startup - # (vllm/config/vllm.py:662). - - # Remove --disable-hybrid-kv-cache-manager and enable hybrid kv cache manager (default) - # This gives extra cache hit than disabling hybrid kv cache manager - OFFLOAD_ARGS=( - --kv_offloading_backend native - --kv_offloading_size "$TOTAL_CPU_DRAM_GB" - ) - ;; -esac - -# ---- LLM server config ---------------------------------------------------------- -PARALLEL_ARGS=(--tensor-parallel-size "$TP") -if [ "${DP_ATTENTION}" = "true" ]; then - PARALLEL_ARGS=( - --tensor-parallel-size 1 - --data-parallel-size "$TP" - --enable-expert-parallel - ) -elif [ "$EP_SIZE" -gt 1 ]; then - PARALLEL_ARGS+=(--enable-expert-parallel) -fi - -echo "Starting vllm server..." -export PYTHONNOUSERSITE=1 - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 -export VLLM_ROCM_USE_AITER=1 -export VLLM_ROCM_USE_AITER_MOE=1 -export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 -# INT4 quantized all-reduce for the (~1.5 MB) decode all-reduces, which are the -# single biggest decode kernel at high concurrency. The MIN_SIZE_KB override is -# required: vLLM's default INT4 quick-reduce size gate for (bf16, TP4) is 16 MB, -# so it never fires for decode-sized tensors without it. -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 -export VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16=0 -export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION_MIN_SIZE_KB=256 - -VLLM_CMD=( - vllm serve "$MODEL_PATH" - --served-model-name "$MODEL" - --host 0.0.0.0 - --port "$PORT" - "${PARALLEL_ARGS[@]}" - --trust-remote-code - --block-size 128 - --gpu-memory-utilization 0.85 - --language-model-only - --attention-backend TRITON_ATTN - --moe-backend aiter - --kv-cache-dtype fp8 - --tool-call-parser minimax_m3 - --enable-auto-tool-choice - # NOTE: --reasoning-parser minimax_m3 is intentionally OMITTED. - # MiniMax-M3 is an interleaved-thinking model: its ... - # block MUST be round-tripped back into the conversation history every turn - # or multi-turn quality collapses (the model loses its plan and degenerates - # into repeating the same command until the step limit -> empty patch). - # The reasoning parser moves out of message.content into the - # response-only reasoning_content field, which the mini-swe-agent/litellm - # OpenAI client does NOT resend. Leaving the parser off keeps the think block - # inline in message.content, so the client preserves it across turns. The - # tool-call parser above still extracts tool calls from the full output. - --max-num-seqs "$CONC" - "${OFFLOAD_ARGS[@]}" -) -printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt" -printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt" -"${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -# ---- Run benchmark ---------------------------------------------------------- -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_h100.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_h100.sh deleted file mode 100755 index c5ab50b37f..0000000000 --- a/benchmarks/single_node/agentic/minimaxm3_fp8_h100.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126 -resolve_trace_source -install_agentic_deps - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export PYTHONNOUSERSITE=1 - -SERVER_LOG="$RESULT_DIR/server.log" -ROUTER_LOG="$RESULT_DIR/router.log" -MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" -mkdir -p "$RESULT_DIR" - -OFFLOAD_ARGS=() -MODEL_CPU_OFFLOAD_GB=26 -MODEL_CHECKPOINT_PAGE_CACHE_GIB=414 -MOONCAKE_LOCAL_BUFFER_GIB=4 -if require_agentic_kv_offload_backend mooncake; then - TOTAL_CPU_DRAM_GIB=$((TOTAL_CPU_DRAM_GB * 1000000000 / 1073741824)) - PER_RANK_GIB=$(((TOTAL_CPU_DRAM_GIB - MODEL_CHECKPOINT_PAGE_CACHE_GIB) / TP - MODEL_CPU_OFFLOAD_GB - MOONCAKE_LOCAL_BUFFER_GIB)) - if (( PER_RANK_GIB <= 0 )); then - echo "Error: CPU DRAM budget is too small for checkpoint cache, model, and KV offload" >&2 - exit 1 - fi - MOONCAKE_VERSION=0.3.11.post1 - agentic_pip_install --quiet --no-cache-dir --no-deps \ - --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION" - python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null - MOONCAKE_MASTER_PORT=$((PORT + 12000)) - MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" - cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & - MOONCAKE_MASTER_PID=$! - sleep 2 - kill -0 "$MOONCAKE_MASTER_PID" - OFFLOAD_ARGS=( - --kv-transfer-config - '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' - ) -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [[ "$DP_ATTENTION" == "true" ]]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if (( EP_SIZE > 1 )); then - EP_ARGS=(--enable-expert-parallel) -fi - -VLLM_BACKEND_PORT="$PORT" -if [[ "$DP_ATTENTION" == "true" ]]; then - VLLM_BACKEND_PORT=$((PORT + 1)) - export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 - agentic_pip_install --quiet 'vllm-router==0.1.14' -fi - -MAX_NUM_SEQS=$((2 * CONC)) -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ - --host 0.0.0.0 \ - --port "$VLLM_BACKEND_PORT" \ - "${PARALLEL_ARGS[@]}" \ - "${EP_ARGS[@]}" \ - --gpu-memory-utilization 0.95 \ - --cpu-offload-gb "$MODEL_CPU_OFFLOAD_GB" \ - --kv-cache-dtype fp8 \ - --attention-backend TRITON_ATTN \ - --block-size 128 \ - --language-model-only \ - --enable-prefix-caching \ - --max-num-seqs "$MAX_NUM_SEQS" \ - --tool-call-parser minimax_m3 \ - --reasoning-parser minimax_m3 \ - --enable-auto-tool-choice \ - --safetensors-load-strategy lazy \ - --trust-remote-code \ - "${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! - -wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [[ "$DP_ATTENTION" == "true" ]]; then - vllm-router \ - --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \ - --policy consistent_hash \ - --intra-node-data-parallel-size "$TP" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$((PORT + 10000))" \ - --request-timeout-secs 14400 \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_h200.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_h200.sh deleted file mode 100755 index 6b984cee03..0000000000 --- a/benchmarks/single_node/agentic/minimaxm3_fp8_h200.sh +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -resolve_complete_model_snapshot() { - python3 - "$1" <<'PY' -import json -import sys -from pathlib import Path - -model_cache_dir = Path(sys.argv[1]) -try: - revision = model_cache_dir.joinpath("refs/main").read_text().strip() -except OSError: - raise SystemExit - -if not revision or Path(revision).name != revision: - raise SystemExit - -snapshot = model_cache_dir / "snapshots" / revision -index_path = snapshot / "model.safetensors.index.json" -required_files = ( - snapshot / "config.json", - snapshot / "tokenizer_config.json", - index_path, -) -if not all(path.is_file() for path in required_files): - raise SystemExit -try: - weight_map = json.loads(index_path.read_text())["weight_map"] -except (KeyError, json.JSONDecodeError, OSError): - raise SystemExit -shards = {snapshot / filename for filename in weight_map.values()} -if shards and all(path.is_file() for path in shards): - print(snapshot) -PY -} - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - MODEL_CACHE_ROOT="${HF_HUB_CACHE:-${HF_HOME:-$HOME/.cache/huggingface/hub}}" - MODEL_CACHE_DIR="$MODEL_CACHE_ROOT/models--${MODEL//\//--}" - mkdir -p "$MODEL_CACHE_ROOT" - MODEL_PATH=$(resolve_complete_model_snapshot "$MODEL_CACHE_DIR") - if [[ -z "$MODEL_PATH" ]]; then - exec 9>"$MODEL_CACHE_ROOT/.minimaxm3-download.lock" - flock -w 3600 9 - MODEL_PATH=$(resolve_complete_model_snapshot "$MODEL_CACHE_DIR") - if [[ -z "$MODEL_PATH" ]]; then - DOWNLOADED_MODEL_PATH=$(hf download "$MODEL") - MODEL_PATH=$(resolve_complete_model_snapshot "$MODEL_CACHE_DIR") - if [[ -z "$MODEL_PATH" ]]; then - echo "Downloaded model snapshot is incomplete: $DOWNLOADED_MODEL_PATH" >&2 - exit 1 - fi - fi - flock -u 9 - fi - echo "Using complete cached model snapshot: $MODEL_PATH" - export MODEL_PATH -fi -nvidia-smi - -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126 -resolve_trace_source -install_agentic_deps - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export PYTHONNOUSERSITE=1 - -SERVER_LOG="$RESULT_DIR/server.log" -ROUTER_LOG="$RESULT_DIR/router.log" -MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" -mkdir -p "$RESULT_DIR" - -OFFLOAD_ARGS=() -if require_agentic_kv_offload_backend mooncake; then - PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / TP)) - MOONCAKE_VERSION=0.3.11.post1 - agentic_pip_install --quiet --no-cache-dir --no-deps \ - --force-reinstall "mooncake-transfer-engine-cuda13==$MOONCAKE_VERSION" - python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null - MOONCAKE_MASTER_PORT=$((PORT + 12000)) - MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" - cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & - MOONCAKE_MASTER_PID=$! - sleep 2 - kill -0 "$MOONCAKE_MASTER_PID" - OFFLOAD_ARGS=( - --kv-transfer-config - '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' - ) -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [[ "$DP_ATTENTION" == "true" ]]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if (( EP_SIZE > 1 )); then - EP_ARGS=(--enable-expert-parallel) -fi - -VLLM_BACKEND_PORT="$PORT" -if [[ "$DP_ATTENTION" == "true" ]]; then - VLLM_BACKEND_PORT=$((PORT + 1)) - export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 - agentic_pip_install --quiet 'vllm-router==0.1.14' -fi - -MAX_NUM_SEQS=$((2 * CONC)) -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ - --host 0.0.0.0 \ - --port "$VLLM_BACKEND_PORT" \ - "${PARALLEL_ARGS[@]}" \ - "${EP_ARGS[@]}" \ - --gpu-memory-utilization 0.92 \ - --kv-cache-dtype fp8 \ - --attention-backend TRITON_ATTN \ - --block-size 128 \ - --language-model-only \ - --enable-prefix-caching \ - --max-num-seqs "$MAX_NUM_SEQS" \ - --tool-call-parser minimax_m3 \ - --reasoning-parser minimax_m3 \ - --enable-auto-tool-choice \ - --trust-remote-code \ - "${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! - -wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [[ "$DP_ATTENTION" == "true" ]]; then - vllm-router \ - --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \ - --policy consistent_hash \ - --intra-node-data-parallel-size "$TP" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$((PORT + 10000))" \ - --request-timeout-secs 14400 \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x.sh deleted file mode 100755 index 4dc06f77af..0000000000 --- a/benchmarks/single_node/agentic/minimaxm3_fp8_mi300x.sh +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -rocm-smi || true -amd-smi || true - -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126 -resolve_trace_source -install_agentic_deps - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 -export PYTHONNOUSERSITE=1 - -SERVER_LOG="$RESULT_DIR/server.log" -ROUTER_LOG="$RESULT_DIR/router.log" -MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" -mkdir -p "$RESULT_DIR" - -install_mooncake_rocm() { - local mooncake_tag="v0.3.11.post1" - local mooncake_src="/tmp/Mooncake-$mooncake_tag" - local mooncake_stage="/tmp/mooncake-stage-$mooncake_tag" - local build_jobs - local cache_root - local cache_key - local cache_archive - local cache_tmp - local engine_path - local os_version - local python_abi - local rocm_version - - build_jobs=$(nproc) - if ((build_jobs > 32)); then - build_jobs=32 - fi - - os_version=$(. /etc/os-release && printf '%s-%s' "$ID" "$VERSION_ID") - python_abi=$(python3 -c 'import sys; print(f"cp{sys.version_info.major}{sys.version_info.minor}")') - rocm_version=$(sed -n '1p' /opt/rocm/.info/version 2>/dev/null || true) - if [[ -z "$rocm_version" ]]; then - rocm_version=$(hipconfig --version) - fi - rocm_version=${rocm_version//[^[:alnum:]._-]/_} - cache_root="${HF_HUB_CACHE:?HF_HUB_CACHE must be set}/inferencex/mooncake" - cache_key="${mooncake_tag}-${os_version}-${python_abi}-${rocm_version}-$(uname -m)-hip" - cache_archive="$cache_root/$cache_key.tar.gz" - mkdir -p "$cache_root" - - apt-get update - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - build-essential cmake git libasio-dev libboost-dev libcurl4-openssl-dev \ - libgflags-dev libgoogle-glog-dev libibverbs-dev libjsoncpp-dev \ - libnuma-dev libpython3-dev libssl-dev libunwind-dev liburing-dev \ - libxxhash-dev libyaml-cpp-dev libzstd-dev ninja-build pybind11-dev - - exec 9>"$cache_archive.lock" - flock -w 1800 9 - if [[ -f "$cache_archive" ]] && ! tar -tzf "$cache_archive" >/dev/null 2>&1; then - rm -f "$cache_archive" - fi - if [[ ! -f "$cache_archive" ]]; then - echo "Building HIP Mooncake cache artifact: $cache_archive" - rm -rf "$mooncake_src" "$mooncake_stage" - git clone --depth 1 --branch "$mooncake_tag" --recurse-submodules \ - --shallow-submodules https://github.com/kvcache-ai/Mooncake.git "$mooncake_src" - cmake -S "$mooncake_src/extern/yalantinglibs" \ - -B "$mooncake_src/extern/yalantinglibs/build" \ - -DBUILD_EXAMPLES=OFF -DBUILD_BENCHMARK=OFF -DBUILD_UNIT_TESTS=OFF - cmake --build "$mooncake_src/extern/yalantinglibs/build" -j "$build_jobs" - cmake --install "$mooncake_src/extern/yalantinglibs/build" - cmake -S "$mooncake_src" -B "$mooncake_src/build" -G Ninja \ - -DCMAKE_BUILD_TYPE=Release -DUSE_CUDA=OFF -DUSE_HIP=ON \ - -DWITH_EP=OFF -DWITH_STORE=ON -DWITH_STORE_RUST=OFF \ - -DWITH_RUST_EXAMPLE=OFF -DBUILD_EXAMPLES=OFF -DBUILD_UNIT_TESTS=OFF - cmake --build "$mooncake_src/build" -j "$build_jobs" - mkdir -p "$mooncake_stage" - DESTDIR="$mooncake_stage" cmake --install "$mooncake_src/build" - cache_tmp=$(mktemp "$cache_root/$cache_key.tmp.XXXXXX") - tar -C "$mooncake_stage" -czf "$cache_tmp" . - mv -f "$cache_tmp" "$cache_archive" - else - echo "Using HIP Mooncake cache artifact: $cache_archive" - fi - tar -C / -xzf "$cache_archive" - engine_path=$(python3 -c 'import mooncake.engine; print(mooncake.engine.__file__)') - ldd "$engine_path" | grep -q 'libamdhip64.so' - exec 9>&- -} - -OFFLOAD_ARGS=() -if require_agentic_kv_offload_backend mooncake; then - PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / TP)) - if ! python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null 2>&1; then - install_mooncake_rocm - fi - python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null - MOONCAKE_MASTER_PORT=$((PORT + 12000)) - MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" - cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & - MOONCAKE_MASTER_PID=$! - sleep 2 - kill -0 "$MOONCAKE_MASTER_PID" - OFFLOAD_ARGS=( - --kv-transfer-config - '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' - ) -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [[ "$DP_ATTENTION" == "true" ]]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if (( EP_SIZE > 1 )); then - EP_ARGS=(--enable-expert-parallel) -fi - -VLLM_BACKEND_PORT="$PORT" -if [[ "$DP_ATTENTION" == "true" ]]; then - VLLM_BACKEND_PORT=$((PORT + 1)) - export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 - agentic_pip_install --quiet 'vllm-router==0.1.14' -fi - -MAX_NUM_SEQS=$((2 * CONC)) -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ - --host 0.0.0.0 \ - --port "$VLLM_BACKEND_PORT" \ - "${PARALLEL_ARGS[@]}" \ - "${EP_ARGS[@]}" \ - --gpu-memory-utilization 0.95 \ - --block-size 128 \ - --language-model-only \ - --attention-backend TRITON_ATTN \ - --kv-cache-dtype fp8 \ - --enable-prefix-caching \ - --max-num-seqs "$MAX_NUM_SEQS" \ - --tool-call-parser minimax_m3 \ - --reasoning-parser minimax_m3 \ - --enable-auto-tool-choice \ - --trust-remote-code \ - "${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! - -wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [[ "$DP_ATTENTION" == "true" ]]; then - vllm-router \ - --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \ - --policy consistent_hash \ - --intra-node-data-parallel-size "$TP" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$((PORT + 10000))" \ - --request-timeout-secs 14400 \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x.sh b/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x.sh deleted file mode 100755 index 7cb862ae88..0000000000 --- a/benchmarks/single_node/agentic/minimaxm3_fp8_mi325x.sh +++ /dev/null @@ -1,218 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL IMAGE TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE DP_ATTENTION - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -if [[ -n "${ROCR_VISIBLE_DEVICES:-}" ]]; then - export HIP_VISIBLE_DEVICES="$ROCR_VISIBLE_DEVICES" -fi - -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -rocm-smi || true -amd-smi || true - -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_062126 -resolve_trace_source -install_agentic_deps - -export VLLM_ENGINE_READY_TIMEOUT_S=3600 -export VLLM_USE_BREAKABLE_CUDAGRAPH=0 -export PYTHONNOUSERSITE=1 - -SERVER_LOG="$RESULT_DIR/server.log" -ROUTER_LOG="$RESULT_DIR/router.log" -MOONCAKE_MASTER_LOG="$RESULT_DIR/mooncake_master.log" -mkdir -p "$RESULT_DIR" - -install_mooncake_rocm() { - local mooncake_tag="v0.3.11.post1" - local mooncake_src="/tmp/Mooncake-$mooncake_tag" - local mooncake_stage="/tmp/mooncake-stage-$mooncake_tag" - local build_jobs - local cache_root - local cache_key - local cache_archive - local cache_tmp - local engine_path - local os_version - local python_abi - local rocm_version - - build_jobs=$(nproc) - if ((build_jobs > 32)); then - build_jobs=32 - fi - - os_version=$(. /etc/os-release && printf '%s-%s' "$ID" "$VERSION_ID") - python_abi=$(python3 -c 'import sys; print(f"cp{sys.version_info.major}{sys.version_info.minor}")') - rocm_version=$(sed -n '1p' /opt/rocm/.info/version 2>/dev/null || true) - if [[ -z "$rocm_version" ]]; then - rocm_version=$(hipconfig --version) - fi - rocm_version=${rocm_version//[^[:alnum:]._-]/_} - cache_root="${HF_HUB_CACHE:?HF_HUB_CACHE must be set}/inferencex/mooncake" - cache_key="${mooncake_tag}-${os_version}-${python_abi}-${rocm_version}-$(uname -m)-hip" - cache_archive="$cache_root/$cache_key.tar.gz" - mkdir -p "$cache_root" - - apt-get update - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - build-essential cmake git libasio-dev libboost-dev libcurl4-openssl-dev \ - libgflags-dev libgoogle-glog-dev libibverbs-dev libjsoncpp-dev \ - libnuma-dev libpython3-dev libssl-dev libunwind-dev liburing-dev \ - libxxhash-dev libyaml-cpp-dev libzstd-dev ninja-build pybind11-dev - - exec 9>"$cache_archive.lock" - flock -w 1800 9 - if [[ -f "$cache_archive" ]] && ! tar -tzf "$cache_archive" >/dev/null 2>&1; then - rm -f "$cache_archive" - fi - if [[ ! -f "$cache_archive" ]]; then - echo "Building HIP Mooncake cache artifact: $cache_archive" - rm -rf "$mooncake_src" "$mooncake_stage" - git clone --depth 1 --branch "$mooncake_tag" --recurse-submodules \ - --shallow-submodules https://github.com/kvcache-ai/Mooncake.git "$mooncake_src" - cmake -S "$mooncake_src/extern/yalantinglibs" \ - -B "$mooncake_src/extern/yalantinglibs/build" \ - -DBUILD_EXAMPLES=OFF -DBUILD_BENCHMARK=OFF -DBUILD_UNIT_TESTS=OFF - cmake --build "$mooncake_src/extern/yalantinglibs/build" -j "$build_jobs" - cmake --install "$mooncake_src/extern/yalantinglibs/build" - cmake -S "$mooncake_src" -B "$mooncake_src/build" -G Ninja \ - -DCMAKE_BUILD_TYPE=Release -DUSE_CUDA=OFF -DUSE_HIP=ON \ - -DWITH_EP=OFF -DWITH_STORE=ON -DWITH_STORE_RUST=OFF \ - -DWITH_RUST_EXAMPLE=OFF -DBUILD_EXAMPLES=OFF -DBUILD_UNIT_TESTS=OFF - cmake --build "$mooncake_src/build" -j "$build_jobs" - mkdir -p "$mooncake_stage" - DESTDIR="$mooncake_stage" cmake --install "$mooncake_src/build" - cache_tmp=$(mktemp "$cache_root/$cache_key.tmp.XXXXXX") - tar -C "$mooncake_stage" -czf "$cache_tmp" . - mv -f "$cache_tmp" "$cache_archive" - else - echo "Using HIP Mooncake cache artifact: $cache_archive" - fi - tar -C / -xzf "$cache_archive" - engine_path=$(python3 -c 'import mooncake.engine; print(mooncake.engine.__file__)') - ldd "$engine_path" | grep -q 'libamdhip64.so' - exec 9>&- -} - -OFFLOAD_ARGS=() -if require_agentic_kv_offload_backend mooncake; then - PER_RANK_GB=$((TOTAL_CPU_DRAM_GB / TP)) - if ! python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null 2>&1; then - install_mooncake_rocm - fi - python3 -c "from mooncake.store import MooncakeDistributedStore" >/dev/null - MOONCAKE_MASTER_PORT=$((PORT + 12000)) - MOONCAKE_CONFIG_PATH="$RESULT_DIR/mooncake_config.json" - cat > "$MOONCAKE_CONFIG_PATH" < "$MOONCAKE_MASTER_LOG" 2>&1 & - MOONCAKE_MASTER_PID=$! - sleep 2 - kill -0 "$MOONCAKE_MASTER_PID" - OFFLOAD_ARGS=( - --kv-transfer-config - '{"kv_connector":"MooncakeStoreConnector","kv_role":"kv_both","kv_connector_extra_config":{"load_async":true}}' - ) -fi - -PARALLEL_ARGS=(--tensor-parallel-size "$TP" --data-parallel-size 1) -if [[ "$DP_ATTENTION" == "true" ]]; then - PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP") -fi - -EP_ARGS=() -if (( EP_SIZE > 1 )); then - EP_ARGS=(--enable-expert-parallel) -fi - -KV_CACHE_ARGS=() -if [[ "$IMAGE" == vllm/vllm-openai-rocm:nightly-* ]]; then - KV_CACHE_ARGS=(--kv-cache-dtype fp8) -fi - -VLLM_BACKEND_PORT="$PORT" -if [[ "$DP_ATTENTION" == "true" ]]; then - VLLM_BACKEND_PORT=$((PORT + 1)) - export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1 - agentic_pip_install --quiet 'vllm-router==0.1.14' -fi - -MAX_NUM_SEQS=$((2 * CONC)) -GPU_MEMORY_UTILIZATION=0.95 -if (( TP == 4 )); then - GPU_MEMORY_UTILIZATION=0.98 -fi - -vllm serve "$MODEL_PATH" --served-model-name "$MODEL" \ - --host 0.0.0.0 \ - --port "$VLLM_BACKEND_PORT" \ - "${PARALLEL_ARGS[@]}" \ - "${EP_ARGS[@]}" \ - --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \ - --block-size 128 \ - --language-model-only \ - --attention-backend TRITON_ATTN \ - --enable-prefix-caching \ - --max-num-seqs "$MAX_NUM_SEQS" \ - --tool-call-parser minimax_m3 \ - --reasoning-parser minimax_m3 \ - --enable-auto-tool-choice \ - --trust-remote-code \ - "${KV_CACHE_ARGS[@]}" \ - "${OFFLOAD_ARGS[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! - -wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [[ "$DP_ATTENTION" == "true" ]]; then - vllm-router \ - --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \ - --policy consistent_hash \ - --intra-node-data-parallel-size "$TP" \ - --host 0.0.0.0 \ - --port "$PORT" \ - --prometheus-host 127.0.0.1 \ - --prometheus-port "$((PORT + 10000))" \ - --request-timeout-secs 14400 \ - --disable-retries > "$ROUTER_LOG" 2>&1 & - ROUTER_PID=$! - wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID" -fi - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_b200.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_b200.sh deleted file mode 100755 index 58857e73c5..0000000000 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_b200.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Qwen3.5 FP8 on B200 using SGLang. -# -# Required env vars: -# MODEL, TP, CONC, RESULT_DIR - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC RESULT_DIR DURATION EP_SIZE - -SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-10} - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# ---- Start SGLang server ---------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -echo "Starting SGLang server..." -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -export NCCL_NVLS_ENABLE=1 -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true - -python3 -m sglang.launch_server \ ---model-path=$MODEL_PATH --served-model-name=$MODEL \ ---host=0.0.0.0 \ ---port=$PORT \ ---served-model-name "Qwen/Qwen3.5-397B-A17B-FP8" \ ---trust-remote-code \ ---tensor-parallel-size=$TP \ ---data-parallel-size=1 \ ---ep-size $EP_SIZE \ ---cuda-graph-max-bs $CONC \ ---max-running-requests $CONC \ ---mem-fraction-static 0.82 \ ---chunked-prefill-size 32768 \ ---max-prefill-tokens 32768 \ ---attention-backend trtllm_mha \ ---moe-runner-backend flashinfer_trtllm \ ---enable-flashinfer-allreduce-fusion \ ---scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ ---tokenizer-worker-num 6 \ ---stream-interval 30 \ ---enable-metrics > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang.sh deleted file mode 100755 index 8ae038dd7e..0000000000 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_b300_sglang.sh +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Qwen3.5 FP8 on B300 using SGLang. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE - -SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-10} - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -CACHE_ARGS=() -if require_agentic_kv_offload_backend hicache; then - # HiCache extends RadixAttention, so do not pass --disable-radix-cache. - # Qwen3.5's hybrid GDN/Mamba path allocates two HiCache host pools per TP - # rank: one for hierarchical KV cache and one for hierarchical Mamba cache. - REQUESTED_HICACHE_TOTAL_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-$TOTAL_CPU_DRAM_GB}" - if [ "$REQUESTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then - echo "Error: requested HiCache pool ${REQUESTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2 - exit 1 - fi - TOTAL_CPU_DRAM_GB="$REQUESTED_HICACHE_TOTAL_GB" - HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-2}" - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}" - # SGLang --hicache-size is per rank per host pool, while the workflow - # input is a node-total DRAM budget. Divide by TP and the number of - # host pools unless HICACHE_SIZE_GB is set directly for one-off tuning. - MAX_HICACHE_SIZE_GB=$((TOTAL_CPU_DRAM_GB / TP / HICACHE_HOST_POOL_COUNT)) - HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}" - if [ "$HICACHE_SIZE_GB" -gt "$MAX_HICACHE_SIZE_GB" ]; then - echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB exceeds configured per-pool limit $MAX_HICACHE_SIZE_GB" >&2 - exit 1 - fi - if [ "$HICACHE_SIZE_GB" -lt 1 ]; then - echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB from TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB, TP=$TP, HICACHE_HOST_POOL_COUNT=$HICACHE_HOST_POOL_COUNT" >&2 - exit 1 - fi - echo "HiCache CPU pool: ${HICACHE_SIZE_GB} GB per rank per host pool across TP=${TP}, host_pool_count=${HICACHE_HOST_POOL_COUNT}" - CACHE_ARGS=( - --page-size 64 - --enable-hierarchical-cache - --hicache-size "$HICACHE_SIZE_GB" - --hicache-io-backend kernel - --hicache-mem-layout page_first - --hicache-write-policy "$HICACHE_WRITE_POLICY" - ) -fi - -echo "Starting SGLang server..." -export TORCH_CUDA_ARCH_LIST="10.0" -export PYTHONNOUSERSITE=1 -export NCCL_NVLS_ENABLE=1 -export SGL_ENABLE_JIT_DEEPGEMM=false -export SGLANG_ENABLE_FLASHINFER_GEMM=true - -{ set +x; } 2>/dev/null -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path="$MODEL_PATH" --served-model-name="$MODEL" - --host=0.0.0.0 - --port="$PORT" - --served-model-name "Qwen/Qwen3.5-397B-A17B-FP8" - --trust-remote-code - --tensor-parallel-size="$TP" - --data-parallel-size=1 - --expert-parallel-size="$EP_SIZE" - --enable-symm-mem - --quantization fp8 - --kv-cache-dtype fp8_e4m3 - --mamba-ssm-dtype bfloat16 - --attention-backend trtllm_mha - --moe-runner-backend flashinfer_trtllm - --cuda-graph-max-bs "$CONC" - --max-running-requests "$CONC" - --max-prefill-tokens 16384 - --chunked-prefill-size 16384 - --mem-fraction-static 0.80 - --stream-interval 50 - --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" - --tokenizer-worker-num 6 - --tokenizer-path "$MODEL" - --enable-metrics - "${CACHE_ARGS[@]}" -) -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" -"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_h100.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_h100.sh deleted file mode 100755 index f58952a434..0000000000 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_h100.sh +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Qwen3.5 FP8 on H100 using SGLang. -# -# H100 has 80 GB HBM3 (vs B300's 192 GB), so weights + KV fit tighter. -# Mem-fraction-static lowered to 0.75 and chunked-prefill-size halved to -# 8192 (mirrors fixed_seq_len/qwen3.5_fp8_h100.sh). Attention backend is -# flashinfer (sm_90); the trtllm_mha path is Blackwell-only. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE - -SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-10} - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -nvidia-smi - -# ---- Resolve traces and install deps ---------------------------------------- -# H100 max_model_len caps at 131k (HBM-bound). The unfiltered with-subagents -# corpus has requests up to ~1M proxy tokens that the server would reject. -# Switch to the 256k-capped variant (470 traces, max in+out <= 256k); even -# at 131k context, the rejection rate is much lower than against the -# unfiltered corpus. -export WEKA_LOADER_OVERRIDE=semianalysis_cc_traces_weka_with_subagents_256k - -resolve_trace_source -install_agentic_deps - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -CACHE_ARGS=() -if require_agentic_kv_offload_backend hicache; then - # HiCache extends RadixAttention, so do not pass --disable-radix-cache. - # Hybrid GDN/Mamba allocates one KV and one Mamba host pool per rank. - REQUESTED_HICACHE_TOTAL_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-$TOTAL_CPU_DRAM_GB}" - if [ "$REQUESTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then - echo "Error: requested HiCache pool ${REQUESTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2 - exit 1 - fi - TOTAL_CPU_DRAM_GB="$REQUESTED_HICACHE_TOTAL_GB" - HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-2}" - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}" - MAX_HICACHE_SIZE_GB=$((TOTAL_CPU_DRAM_GB / TP / HICACHE_HOST_POOL_COUNT)) - HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}" - if [ "$HICACHE_SIZE_GB" -gt "$MAX_HICACHE_SIZE_GB" ]; then - echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB exceeds configured per-pool limit $MAX_HICACHE_SIZE_GB" >&2 - exit 1 - fi - if [ "$HICACHE_SIZE_GB" -lt 1 ]; then - echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB from TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB, TP=$TP, HICACHE_HOST_POOL_COUNT=$HICACHE_HOST_POOL_COUNT" >&2 - exit 1 - fi - echo "HiCache CPU pool: ${HICACHE_SIZE_GB} GB per rank per host pool across TP=${TP}, host_pool_count=${HICACHE_HOST_POOL_COUNT}" - CACHE_ARGS=( - --page-size 64 - --enable-hierarchical-cache - --hicache-size "$HICACHE_SIZE_GB" - --hicache-io-backend kernel - --hicache-mem-layout page_first - --hicache-write-policy "$HICACHE_WRITE_POLICY" - ) -fi - -echo "Starting SGLang server..." -export PYTHONNOUSERSITE=1 - -SGLANG_MULTI_TOKENIZER=/sgl-workspace/sglang/python/sglang/srt/managers/multi_tokenizer_mixin.py -if ! sed -n '/elif isinstance(output, BatchStrOutput):/,/input_token_logprobs_val=_extract_field_by_index/p' "$SGLANG_MULTI_TOKENIZER" \ - | grep -q 'cached_tokens_details=_extract_field_by_index'; then - sed -i '/elif isinstance(output, BatchStrOutput):/,/input_token_logprobs_val=_extract_field_by_index/ { - /cached_tokens=_extract_field_by_index(output, "cached_tokens", i),/a\ - cached_tokens_details=_extract_field_by_index(\ - output, "cached_tokens_details", i\ - ), - }' "$SGLANG_MULTI_TOKENIZER" -fi - -{ set +x; } 2>/dev/null -SGLANG_CMD=( - python3 -m sglang.launch_server - --model-path="$MODEL_PATH" --served-model-name="$MODEL" - --host=0.0.0.0 - --port="$PORT" - --served-model-name "Qwen/Qwen3.5-397B-A17B-FP8" - --trust-remote-code - --tensor-parallel-size="$TP" - --data-parallel-size=1 - --expert-parallel-size="$EP_SIZE" - --quantization fp8 - --kv-cache-dtype fp8_e4m3 - --mamba-ssm-dtype bfloat16 - --attention-backend flashinfer - --enable-flashinfer-allreduce-fusion - # --cuda-graph-max-bs "$CONC" - # --max-running-requests "$CONC" - # --max-prefill-tokens 8192 - # --chunked-prefill-size 8192 - --mem-fraction-static 0.75 - --stream-interval 50 - --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" - --tokenizer-worker-num 6 - --tokenizer-path "$MODEL" - --enable-metrics - "${CACHE_ARGS[@]}" -) -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" -"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_h100_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_h100_mtp.sh index 95dae3834f..41527c775d 100755 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_h100_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_h100_mtp.sh @@ -3,8 +3,7 @@ set -euo pipefail set -x # Agentic trace replay benchmark for Qwen3.5 FP8 on H100 using SGLang with MTP -# speculative decoding -- the spec-decoding=mtp variant of -# agentic/qwen3.5_fp8_h100.sh. +# speculative decoding. # # H100 has 80 GB HBM3 (vs B300's 192 GB), so weights + KV fit tighter. # Mem-fraction-static lowered to 0.75 and chunked-prefill-size halved to diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh index 62982a3974..518cd7e17e 100755 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh @@ -7,7 +7,7 @@ set -x # only, per the AgentX policy that new agentic arms ship with speculative # decoding enabled rather than as an STP/MTP A/B (MODELS.md). # -# Structure follows the proven agentic/qwen3.5_fp8_h100.sh replay path +# Structure follows the proven H100 MTP AgentX replay path # (HiCache host-DRAM offload, the multi_tokenizer cached_tokens_details patch, # aiperf-driven trace replay). H200's 141 GB HBM3e is roomier than H100's 80 GB, # so --mem-fraction-static is 0.8 rather than 0.75, matching diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_mi355x.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_mi355x.sh deleted file mode 100755 index 0bcf3cf163..0000000000 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_mi355x.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Qwen3.5 FP8 on MI355X using SGLang. -# -# Required env vars: -# MODEL, TP, CONC, RESULT_DIR - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC RESULT_DIR DURATION EP_SIZE - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -rocm-smi || true -amd-smi || true - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# ---- Start SGLang server ---------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -echo "Starting SGLang server..." -export PYTHONNOUSERSITE=1 - -python3 -m sglang.launch_server \ - --attention-backend triton \ - --model-path "$MODEL_PATH" --served-model-name "$MODEL" \ - --host=0.0.0.0 \ - --port $PORT \ - --tensor-parallel-size $TP \ - --ep-size $EP_SIZE \ - --trust-remote-code \ - --tokenizer-worker-num 6 \ - --enable-aiter-allreduce-fusion \ - --cuda-graph-max-bs $CONC \ - --max-running-requests $CONC \ - --max-prefill-tokens 32768 \ - --scheduler-recv-interval 30 \ - --mem-fraction-static 0.8 \ - --enable-metrics > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/benchmarks/single_node/agentic/qwen3.5_fp8_mi355x_sglang.sh b/benchmarks/single_node/agentic/qwen3.5_fp8_mi355x_sglang.sh deleted file mode 100755 index 3e2a70d003..0000000000 --- a/benchmarks/single_node/agentic/qwen3.5_fp8_mi355x_sglang.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set -x - -# Agentic trace replay benchmark for Qwen3.5 FP8 on MI355X using SGLang. -# -# Required env vars: -# MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR -# -# KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache. - -source "$(dirname "$0")/../../benchmark_lib.sh" - -check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION EP_SIZE - -SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-30} - -if [[ -n "${SLURM_JOB_ID:-}" ]]; then - echo "JOB $SLURM_JOB_ID running on ${SLURMD_NODENAME:-unknown}" -fi - -# `hf download` creates the target dir if missing and is itself idempotent. -# When MODEL_PATH is unset (stand-alone runs), fall back to the HF_HUB_CACHE -# Either way, MODEL_PATH is what the server is launched with. -if [[ -n "${MODEL_PATH:-}" ]]; then - if [[ ! -d "$MODEL_PATH" || -z "$(ls -A "$MODEL_PATH" 2>/dev/null)" ]]; then - hf download "$MODEL" --local-dir "$MODEL_PATH" - fi -else - hf download "$MODEL" - export MODEL_PATH="$MODEL" -fi -rocm-smi || true -amd-smi || true - -# ---- Resolve traces and install deps ---------------------------------------- -resolve_trace_source -install_agentic_deps - -# ---- Server config ---------------------------------------------------------- -SERVER_LOG="$RESULT_DIR/server.log" -mkdir -p "$RESULT_DIR" - -CACHE_ARGS=() -WARMUP_ARGS=() -CUDA_GRAPH_MAX_BS="$CONC" -if require_agentic_kv_offload_backend hicache; then - # Qwen3.5 allocates one KV and one Mamba host pool per TP rank. - REQUESTED_HICACHE_TOTAL_GB="${HICACHE_TOTAL_CPU_DRAM_GB:-$TOTAL_CPU_DRAM_GB}" - if [ "$REQUESTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then - echo "Error: requested HiCache pool ${REQUESTED_HICACHE_TOTAL_GB} GB exceeds configured capacity ${TOTAL_CPU_DRAM_GB} GB" >&2 - exit 1 - fi - TOTAL_CPU_DRAM_GB="$REQUESTED_HICACHE_TOTAL_GB" - HICACHE_HOST_POOL_COUNT="${HICACHE_HOST_POOL_COUNT:-2}" - HICACHE_MAX_SIZE_GB_PER_RANK_POOL="${HICACHE_MAX_SIZE_GB_PER_RANK_POOL:-${HICACHE_MAX_SIZE_GB_PER_RANK:-180}}" - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}" - # Qwen3.5's hybrid Mamba path runs SGLang's no_buffer scheduler on MI355X, - # which requires page_size=1. The kernel/page_first HiCache transfer path - # faults on first prefill in this mode on ROCm, so keep the default on the - # safer direct/layer_first copy path. These remain env-overridable. - HICACHE_PAGE_SIZE="${HICACHE_PAGE_SIZE:-1}" - HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" - HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-layer_first}" - # SGLang --hicache-size is per rank per host pool, while the workflow - # input is a node-total DRAM budget. Divide by TP and the number of - # host pools unless HICACHE_SIZE_GB is set directly for one-off tuning. - MAX_HICACHE_SIZE_GB=$((TOTAL_CPU_DRAM_GB / TP / HICACHE_HOST_POOL_COUNT)) - HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}" - if [ "$HICACHE_SIZE_GB" -gt "$MAX_HICACHE_SIZE_GB" ]; then - echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB exceeds configured per-pool limit $MAX_HICACHE_SIZE_GB" >&2 - exit 1 - fi - if [ "$HICACHE_SIZE_GB" -gt "$HICACHE_MAX_SIZE_GB_PER_RANK_POOL" ]; then - HICACHE_SIZE_GB="$HICACHE_MAX_SIZE_GB_PER_RANK_POOL" - fi - if [ "$HICACHE_SIZE_GB" -lt 1 ]; then - echo "Error: computed HICACHE_SIZE_GB=$HICACHE_SIZE_GB from TOTAL_CPU_DRAM_GB=$TOTAL_CPU_DRAM_GB, TP=$TP, HICACHE_HOST_POOL_COUNT=$HICACHE_HOST_POOL_COUNT" >&2 - exit 1 - fi - echo "HiCache CPU pool: ${HICACHE_SIZE_GB} GB per rank per host pool across TP=${TP}, host_pool_count=${HICACHE_HOST_POOL_COUNT}" - CACHE_ARGS=( - --page-size "$HICACHE_PAGE_SIZE" - --enable-hierarchical-cache - --hicache-size "$HICACHE_SIZE_GB" - --hicache-io-backend "$HICACHE_IO_BACKEND" - --hicache-mem-layout "$HICACHE_MEM_LAYOUT" - --hicache-write-policy "$HICACHE_WRITE_POLICY" - ) - # HiCache startup reaches API readiness, but SGLang's internal warmup - # request has timed out after 600s on this Qwen MI355X path. Let aiperf - # own benchmark traffic instead of blocking server readiness on it. - WARMUP_ARGS=(--skip-server-warmup) - # Keep request concurrency as the swept variable, but do not force HiCache - # runs to capture ROCm graphs at every high concurrency point. - HICACHE_CUDA_GRAPH_MAX_BS="${HICACHE_CUDA_GRAPH_MAX_BS:-16}" - if [ "$HICACHE_CUDA_GRAPH_MAX_BS" -lt "$CUDA_GRAPH_MAX_BS" ]; then - CUDA_GRAPH_MAX_BS="$HICACHE_CUDA_GRAPH_MAX_BS" - fi -fi - -echo "Starting SGLang server..." -export PYTHONNOUSERSITE=1 - -{ set +x; } 2>/dev/null -SGLANG_CMD=( - python3 -m sglang.launch_server - --attention-backend triton - --model-path "$MODEL_PATH" --served-model-name "$MODEL" - --host=0.0.0.0 - --port "$PORT" - --tensor-parallel-size "$TP" - --ep-size "$EP_SIZE" - --trust-remote-code - --tokenizer-worker-num 6 - --enable-aiter-allreduce-fusion - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" - --max-running-requests "$CONC" - --max-prefill-tokens 32768 - --scheduler-recv-interval "$SCHEDULER_RECV_INTERVAL" - --mem-fraction-static 0.8 - --enable-metrics - "${CACHE_ARGS[@]}" - "${WARMUP_ARGS[@]}" -) -printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" -printf '\n' | tee -a "$RESULT_DIR/sglang_command.txt" -"${SGLANG_CMD[@]}" > "$SERVER_LOG" 2>&1 & -SERVER_PID=$! -echo "Server PID: $SERVER_PID" - -wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" - -if [ "${EVAL_ONLY}" = "true" ]; then - run_eval --port "$PORT" -else - build_replay_cmd "$RESULT_DIR" - run_agentic_replay_and_write_outputs "$RESULT_DIR" -fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 9bc224b161..3031fa0a2b 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -215,19 +215,6 @@ qwen3.5-fp8-mi355x-sglang-mtp: search-space: - { tp: 4, ep: 1, conc-start: 4, conc-end: 256, spec-decoding: mtp } -qwen3.5-fp8-mi355x-sglang-agentic: - image: lmsysorg/sglang-rocm:v0.5.10rc0-rocm720-mi35x-20260414 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang - multinode: false - scenarios: - agentic-coding: - - search-space: - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } - qwen3.5-fp8-mi355x-atom: image: rocm/atom:rocm7.2.3_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom20260511 model: Qwen/Qwen3.5-397B-A17B-FP8 @@ -1212,26 +1199,6 @@ qwen3.5-fp8-mi325x-sglang-mtp: search-space: - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } -qwen3.5-fp8-mi355x-sglang-agentic-hicache: - image: lmsysorg/sglang-rocm:v0.5.12-rocm720-mi35x-20260521 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: cluster:mi355x-amds - precision: fp8 - framework: sglang - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } - - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 32, 48, 64] } - -# MTP twin of dsv4-fp4-mi355x-vllm-agentic. The topologies, KV-offload backends, -# and concurrency points are identical; speculative decoding is enabled on the -# current immutable ROCm nightly because the base config's retired nightly tag -# is no longer available from Docker Hub. Throughput uses the three-token golden -# synthetic acceptance length, while eval-only runs retain real verification. dsv4-fp4-mi355x-vllm-agentic-mtp: image: vllm/vllm-openai-rocm:nightly-821717118fc26667dd474b9b0ab81d29259dfc5c model: deepseek-ai/DeepSeek-V4-Pro @@ -1433,30 +1400,6 @@ dsv4-fp4-mi355x-atom-disagg: additional-settings: - "DECODE_NODES=1" # 1P1D TP8 -minimaxm3-fp8-mi300x-vllm-agentic: - image: vllm/vllm-openai-rocm:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: cluster:mi300x-amds - precision: fp8 - framework: vllm - multinode: false - scenarios: - agentic-coding: - # FP8 KV cache is 29,952 bytes/token/GPU for TP/TEP. Projecting the - # measured BF16 cache budget gives about 2.76M active tokens on MI300X; - # the June-21 service-time-weighted request is 269k tokens, so sample every - # integer around the expected conc-10 cliff rather than geometric jumps. - - dram-utilization: 0.80 - search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - - { tp: 8, ep: 8, kv-offloading: none, conc-list: [2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - -# GLM-5.2 FP8 full-context AgentX refresh on MI325X. This preserves the TP8 -# GPU-resident-KV c1/c2/c3/c4/c5/c6/c8 curve from Actions run 29657732517 -# and enables EAGLE MTP with the committed thinking-on golden AL. glm5.2-fp8-mi325x-sglang-agentic-mtp: image: lmsysorg/sglang:v0.5.16-rocm720-mi30x model: zai-org/GLM-5.2-FP8 @@ -1471,46 +1414,6 @@ glm5.2-fp8-mi325x-sglang-agentic-mtp: search-space: - { tp: 8, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 3, 4, 5, 6, 8] } -minimaxm3-fp8-mi325x-vllm-agentic: - image: vllm/vllm-openai-rocm:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: cluster:mi325x-amds - precision: fp8 - framework: vllm - multinode: false - scenarios: - agentic-coding: - # Measured FP8 capacities: TP4 1.66M tokens, TP8 4.93M tokens, and DEP8 - # 12.69M aggregate tokens. With the June-21 service-time-weighted 269k - # active request, their expected cliffs are conc 6, 18, and 47. Use dense - # bands around each cliff; Mooncake rows overlap those bands because host - # storage improves prefix retention but does not hold active decode KV. - - dram-utilization: 0.80 - search-space: - - { tp: 4, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - - { tp: 4, ep: 4, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - - { tp: 8, ep: 8, kv-offloading: none, conc-list: [2, 4, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [16, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80], router: { name: vllm-router, version: "0.1.14" } } - - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 32] } - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [24, 32, 36, 40, 44, 48, 52, 56, 60, 64, 72, 80, 96], router: { name: vllm-router, version: "0.1.14" } } - -minimaxm3-fp4-mi355x-vllm-agentic: - image: vllm/vllm-openai-rocm:nightly-dcfebf93f4eccf30f71872283331eee757915daf - model: amd/MiniMax-M3-MXFP4 - model-prefix: minimaxm3 - runner: cluster:mi355x-amds - precision: fp4 - framework: vllm - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 4, 8, 16] } - dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp: image: lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719 model: deepseek-ai/DeepSeek-V4-Pro @@ -1633,4 +1536,3 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: search-space: - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 10, 12, 16], spec-decoding: mtp } - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4], spec-decoding: mtp } - diff --git a/configs/deprecated/amd-kimik2.5-agentic-master.yaml b/configs/deprecated/amd-kimik2.5-agentic-master.yaml deleted file mode 100644 index 641bb41f1b..0000000000 --- a/configs/deprecated/amd-kimik2.5-agentic-master.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# Deprecated Kimi-K2.5/2.6/2.7-Code agentic-coding entries archived from -# amd-master.yaml. Deprecated after Monday, August 3, 2026 (see the -# Deprecation Notice in MODELS.md). Single-turn 8k1k stays active until -# August 6, 2026. -# Removed from the active master config so sweep generation no longer selects them. - - -kimik2.5-fp4-mi355x-vllm-agentic: - image: vllm/vllm-openai-rocm:v0.22.0 - model: amd/Kimi-K2.5-MXFP4 - model-prefix: kimik2.5 - runner: cluster:mi355x-amds - precision: fp4 - framework: vllm - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 24, 32, 40, 48] } - # DRAM offload only above the KV cliff. Lower concurrencies fit - # entirely on-GPU, so paying the offload-path overhead there would - # just slow them down without measuring anything new. - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-native }, conc-list: [32, 40, 48, 56] } - # TP=4 probe: half-node layout doubles per-GPU weight footprint - # (~62 GB on MI355X's 288 GB HBM, plenty of headroom). Restrict to - # cliff-region concurrencies on both offload modes so we can directly - # compare TP=4 vs TP=8 at the same conc points. - - { tp: 4, kv-offloading: none, conc-list: [16, 24, 32, 40] } - - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-native }, conc-list: [16, 24, 32, 40] } diff --git a/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml b/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml index a92bea3aa6..5a2e6d62e5 100644 --- a/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml +++ b/configs/deprecated/amd-minimaxm2.5-m2.7-master.yaml @@ -29,29 +29,6 @@ minimaxm2.5-fp8-mi355x-vllm: # the original minimaxm2.5-fp8-mi355x-vllm entry is left identical to origin/main so # its fixed-seq-len sweep is unaffected. # - image: 'vllm/vllm-openai-rocm:v0.19.0' -> 'vllm/vllm-openai-rocm:nightly-51f22dcfd068fe8f1e3192da2a1e825b930223cf' -minimaxm2.5-fp8-mi355x-vllm-agentic: - # Nightly carrying vllm-project/vllm@20cac26b ("[Bug fix][KV Connector] - # add cpu_offload_blocks > 0 check before maybe_run_layer_kv_offload"), - # which enables SimpleCPUOffloadConnector on ROCm. Required for the - # cpu-offload sweep points to use the same offload path as the NVIDIA - # agentic-coding configs. - image: vllm/vllm-openai-rocm:nightly-51f22dcfd068fe8f1e3192da2a1e825b930223cf - model: MiniMaxAI/MiniMax-M2.5 - model-prefix: minimaxm2.5 - runner: mi355x - precision: fp8 - framework: vllm - multinode: false - scenarios: - agentic-coding: - # MI355X tp=4 ep=4: compute ceiling ~60 (empirical), KV cliff ~91 (analytical). - # Compute saturates first; cpu offload likely won't help, but worth confirming. - # AMD uses native OffloadingConnector (NOT SimpleCPUOffloadConnector). - - duration: 1800 - search-space: - - { tp: 4, ep: 4, offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 48, 56, 64, 72, 96] } - - { tp: 4, ep: 4, offloading: cpu, conc-list: [48, 56, 64, 72, 96] } - minimaxm2.5-fp8-mi355x-atom: image: rocm/atom:rocm7.2.2_ubuntu24.04_py3.12_pytorch_release_2.10.0_atom0.1.2.post model: MiniMaxAI/MiniMax-M2.5 @@ -146,25 +123,6 @@ minimaxm2.5-fp8-mi300x-vllm: # the original minimaxm2.5-fp8-mi300x-vllm entry is left identical to origin/main so # its fixed-seq-len sweep is unaffected. # - image: 'vllm/vllm-openai-rocm:v0.16.0' -> 'vllm/vllm-openai-rocm:nightly-51f22dcfd068fe8f1e3192da2a1e825b930223cf' -minimaxm2.5-fp8-mi300x-vllm-agentic: - # Nightly carrying vllm-project/vllm@20cac26b — see mi355x config above. - image: vllm/vllm-openai-rocm:nightly-51f22dcfd068fe8f1e3192da2a1e825b930223cf - model: MiniMaxAI/MiniMax-M2.5 - model-prefix: minimaxm2.5 - runner: mi300x - precision: fp8 - framework: vllm - multinode: false - scenarios: - agentic-coding: - # MI300X tp=4: compute ceiling ~25 (estimated, between H100 and H200); - # KV cliff ~52. Compute saturates first. - # AMD uses native OffloadingConnector (NOT SimpleCPUOffloadConnector). - - duration: 1800 - search-space: - - { tp: 4, offloading: none, conc-list: [1, 2, 4, 8, 16, 20, 24, 28, 32, 40, 48] } - - { tp: 4, offloading: cpu, conc-list: [16, 20, 24, 28, 32] } - minimaxm2.5-fp8-mi325x-vllm: image: vllm/vllm-openai-rocm:v0.22.0 model: MiniMaxAI/MiniMax-M2.5 @@ -190,26 +148,6 @@ minimaxm2.5-fp8-mi325x-vllm: # the original minimaxm2.5-fp8-mi325x-vllm entry is left identical to origin/main so # its fixed-seq-len sweep is unaffected. # - image: 'vllm/vllm-openai-rocm:v0.18.0' -> 'vllm/vllm-openai-rocm:nightly-51f22dcfd068fe8f1e3192da2a1e825b930223cf' -minimaxm2.5-fp8-mi325x-vllm-agentic: - # Nightly carrying vllm-project/vllm@20cac26b — see mi355x config above. - image: vllm/vllm-openai-rocm:nightly-51f22dcfd068fe8f1e3192da2a1e825b930223cf - model: MiniMaxAI/MiniMax-M2.5 - model-prefix: minimaxm2.5 - runner: mi325x - precision: fp8 - framework: vllm - multinode: false - scenarios: - agentic-coding: - # MI325X tp=4: cloned from MI300X recipe (slightly faster compute, - # similar HBM profile). Compute saturates first; cpu-offload window - # exercises the SimpleCPUOffloadConnector path enabled by the rocm - # nightly. Mirror MI300X conc grid for cross-vendor comparability. - - duration: 1800 - search-space: - - { tp: 4, offloading: none, conc-list: [1, 2, 4, 8, 16, 20, 24, 28, 32, 40, 48] } - - { tp: 4, offloading: cpu, conc-list: [16, 20, 24, 28, 32] } - minimaxm2.5-fp8-mi355x-vllm-disagg: image: vllm/vllm-openai-rocm:nightly-a6682d1d259cca69a9ae737ea5608fbbe7520031 model: MiniMaxAI/MiniMax-M2.5 diff --git a/configs/deprecated/nvidia-kimik2.5-agentic-master.yaml b/configs/deprecated/nvidia-kimik2.5-agentic-master.yaml deleted file mode 100644 index 4be8befaa5..0000000000 --- a/configs/deprecated/nvidia-kimik2.5-agentic-master.yaml +++ /dev/null @@ -1,111 +0,0 @@ -# Deprecated Kimi-K2.5/2.6/2.7-Code agentic-coding entries archived from -# nvidia-master.yaml. Deprecated after Monday, August 3, 2026 (see the -# Deprecation Notice in MODELS.md). Single-turn 8k1k stays active until -# August 6, 2026. -# Removed from the active master config so sweep generation no longer selects them. - - -kimik2.5-int4-b200-vllm-agentic: - image: vllm/vllm-openai:v0.22.0 - model: moonshotai/Kimi-K2.5 - model-prefix: kimik2.5 - runner: cluster:b200-dgxc - precision: int4 - framework: vllm - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [32, 64, 96, 128] } - -kimik2.5-int4-h200-vllm-agentic: - image: vllm/vllm-openai:v0.22.0 - model: moonshotai/Kimi-K2.5 - model-prefix: kimik2.5 - runner: cluster:h200-dgxc - precision: int4 - framework: vllm - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [6, 7, 8, 9, 10, 11, 12, 13, 14] } - -kimik2.5-fp4-b300-vllm-agentic: - # v0.20.2 (cu129) lacks the flashinfer kernels for B300's reported SM - # (sm_12x); workers hit "Only SM 10.x and 11.x are supported" in the - # trtllm_fp4_block_scale_moe path. v0.20.0-cu130 is the Blackwell-targeted - # build that has the full sm_10x/sm_11x/sm_12x kernel set and is what the - # INT4 B300 sister already uses successfully. - image: vllm/vllm-openai:v0.22.0 - model: nvidia/Kimi-K2.5-NVFP4 - model-prefix: kimik2.5 - runner: cluster:b300-nv - precision: fp4 - framework: vllm - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 40, 48, 56, 64] } - - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2, 4, 8, 16, 32, 40, 48, 56, 64] } - -kimik2.5-fp4-b300-vllm-agentic-mtp: - image: vllm/vllm-openai:nightly-94c0ef300180f8fd1071d9cbe7270a8348155f94 - model: Kimi-K2.6-NVFP4 - model-prefix: kimik2.5 - runner: cluster:b300-nv - precision: fp4 - framework: vllm - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 8, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1] } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [2, 4, 8] } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [8, 16, 32] } - - { tp: 4, ep: 1, spec-decoding: mtp, dcp-size: 4, kv-offloading: none, conc-list: [32, 64, 80, 96, 112, 128] } - - { tp: 4, ep: 1, spec-decoding: mtp, dcp-size: 4, kv-offloading: dram, kv-offload-backend: { name: native }, conc-list: [64, 80, 96, 112, 128, 144, 160] } - -kimik2.5-fp4-b200-vllm-agentic-lmcache: - image: vllm/vllm-openai:v0.22.0 - model: nvidia/Kimi-K2.5-NVFP4 - model-prefix: kimik2.5 - runner: cluster:b200-dgxc - precision: fp4 - framework: vllm - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 24] } - - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.1" }, conc-list: [16, 24, 32, 36] } - - { tp: 4, ep: 1, kv-offloading: none, conc-list: [8, 12, 14, 16, 18, 20] } - - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.1" }, conc-list: [12, 14, 16, 18, 20, 22, 24, 32] } - -# Left behind by #2493: this key stayed in nvidia-master.yaml while its script -# moved to benchmarks/single_node/agentic/deprecated/, so the config pointed at -# a path that no longer existed. Archived here with its siblings. -kimik2.5-int4-h100-vllm: - image: vllm/vllm-openai:v0.22.0 - model: moonshotai/Kimi-K2.5 - model-prefix: kimik2.5 - runner: cluster:h100-dgxc - precision: int4 - framework: vllm - multinode: false - scenarios: - # H100 has 80 GB HBM per GPU (smallest in this set); the KV cliff arrives - # early. Sweep saturates conc=20 to keep total HBM headroom. - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 16, 20] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2, 4, 8, 12, 16, 20] } diff --git a/configs/deprecated/nvidia-minimaxm2.5-m2.7-master.yaml b/configs/deprecated/nvidia-minimaxm2.5-m2.7-master.yaml index 50be2d4e8d..cbeef276f7 100644 --- a/configs/deprecated/nvidia-minimaxm2.5-m2.7-master.yaml +++ b/configs/deprecated/nvidia-minimaxm2.5-m2.7-master.yaml @@ -32,28 +32,6 @@ minimaxm2.5-fp8-b200-vllm: # its fixed-seq-len sweep is unaffected. # - image: 'vllm/vllm-openai:v0.19.0-cu130' -> 'vllm/vllm-openai:v0.19.1' # - runner: 'b200' -> 'b200-dgxc' -minimaxm2.5-fp8-b200-vllm-agentic: - image: vllm/vllm-openai:v0.19.1 - model: MiniMaxAI/MiniMax-M2.5 - model-prefix: minimaxm2.5 - runner: b200-dgxc - precision: fp8 - framework: vllm - multinode: false - scenarios: - agentic-coding: - # B200 tp=4: compute ceiling ~50 (empirical), KV cliff ~48 (analytical). - # Push none past the KV cliff (96, 128) to make the no-offload throughput - # collapse visible; cpu range overlaps fully for same-conc comparison. - - duration: 1800 - search-space: - - { tp: 4, offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 48, 56, 64, 96, 128] } - - { tp: 4, offloading: cpu, conc-list: [48, 56, 64, 96, 128] } - - # NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/MiniMax/MiniMax-M2.html - # does not have a B300-specific recipe, so this config reuses the existing - # MiniMax-M2.5 FP8 B200 vLLM recipe as-is until B300-specific tuning is available. - minimaxm2.5-fp8-b300-vllm: image: vllm/vllm-openai:v0.21.0 model: MiniMaxAI/MiniMax-M2.5 @@ -82,27 +60,6 @@ minimaxm2.5-fp8-b300-vllm: # the original minimaxm2.5-fp8-b300-vllm entry is left identical to origin/main so # its fixed-seq-len sweep is unaffected. # - image: 'vllm/vllm-openai:v0.19.0-cu130' -> 'vllm/vllm-openai:v0.19.1' -minimaxm2.5-fp8-b300-vllm-agentic: - image: vllm/vllm-openai:v0.19.1 - model: MiniMaxAI/MiniMax-M2.5 - model-prefix: minimaxm2.5 - runner: b300 - precision: fp8 - framework: vllm - multinode: false - scenarios: - agentic-coding: - # B300 tp=4: compute ceiling ~60 (empirical), KV cliff ~85 (analytical). - # Push none past the KV cliff (96, 128, 192) so the no-offload throughput - # collapse is visible; cpu range overlaps fully so each high-conc point - # has a same-conc no-offload counterpart for direct comparison. - # Dense sampling between 96 and 128 (step=4) to resolve the sharp dropoff - # observed in v6 cpu data right past conc=96. - - duration: 1800 - search-space: - - { tp: 4, offloading: none, conc-list: [1, 2, 4, 8, 16, 32, 48, 64, 96, 100, 104, 108, 112, 116, 120, 124, 128, 192] } - - { tp: 4, offloading: cpu, conc-list: [48, 64, 96, 100, 104, 108, 112, 116, 120, 124, 128, 192] } - minimaxm2.5-fp4-b200-vllm: image: vllm/vllm-openai:v0.22.0 model: nvidia/MiniMax-M2.5-NVFP4 @@ -140,24 +97,6 @@ minimaxm2.5-fp4-b200-vllm: # PR adds an agentic-coding scenarios block that differs from main # (either main had none or had a different conc/offload sweep). # The original minimaxm2.5-fp4-b200-vllm entry stays byte-identical to origin/main. -minimaxm2.5-fp4-b200-vllm-agentic: - image: vllm/vllm-openai:v0.19.0-cu130 - model: nvidia/MiniMax-M2.5-NVFP4 - model-prefix: minimaxm2.5 - runner: b200 - precision: fp4 - framework: vllm - multinode: false - scenarios: - agentic-coding: - - duration: 1800 - search-space: - - { tp: 4, offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } - - # NOTE: At the time of submission, https://docs.vllm.ai/projects/recipes/en/latest/MiniMax/MiniMax-M2.html - # does not have a B300-specific recipe, so this config reuses the existing - # MiniMax-M2.5 FP4 B200 vLLM recipe as-is until B300-specific tuning is available. - minimaxm2.5-fp4-b200-trt: image: nvcr.io#nvidia/tensorrt-llm/release:1.3.0rc18 model: nvidia/MiniMax-M2.5-NVFP4 @@ -265,24 +204,6 @@ minimaxm2.5-fp8-h100-vllm: # PR adds an agentic-coding scenarios block that differs from main # (either main had none or had a different conc/offload sweep). # The original minimaxm2.5-fp8-h100-vllm entry stays byte-identical to origin/main. -minimaxm2.5-fp8-h100-vllm-agentic: - image: vllm/vllm-openai:v0.20.2 - model: MiniMaxAI/MiniMax-M2.5 - model-prefix: minimaxm2.5 - runner: h100 - precision: fp8 - framework: vllm - multinode: false - scenarios: - agentic-coding: - # H100 tp=4 ep=4: compute ceiling ~10 (empirical), KV cliff ~6 (analytical). - # Best cpu-offload demo SKU — 4-conc-point window between cliffs. - # Dense sampling 4-12 covers both cliffs; conc 16 confirms compute plateau. - - duration: 1800 - search-space: - - { tp: 4, ep: 4, offloading: none, conc-list: [1, 2, 4, 5, 6, 7, 8, 10, 12, 16] } - - { tp: 4, ep: 4, offloading: cpu, conc-list: [5, 6, 7, 8, 10, 12] } - minimaxm2.5-fp8-h200-vllm: image: vllm/vllm-openai:v0.22.0 model: MiniMaxAI/MiniMax-M2.5 @@ -307,23 +228,6 @@ minimaxm2.5-fp8-h200-vllm: # PR adds an agentic-coding scenarios block that differs from main # (either main had none or had a different conc/offload sweep). # The original minimaxm2.5-fp8-h200-vllm entry stays byte-identical to origin/main. -minimaxm2.5-fp8-h200-vllm-agentic: - image: vllm/vllm-openai:v0.20.2 - model: MiniMaxAI/MiniMax-M2.5 - model-prefix: minimaxm2.5 - runner: h200 - precision: fp8 - framework: vllm - multinode: false - scenarios: - agentic-coding: - # H200 tp=4: compute ceiling ~35 (empirical), KV cliff ~29 (analytical). - # cpu offload window conc 29-35 — dense sampling 24-40 captures both cliffs. - - duration: 1800 - search-space: - - { tp: 4, offloading: none, conc-list: [1, 2, 4, 8, 16, 24, 28, 32, 36, 48] } - - { tp: 4, offloading: cpu, conc-list: [24, 28, 32, 36, 40, 48] } - minimaxm2.5-fp4-gb300-dynamo-vllm: image: vllm/vllm-openai:v0.20.1 model: nvidia/MiniMax-M2.5-NVFP4 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index f916be18dc..da5df2ab47 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1185,19 +1185,6 @@ qwen3.5-fp8-b200-sglang: - { tp: 8, conc-start: 4, conc-end: 4 } - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } -qwen3.5-fp8-b200-sglang-agentic: - image: lmsysorg/sglang:nightly-dev-20260422-de962f32 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: cluster:b200-dgxc - precision: fp8 - framework: sglang - multinode: false - scenarios: - agentic-coding: - - search-space: - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } - qwen3.5-fp8-b200-sglang-agentic-mtp: image: lmsysorg/sglang:v0.5.16-cu130 model: Qwen/Qwen3.5-397B-A17B-FP8 @@ -1472,37 +1459,6 @@ dsr1-fp8-b300-sglang-mtp: search-space: - { tp: 8, ep: 1, conc-start: 1, conc-end: 512, spec-decoding: mtp } -kimik3-fp4-b300-vllm-agentic: - # Day-zero Kimi-K3 recipe. `vllm/vllm-openai:kimi-k3` is the pre-release - # Kimi-K3 build (vLLM 0.1.dev19262+gb6bbf29dd, pushed 2026-07-27); K3 ships as - # the out-of-tree `vllm.models.kimi_k3` plugin package (DSpark MLA + KDA - # kernels), so a generic vllm-openai release cannot serve this checkpoint. - image: vllm/vllm-openai:kimi-k3 - model: moonshotai/Kimi-K3 - model-prefix: kimik3 - runner: cluster:b300-nv - precision: fp4 - framework: vllm - multinode: false - scenarios: - # Agentic-coding only: no fixed-seq-len (1k1k / 8k1k) arms for this recipe. - agentic-coding: - # 0.63 resolves to ~220 GiB of host DRAM per rank across the 8 TP ranks on - # cluster:b300-nv (total-cpu-dram-gb 1889), which is what the - # SimpleCPUOffloadConnector pool is sized to in the script. - - dram-utilization: 0.63 - search-space: - # TP8 is the only single-node layout that fits: the MXFP4 checkpoint is - # ~1.5 TB (~188 GB/GPU across 8 B300s), and TP4 would need ~375 GB/GPU - # against 288 GB of HBM. The conc ceiling is set by the ~70 GB/GPU left - # for MLA KV after weights at gpu-memory-utilization 0.90. - # Both arms share one ladder so GPU-resident and DRAM-offload are directly - # comparable at equal concurrency (same shape as the kimik2.5 B300 sister). - # TP8 GPU-resident - - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 24] } - # TP8 SimpleCPUOffload (host DRAM) - - { tp: 8, ep: 1, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [1, 2, 4, 8, 16, 24] } - kimik3-fp4-b300-vllm-agentic-dspark: # DSpark speculative decoding on the Inferact/Kimi-K3-DSpark draft head, at # level 2 with probabilistic drafting and synthetic acceptance pinned to the @@ -1651,22 +1607,6 @@ dsv4-fp8-h200-vllm-mtp: # Pinned to the h200-dgxc-slurm runner pool because the deepseek-v4-hopper # image needs the /ix mount layout that only launch_h200-dgxc-slurm.sh sets up. -dsv4-fp8-h200-vllm-agentic: - image: vllm/vllm-openai:v0.22.0 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:h200-dgxc - precision: fp8 - framework: vllm - multinode: false - scenarios: - agentic-coding: - - search-space: - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, conc-list: [1, 2, 4, 8, 16] } - -# MTP variant of dsv4-fp8-h200-sglang. Mirrors the non-MTP recipe (same image, -# runner pool, search space) and adds EAGLE speculative decoding via -# --speculative-algorithm EAGLE with the (3,1,4) chain matching dsv4-fp4-b300-sglang-mtp. dsv4-fp8-h200-sglang: image: lmsysorg/sglang:deepseek-v4-hopper@sha256:1bf5d508ab110cc0fe1659a5f21d1be02a7f0d7ca8f58cea7e7f4e11f6ae208f model: deepseek-ai/DeepSeek-V4-Pro @@ -6863,21 +6803,6 @@ qwen3.5-fp4-gb300-dynamo-sglang-agentic-disagg: dp-attn: false # ---------- 1k1k high-throughput (wide-EP decode, EAGLE MTP) ---------- -qwen3.5-fp8-b300-sglang-agentic-hicache: - image: lmsysorg/sglang:nightly-dev-cu13-20260520-425dffbd - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: cluster:b300-nv - precision: fp8 - framework: sglang - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 4, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 8, 16, 32] } - - { tp: 4, ep: 1, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 32, 48, 64] } - qwen3.5-fp8-b300-sglang-agentic-mtp: image: lmsysorg/sglang:v0.5.16-cu130 model: Qwen/Qwen3.5-397B-A17B-FP8 @@ -6910,111 +6835,6 @@ qwen3.5-fp4-b300-sglang-agentic-mtp: - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32] } - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [32, 34, 36, 38, 40, 44, 48, 52, 56] } -# CONC range conservative for H100's 80 GB HBM3 under the long-ISL with- -# subagents corpus. hicache arm capped at conc 16 since high-conc + hicache -# tends to flake on first runs and conc 16 covers the cliff. The bench script -# sets WEKA_LOADER_OVERRIDE to the 256k-capped corpus variant. -dsv4-fp4-gb300-dynamo-vllm-agentic: - image: vllm/vllm-openai:v0.21.0-ubuntu2404 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:gb300-nv - precision: fp4 - framework: dynamo-vllm - router: { name: dynamo-router, version: "1.2.0.dev20260426" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - agentic-coding: - - search-space: - # Low-latency: same 1p6d shape as the mid tier but at much lower conc - # (32 vs 192). 32/6 ≈ 5 seqs per decode worker — well below saturation, - # so each request gets ~6× the per-request decode compute it would get - # at conc=192. Reuses the 1p6d recipe; no separate recipe file needed. - - spec-decoding: none - conc-list: [32] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tp4-agentic.yaml" - decode: - num-worker: 6 - tp: 4 - ep: 1 - dp-attn: false - # Mid: 1 prefill (DEP=4) + 6 decode (TP=4). 7 nodes / 28 GPUs. - # Mirrors fixed-seq-len conc=192 entry. - - spec-decoding: none - conc-list: [192] - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p6d-dep4-tp4-agentic.yaml" - decode: - num-worker: 6 - tp: 4 - ep: 1 - dp-attn: false - # High-throughput: 4 prefill (DEP=4 each) + 1 decode (DEP=8). 6 nodes / - # 24 GPUs. Smallest 4096-class shape in fixed-seq-len; deep_gemm_mega_moe - # on both sides. Mirrors fixed-seq-len conc=4096 entry (4p1d variant). - - spec-decoding: none - conc-list: [4096] - prefill: - num-worker: 4 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-4p1d-dep4-dep8-24-c4096-agentic.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - -qwen3.5-fp8-h100-sglang-agentic: - image: lmsysorg/sglang:v0.5.12-cu130 - model: Qwen/Qwen3.5-397B-A17B-FP8 - model-prefix: qwen3.5 - runner: cluster:h100-dgxc - precision: fp8 - framework: sglang - multinode: false - scenarios: - agentic-coding: - - dram-utilization: 0.80 - search-space: - - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 14, 16] } - - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [12, 14, 16, 20, 24, 28, 32, 42] } - -# MTP speculative-decoding (spec-decoding: mtp) variant of -# qwen3.5-fp8-h100-sglang-agentic: same TP8/EP8 GPU-resident and HiCache arms, -# plus SGLang EAGLE MTP (num-steps 3, eagle-topk 1, 4 draft tokens = 3 -# speculative tokens) with simulated acceptance pinned to the golden AL 3.39 -# (golden_al_distribution/qwen3.5_mtp.yaml, thinking_on, K=3) -- the same value -# the GB300 Qwen3.5 AgentX srt-slurm recipes use. Image moves to -# lmsysorg/sglang:v0.5.16-cu130 because SGLANG_SIMULATE_ACC_TOKEN_MODE only -# exists from v0.5.16; v0.5.14 is the version the fixed-seq-len H100 MTP entry -# already runs. Conc lists are trimmed at the top end for the draft head's KV. - -# First Qwen3.5 AgentX recipe on H200, shipped spec-decode-only per the AgentX -# policy that new agentic arms enable speculative decoding rather than running a -# separate STP baseline (MODELS.md). SGLang EAGLE MTP (num-steps 3, eagle-topk 1, -# 4 draft tokens = 3 speculative tokens) with simulated acceptance pinned to the -# golden AL 3.39 (golden_al_distribution/qwen3.5_mtp.yaml, thinking_on, K=3). -# Image is lmsysorg/sglang:v0.5.16-cu130: SGLANG_SIMULATE_ACC_TOKEN_MODE only -# exists from v0.5.16, and v0.5.14 is what the fixed-seq-len H200 MTP entry runs. -# Search space is the H100 AgentX shape widened for H200's 141 GB: the -# fixed-seq-len H200 MTP entry already runs conc 4-128 at TP8/EP8 versus H100's -# 4-32, so the GPU-resident arm extends to 24 and HiCache to 48. qwen3.5-fp8-h200-sglang-agentic-mtp: image: lmsysorg/sglang:v0.5.16-cu130 model: Qwen/Qwen3.5-397B-A17B-FP8 @@ -7086,48 +6906,6 @@ qwen3.5-fp4-b200-trt-mtp: - { tp: 8, ep: 8, spec-decoding: "mtp", conc-list: [4] } - { tp: 8, ep: 8, dp-attn: true, spec-decoding: "mtp", conc-list: [128, 256, 1024] } -minimaxm3-fp8-h100-vllm-agentic: - image: vllm/vllm-openai:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: cluster:h100-dgxc - precision: fp8 - framework: vllm - multinode: false - scenarios: - agentic-coding: - # FP8 KV cache is 29,952 bytes/token/GPU for TP/TEP: 60 layers of - # sharded FP8 K/V plus 57 layers of BF16 indexer cache. The June-21 trace - # has a 269k-token service-time-weighted active request, placing the H100 - # 1.64M-token HBM cliff near conc 6. Sample every integer through the - # cliff; Mooncake extends prefix retention, not active-request HBM. - - dram-utilization: 0.80 - search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [3, 4, 5, 6, 7, 8, 9, 10, 12, 14, 16] } - -minimaxm3-fp8-h200-vllm-agentic: - image: vllm/vllm-openai:nightly-04c2a8deac44fdb1ca3e2b5ec3e6bf16f3f6a914 - model: MiniMaxAI/MiniMax-M3-MXFP8 - model-prefix: minimaxm3 - runner: cluster:h200-dgxc - precision: fp8 - framework: vllm - multinode: false - scenarios: - agentic-coding: - # The same 29,952-byte FP8 TP/TEP cache layout gives H200 about 2.56M - # active tokens. Against the June-21 service-time-weighted 269k-token - # request, the expected HBM cliff is near conc 10; sample densely from - # 5-14 and retain post-cliff points through 20. - - dram-utilization: 0.80 - search-space: - - { tp: 8, kv-offloading: none, conc-list: [1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - - { tp: 8, ep: 8, kv-offloading: none, conc-list: [2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: mooncake, version: "0.3.11.post1" }, conc-list: [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 18, 20] } - qwen3.5-fp4-b200-sglang-agentic-mtp: image: lmsysorg/sglang:v0.5.16-cu130 model: nvidia/Qwen3.5-397B-A17B-NVFP4 @@ -7216,221 +6994,6 @@ minimaxm3-fp4-b200-vllm-agentic-mtp: search-space: - { tp: 4, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 2, 5, 8, 10, 12, 15, 20] } - { tp: 4, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [20, 30, 40] } -dsv4-fp4-gb200-dynamo-vllm-agentic-3p2d-tep8-tp8: - image: vllm/vllm-openai:v0.23.0 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:gb200-nv - precision: fp4 - framework: dynamo-vllm - router: { name: dynamo-router, version: "1.3.0.dev20260618" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - agentic-coding: - - search-space: - # Ultra-high-interactivity probes below the historical c16 endpoint. - - spec-decoding: none - conc-list: [4, 8] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: none - conc-list: [16, 24, 32, 40] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: none - conc-list: [48, 56, 64, 80] - prefill: - num-worker: 3 - tp: 8 - ep: 8 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-3p2d-tep8-tp8-agentic.yaml" - decode: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: false - -dsv4-fp4-gb200-dynamo-vllm-agentic-2p1d-dep8-dep8: - image: vllm/vllm-openai:v0.23.0 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:gb200-nv - precision: fp4 - framework: dynamo-vllm - router: { name: dynamo-router, version: "1.3.0.dev20260618" } - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - agentic-coding: - - search-space: - - spec-decoding: none - conc-list: [32, 48, 64, 80] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: none - conc-list: [96, 128, 160] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - # Exploratory tail beyond the measured c160 normalized-throughput peak. - - spec-decoding: none - conc-list: [192, 224, 256] - prefill: - num-worker: 2 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb200-2p1d-dep8-dep8-agentic.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - -dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-agg: - image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:gb300-nv - precision: fp4 - framework: dynamo-vllm - router: { name: dynamo-router, version: "1.2.1" } - multinode: true - disagg: false - scenarios: - agentic-coding: - - search-space: - - spec-decoding: mtp - conc-list: [1] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp8-mtp-agentic.yaml" - # The aggregate worker also performs decode; keep the decode worker - # count at zero so result aggregation counts eight GPUs only once. - decode: - num-worker: 0 - tp: 8 - ep: 1 - dp-attn: false - - spec-decoding: mtp - conc-list: [4] - prefill: - num-worker: 1 - tp: 4 - ep: 1 - dp-attn: false - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/agg-gb300-tp4-mtp-agentic.yaml" - # The aggregate worker also performs decode; keep the decode worker - # count at zero so result aggregation counts four GPUs only once. - decode: - num-worker: 0 - tp: 4 - ep: 1 - dp-attn: false - -dsv4-fp4-gb300-dynamo-vllm-agentic-mtp-disagg: - image: vllm/vllm-openai:nightly-dev-arm64-cu13.0.1-426e59f - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:gb300-nv - precision: fp4 - framework: dynamo-vllm - kv-p2p-transfer: nixl - multinode: true - disagg: true - scenarios: - agentic-coding: - - search-space: - - spec-decoding: mtp - conc-list: [128] - router: { name: dynamo-router, version: "1.3.0.dev20260720" } - prefill: - num-worker: 1 - tp: 4 - ep: 4 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep4-dep8-c128-mtp-agentic.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - - spec-decoding: mtp - conc-list: [384] - router: { name: dynamo-router, version: "1.3.0.dev20260720" } - prefill: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - additional-settings: - - "CONFIG_FILE=recipes/vllm/deepseek-v4/agentic/disagg-gb300-1p1d-dep8-dep8-c384-mtp-agentic.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 8 - dp-attn: true - -# Day-0 Kimi K3 AgentX coverage on GB200. The three aggregate srt-slurm -# profiles mirror the serving strategies synthesized by the official recipe: -# latency: multi-node TP16 (4 GB200 nodes / 16 GPUs) -# balanced: multi-node TEP16 (4 GB200 nodes / 16 GPUs) -# throughput: multi-node TP4 x DP4 (4 GB200 nodes / EP16) -# All profiles enable Kimi K3 DSpark level 2 at the committed golden AL 2.51. -# `spec-decoding: mtp` is the existing matrix label used for speculative -# decoding methods; the checked-in recipes select DSpark explicitly. -# https://recipes.vllm.ai/moonshotai/Kimi-K3?hardware=gb200 kimik3-fp4-gb200-dynamo-vllm-agentic: image: vllm/vllm-openai:kimi-k3 model: moonshotai/Kimi-K3