From 52655766ff317d3c3ea648b5e0999072069898d2 Mon Sep 17 00:00:00 2001 From: richardhuo-nv Date: Wed, 12 Aug 2026 15:41:39 -0700 Subject: [PATCH 1/3] submit dynamo-trt for qwen3.5 fp4 for agentx --- ...p1d-dep1-dep16-c606-b32-mtp-kvoffload.yaml | 217 +++++++++++++++++ ...p3d-dep2-dep8-c1452-b64-mtp-kvoffload.yaml | 219 +++++++++++++++++ ...p1d-dep1-dep16-c225-b16-mtp-kvoffload.yaml | 215 ++++++++++++++++ ...p1d-dep1-dep16-c449-b16-mtp-kvoffload.yaml | 215 ++++++++++++++++ ...-1p1d-tep2-tep4-c8-b128-mtp-kvoffload.yaml | 229 +++++++++++++++++ ...2p3d-tep2-tep4-c24-b128-mtp-kvoffload.yaml | 230 ++++++++++++++++++ ...-2p3d-tep2-tep4-c6-b128-mtp-kvoffload.yaml | 230 ++++++++++++++++++ ...-2p3d-tep2-tep8-c6-b128-mtp-kvoffload.yaml | 230 ++++++++++++++++++ ...4p1d-dep4-dep8-c896-b64-mtp-kvoffload.yaml | 218 +++++++++++++++++ configs/nvidia-master.yaml | 150 ++++++++++++ perf-changelog.yaml | 10 + runners/launch_gb300-nv.sh | 9 +- 12 files changed, 2171 insertions(+), 1 deletion(-) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-15p1d-dep1-dep16-c606-b32-mtp-kvoffload.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-17p3d-dep2-dep8-c1452-b64-mtp-kvoffload.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c225-b16-mtp-kvoffload.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c449-b16-mtp-kvoffload.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p1d-tep2-tep4-c8-b128-mtp-kvoffload.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c24-b128-mtp-kvoffload.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c6-b128-mtp-kvoffload.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c6-b128-mtp-kvoffload.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-4p1d-dep4-dep8-c896-b64-mtp-kvoffload.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-15p1d-dep1-dep16-c606-b32-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-15p1d-dep1-dep16-c606-b32-mtp-kvoffload.yaml new file mode 100644 index 0000000000..0bd371d65e --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-15p1d-dep1-dep16-c606-b32-mtp-kvoffload.yaml @@ -0,0 +1,217 @@ +name: disagg-gb300-15p1d-dep1-dep16-c606-b32-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 4 + prefill_workers: 15 + gpus_per_prefill: 1 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 +backend: + type: trtllm + sequential_node_start: 2 + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 8192 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 1 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + enable_padding: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: true + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 32 + max_num_tokens: 224 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 \ No newline at end of file diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-17p3d-dep2-dep8-c1452-b64-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-17p3d-dep2-dep8-c1452-b64-mtp-kvoffload.yaml new file mode 100644 index 0000000000..39d2ed78bc --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-17p3d-dep2-dep8-c1452-b64-mtp-kvoffload.yaml @@ -0,0 +1,219 @@ +name: disagg-gb300-17p3d-dep2-dep8-c1452-b64-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 9 + prefill_workers: 17 + gpus_per_prefill: 2 + decode_nodes: 6 + decode_workers: 3 + gpus_per_decode: 8 +backend: + type: trtllm + sequential_node_start: 2 + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 8192 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 2 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 2 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + enable_padding: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: true + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 448 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 \ No newline at end of file diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c225-b16-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c225-b16-mtp-kvoffload.yaml new file mode 100644 index 0000000000..4cd5e33fe7 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c225-b16-mtp-kvoffload.yaml @@ -0,0 +1,215 @@ +name: disagg-gb300-19p1d-dep1-dep16-c225-b16-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 5 + prefill_workers: 19 + gpus_per_prefill: 1 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 +backend: + type: trtllm + sequential_node_start: 2 + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.41' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.41' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 8192 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 1 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_padding: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: true + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 16 + max_num_tokens: 96 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 \ No newline at end of file diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c449-b16-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c449-b16-mtp-kvoffload.yaml new file mode 100644 index 0000000000..b554c34196 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c449-b16-mtp-kvoffload.yaml @@ -0,0 +1,215 @@ +name: disagg-gb300-19p1d-dep1-dep16-c449-b16-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 5 + prefill_workers: 19 + gpus_per_prefill: 1 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 +backend: + type: trtllm + sequential_node_start: 2 + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.41' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.41' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 8192 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 1 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_padding: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: true + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 16 + max_num_tokens: 96 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 \ No newline at end of file diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p1d-tep2-tep4-c8-b128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p1d-tep2-tep4-c8-b128-mtp-kvoffload.yaml new file mode 100644 index 0000000000..11e7448380 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p1d-tep2-tep4-c8-b128-mtp-kvoffload.yaml @@ -0,0 +1,229 @@ +name: disagg-gb300-1p1d-tep2-tep4-c8-b128-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 2 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 4 +backend: + type: trtllm + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KV_TRANSFER_NUM_THREADS: '8' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KV_TRANSFER_NUM_THREADS: '8' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: false + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 16384 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 2 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 2 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + - 16384 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + enable_padding: true + enable_attention_dp: false + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 128 + max_num_tokens: 896 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 4 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 \ No newline at end of file diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c24-b128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c24-b128-mtp-kvoffload.yaml new file mode 100644 index 0000000000..fdd1593c56 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c24-b128-mtp-kvoffload.yaml @@ -0,0 +1,230 @@ +name: disagg-gb300-2p3d-tep2-tep4-c24-b128-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 2 + gpus_per_prefill: 2 + decode_nodes: 3 + decode_workers: 3 + gpus_per_decode: 4 +backend: + type: trtllm + sequential_node_start: true + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KV_TRANSFER_NUM_THREADS: '8' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KV_TRANSFER_NUM_THREADS: '8' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: false + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 16384 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 2 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 2 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + - 16384 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + enable_padding: true + enable_attention_dp: false + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 128 + max_num_tokens: 896 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 4 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 \ No newline at end of file diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c6-b128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c6-b128-mtp-kvoffload.yaml new file mode 100644 index 0000000000..9d2dd9aca9 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c6-b128-mtp-kvoffload.yaml @@ -0,0 +1,230 @@ +name: disagg-gb300-2p3d-tep2-tep4-c6-b128-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 2 + gpus_per_prefill: 2 + decode_nodes: 3 + decode_workers: 3 + gpus_per_decode: 4 +backend: + type: trtllm + sequential_node_start: true + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KV_TRANSFER_NUM_THREADS: '8' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KV_TRANSFER_NUM_THREADS: '8' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: false + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 16384 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 2 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 2 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + - 16384 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + enable_padding: true + enable_attention_dp: false + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 128 + max_num_tokens: 896 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 4 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 \ No newline at end of file diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c6-b128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c6-b128-mtp-kvoffload.yaml new file mode 100644 index 0000000000..bed82a7a37 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c6-b128-mtp-kvoffload.yaml @@ -0,0 +1,230 @@ +name: disagg-gb300-2p3d-tep2-tep8-c6-b128-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 2 + gpus_per_prefill: 2 + decode_nodes: 6 + decode_workers: 3 + gpus_per_decode: 8 +backend: + type: trtllm + sequential_node_start: true + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KV_TRANSFER_NUM_THREADS: '8' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KV_TRANSFER_NUM_THREADS: '8' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: false + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 16384 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 2 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 2 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + - 16384 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + enable_padding: true + enable_attention_dp: false + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 128 + max_num_tokens: 896 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 \ No newline at end of file diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-4p1d-dep4-dep8-c896-b64-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-4p1d-dep4-dep8-c896-b64-mtp-kvoffload.yaml new file mode 100644 index 0000000000..52dc9511a3 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-4p1d-dep4-dep8-c896-b64-mtp-kvoffload.yaml @@ -0,0 +1,218 @@ +name: disagg-gb300-4p1d-dep4-dep8-c896-b64-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 4 + prefill_workers: 4 + gpus_per_prefill: 4 + decode_nodes: 2 + decode_workers: 1 + gpus_per_decode: 8 +backend: + type: trtllm + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 8192 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 4 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + enable_padding: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: true + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 448 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 \ No newline at end of file diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 5c049d9f0c..0f0b34c696 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -6724,6 +6724,156 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: ep: 16 dp-attn: true +qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-prefix: qwen3.5 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-trt + router: { name: dynamo-router, version: "1.4.0.dev20260807" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + # 1P1D: 1 prefill (TP2/EP2), 1 decode (TP4/EP4), conc=8 + - spec-decoding: "mtp" + conc-list: [8] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p1d-tep2-tep4-c8-b128-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: false + # 2P3D: 2 prefill (TP2/EP2), 3 decode (TP4/EP4), conc=6 + - spec-decoding: "mtp" + conc-list: [6] + prefill: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c6-b128-mtp-kvoffload.yaml" + decode: + num-worker: 3 + tp: 4 + ep: 4 + dp-attn: false + # 2P3D: 2 prefill (TP2/EP2), 3 decode (TP8/EP8), conc=6 + - spec-decoding: "mtp" + conc-list: [6] + prefill: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c6-b128-mtp-kvoffload.yaml" + decode: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: false + # 2P3D: 2 prefill (TP2/EP2), 3 decode (TP4/EP4), conc=24 + - spec-decoding: "mtp" + conc-list: [24] + prefill: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c24-b128-mtp-kvoffload.yaml" + decode: + num-worker: 3 + tp: 4 + ep: 4 + dp-attn: false + # 4P1D: 4 prefill (TP4/EP4/dp-attn), 1 decode (TP8/EP8/dp-attn), conc=896 + - spec-decoding: "mtp" + conc-list: [896] + prefill: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-4p1d-dep4-dep8-c896-b64-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # 15P1D: 15 prefill (TP1/EP1/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=606 + - spec-decoding: "mtp" + conc-list: [606] + prefill: + num-worker: 15 + tp: 1 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-15p1d-dep1-dep16-c606-b32-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 17P3D: 17 prefill (TP2/EP2/dp-attn), 3 decode (TP8/EP8/dp-attn), conc=1452 + - spec-decoding: "mtp" + conc-list: [1452] + prefill: + num-worker: 17 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-17p3d-dep2-dep8-c1452-b64-mtp-kvoffload.yaml" + decode: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: true + # 19P1D: 19 prefill (TP1/EP1/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=225 + - spec-decoding: "mtp" + conc-list: [225] + prefill: + num-worker: 19 + tp: 1 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c225-b16-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 19P1D: 19 prefill (TP1/EP1/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=449 + - spec-decoding: "mtp" + conc-list: [449] + prefill: + num-worker: 19 + tp: 1 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c449-b16-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260724-433429b1 model: nvidia/Qwen3.5-397B-A17B-NVFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f118b99aa7..df03357f6b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5909,3 +5909,13 @@ - "Increase MTP speculative steps from 3 to 5 (num-draft-tokens 4→6) and raise SGLANG_SIMULATE_ACC_LEN from 2.99 to 3.61 to reflect higher measured acceptance rate, targeting ~15% throughput improvement" - "Replace the TP8/EP8 low-concurrency arm (conc [1,2,4]) hicache offload with kv-offloading: none to reduce per-request latency at low load; conc [1,2,4] are now tested on both tp=4+hicache and tp=8+no-offload so SemiAnalysis can select the Pareto-optimal point per concurrency" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2570 + +- config-keys: + - qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Add Qwen3.5-397B-A17B-NVFP4-V2 GB300 TRT-LLM disaggregated AgentX with Dynamo router 1.4.0.dev20260807 and NIXL KV transfer." + - "Cover 9 disaggregated topologies (1P1D through 19P1D/17P3D) with MTP speculative decoding across conc 6-1452 using TRT-LLM release:1.3.0rc24." + - "Attention-DP enabled (dep shapes) and disabled (tep shapes) variants, with recipe CONFIG_FILEs from recipes/trtllm/qwen3.5-fp4/agentx/submission/." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/TODO diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 27154c404e..a5038c572a 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -184,7 +184,14 @@ SRT_REPO_DIR="${GITHUB_WORKSPACE}/srt-slurm-${GITHUB_RUN_ID:-manual}-${GITHUB_RU SRTCTL_SETUP_SCRIPT="" rm -rf "$SRT_REPO_DIR" -if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then +if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "qwen3.5" ]]; then + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout v1.0.48 + mkdir -p recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx" \ + recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx +elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then # Qwen3.5 agentic uses NVIDIA/srt-slurm v1.0.38: the two features the # cquil11 fork was pinned for are merged upstream (present in v1.0.36) — # - `srtctl apply --no-preflight` (skip the in-process model FS check): From 463ad34dbc2e68ff65848e961837c459f7308854 Mon Sep 17 00:00:00 2001 From: richardhuo-nv Date: Wed, 12 Aug 2026 15:54:17 -0700 Subject: [PATCH 2/3] submit dynamo-trt for qwen3.5 fp4 for agentx --- .../disagg-gb300-15p1d-dep1-dep16-c606-b32-mtp-kvoffload.yaml | 2 +- .../disagg-gb300-17p3d-dep2-dep8-c1452-b64-mtp-kvoffload.yaml | 2 +- .../disagg-gb300-19p1d-dep1-dep16-c225-b16-mtp-kvoffload.yaml | 2 +- .../disagg-gb300-19p1d-dep1-dep16-c449-b16-mtp-kvoffload.yaml | 2 +- .../disagg-gb300-1p1d-tep2-tep4-c8-b128-mtp-kvoffload.yaml | 2 +- .../disagg-gb300-2p3d-tep2-tep4-c24-b128-mtp-kvoffload.yaml | 2 +- .../disagg-gb300-2p3d-tep2-tep4-c6-b128-mtp-kvoffload.yaml | 2 +- .../disagg-gb300-2p3d-tep2-tep8-c6-b128-mtp-kvoffload.yaml | 2 +- .../disagg-gb300-4p1d-dep4-dep8-c896-b64-mtp-kvoffload.yaml | 2 +- perf-changelog.yaml | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-15p1d-dep1-dep16-c606-b32-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-15p1d-dep1-dep16-c606-b32-mtp-kvoffload.yaml index 0bd371d65e..1ef28e9a55 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-15p1d-dep1-dep16-c606-b32-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-15p1d-dep1-dep16-c606-b32-mtp-kvoffload.yaml @@ -214,4 +214,4 @@ benchmark: AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache HF_HUB_CACHE: /hf_hub_cache WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 \ No newline at end of file + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-17p3d-dep2-dep8-c1452-b64-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-17p3d-dep2-dep8-c1452-b64-mtp-kvoffload.yaml index 39d2ed78bc..5377338e5b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-17p3d-dep2-dep8-c1452-b64-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-17p3d-dep2-dep8-c1452-b64-mtp-kvoffload.yaml @@ -216,4 +216,4 @@ benchmark: AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache HF_HUB_CACHE: /hf_hub_cache WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 \ No newline at end of file + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c225-b16-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c225-b16-mtp-kvoffload.yaml index 4cd5e33fe7..46b7341885 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c225-b16-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c225-b16-mtp-kvoffload.yaml @@ -212,4 +212,4 @@ benchmark: AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache HF_HUB_CACHE: /hf_hub_cache WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 \ No newline at end of file + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c449-b16-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c449-b16-mtp-kvoffload.yaml index b554c34196..c462435df8 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c449-b16-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c449-b16-mtp-kvoffload.yaml @@ -212,4 +212,4 @@ benchmark: AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache HF_HUB_CACHE: /hf_hub_cache WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 \ No newline at end of file + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p1d-tep2-tep4-c8-b128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p1d-tep2-tep4-c8-b128-mtp-kvoffload.yaml index 11e7448380..fddf6697eb 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p1d-tep2-tep4-c8-b128-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p1d-tep2-tep4-c8-b128-mtp-kvoffload.yaml @@ -226,4 +226,4 @@ benchmark: AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache HF_HUB_CACHE: /hf_hub_cache WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 \ No newline at end of file + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c24-b128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c24-b128-mtp-kvoffload.yaml index fdd1593c56..2a05970a5a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c24-b128-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c24-b128-mtp-kvoffload.yaml @@ -227,4 +227,4 @@ benchmark: AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache HF_HUB_CACHE: /hf_hub_cache WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 \ No newline at end of file + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c6-b128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c6-b128-mtp-kvoffload.yaml index 9d2dd9aca9..edbeeaf2b1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c6-b128-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c6-b128-mtp-kvoffload.yaml @@ -227,4 +227,4 @@ benchmark: AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache HF_HUB_CACHE: /hf_hub_cache WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 \ No newline at end of file + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c6-b128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c6-b128-mtp-kvoffload.yaml index bed82a7a37..17fb460be0 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c6-b128-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c6-b128-mtp-kvoffload.yaml @@ -227,4 +227,4 @@ benchmark: AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache HF_HUB_CACHE: /hf_hub_cache WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 \ No newline at end of file + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-4p1d-dep4-dep8-c896-b64-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-4p1d-dep4-dep8-c896-b64-mtp-kvoffload.yaml index 52dc9511a3..19d999cf63 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-4p1d-dep4-dep8-c896-b64-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-4p1d-dep4-dep8-c896-b64-mtp-kvoffload.yaml @@ -215,4 +215,4 @@ benchmark: AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache HF_HUB_CACHE: /hf_hub_cache WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 - SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 \ No newline at end of file + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index df03357f6b..a2d8470ec6 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5918,4 +5918,4 @@ - "Add Qwen3.5-397B-A17B-NVFP4-V2 GB300 TRT-LLM disaggregated AgentX with Dynamo router 1.4.0.dev20260807 and NIXL KV transfer." - "Cover 9 disaggregated topologies (1P1D through 19P1D/17P3D) with MTP speculative decoding across conc 6-1452 using TRT-LLM release:1.3.0rc24." - "Attention-DP enabled (dep shapes) and disabled (tep shapes) variants, with recipe CONFIG_FILEs from recipes/trtllm/qwen3.5-fp4/agentx/submission/." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/TODO + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2584 From 7c817f9f5f892a1320d75d5f4c3597e32f71dc90 Mon Sep 17 00:00:00 2001 From: richardhuo-nv Date: Wed, 12 Aug 2026 17:10:49 -0700 Subject: [PATCH 3/3] fix --- .../disagg-gb300-2p3d-tep2-tep4-c24-b128-mtp-kvoffload.yaml | 2 +- .../disagg-gb300-2p3d-tep2-tep4-c6-b128-mtp-kvoffload.yaml | 2 +- .../disagg-gb300-2p3d-tep2-tep8-c6-b128-mtp-kvoffload.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c24-b128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c24-b128-mtp-kvoffload.yaml index 2a05970a5a..a3cec9ad8a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c24-b128-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c24-b128-mtp-kvoffload.yaml @@ -29,7 +29,7 @@ resources: gpus_per_decode: 4 backend: type: trtllm - sequential_node_start: true + sequential_node_start: 1 prefill_environment: CUDA_SCALE_LAUNCH_QUEUES: 4x MIMALLOC_ARENA_RESERVE: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c6-b128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c6-b128-mtp-kvoffload.yaml index edbeeaf2b1..68bcb278d1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c6-b128-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c6-b128-mtp-kvoffload.yaml @@ -29,7 +29,7 @@ resources: gpus_per_decode: 4 backend: type: trtllm - sequential_node_start: true + sequential_node_start: 1 prefill_environment: CUDA_SCALE_LAUNCH_QUEUES: 4x MIMALLOC_ARENA_RESERVE: '0' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c6-b128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c6-b128-mtp-kvoffload.yaml index 17fb460be0..6be4e07c9a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c6-b128-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c6-b128-mtp-kvoffload.yaml @@ -29,7 +29,7 @@ resources: gpus_per_decode: 8 backend: type: trtllm - sequential_node_start: true + sequential_node_start: 1 prefill_environment: CUDA_SCALE_LAUNCH_QUEUES: 4x MIMALLOC_ARENA_RESERVE: '0'