diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-15p1d-dep1-dep16-c606-b32-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-15p1d-dep1-dep16-c606-b32-mtp-kvoffload.yaml new file mode 100644 index 0000000000..1ef28e9a55 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-15p1d-dep1-dep16-c606-b32-mtp-kvoffload.yaml @@ -0,0 +1,217 @@ +name: disagg-gb300-15p1d-dep1-dep16-c606-b32-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 4 + prefill_workers: 15 + gpus_per_prefill: 1 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 +backend: + type: trtllm + sequential_node_start: 2 + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 8192 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 1 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + enable_padding: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: true + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 32 + max_num_tokens: 224 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-17p3d-dep2-dep8-c1452-b64-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-17p3d-dep2-dep8-c1452-b64-mtp-kvoffload.yaml new file mode 100644 index 0000000000..5377338e5b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-17p3d-dep2-dep8-c1452-b64-mtp-kvoffload.yaml @@ -0,0 +1,219 @@ +name: disagg-gb300-17p3d-dep2-dep8-c1452-b64-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 9 + prefill_workers: 17 + gpus_per_prefill: 2 + decode_nodes: 6 + decode_workers: 3 + gpus_per_decode: 8 +backend: + type: trtllm + sequential_node_start: 2 + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 8192 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 2 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 2 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + enable_padding: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: true + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 448 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c225-b16-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c225-b16-mtp-kvoffload.yaml new file mode 100644 index 0000000000..46b7341885 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c225-b16-mtp-kvoffload.yaml @@ -0,0 +1,215 @@ +name: disagg-gb300-19p1d-dep1-dep16-c225-b16-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 5 + prefill_workers: 19 + gpus_per_prefill: 1 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 +backend: + type: trtllm + sequential_node_start: 2 + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.41' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.41' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 8192 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 1 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_padding: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: true + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 16 + max_num_tokens: 96 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c449-b16-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c449-b16-mtp-kvoffload.yaml new file mode 100644 index 0000000000..c462435df8 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c449-b16-mtp-kvoffload.yaml @@ -0,0 +1,215 @@ +name: disagg-gb300-19p1d-dep1-dep16-c449-b16-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 5 + prefill_workers: 19 + gpus_per_prefill: 1 + decode_nodes: 4 + decode_workers: 1 + gpus_per_decode: 16 +backend: + type: trtllm + sequential_node_start: 2 + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.41' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.41' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRT_LLM_DISABLE_LOAD_WEIGHTS_IN_PARALLEL: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 8192 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 1 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 1 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + enable_padding: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: true + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 16 + max_num_tokens: 96 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 16 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 5 + stream_interval: 20 + tensor_parallel_size: 16 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p1d-tep2-tep4-c8-b128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p1d-tep2-tep4-c8-b128-mtp-kvoffload.yaml new file mode 100644 index 0000000000..fddf6697eb --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p1d-tep2-tep4-c8-b128-mtp-kvoffload.yaml @@ -0,0 +1,229 @@ +name: disagg-gb300-1p1d-tep2-tep4-c8-b128-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 2 + decode_nodes: 1 + decode_workers: 1 + gpus_per_decode: 4 +backend: + type: trtllm + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KV_TRANSFER_NUM_THREADS: '8' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KV_TRANSFER_NUM_THREADS: '8' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: false + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 16384 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 2 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 2 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + - 16384 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + enable_padding: true + enable_attention_dp: false + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 128 + max_num_tokens: 896 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 4 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c24-b128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c24-b128-mtp-kvoffload.yaml new file mode 100644 index 0000000000..a3cec9ad8a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c24-b128-mtp-kvoffload.yaml @@ -0,0 +1,230 @@ +name: disagg-gb300-2p3d-tep2-tep4-c24-b128-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 2 + gpus_per_prefill: 2 + decode_nodes: 3 + decode_workers: 3 + gpus_per_decode: 4 +backend: + type: trtllm + sequential_node_start: 1 + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KV_TRANSFER_NUM_THREADS: '8' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KV_TRANSFER_NUM_THREADS: '8' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: false + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 16384 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 2 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 2 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + - 16384 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + enable_padding: true + enable_attention_dp: false + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 128 + max_num_tokens: 896 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 4 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c6-b128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c6-b128-mtp-kvoffload.yaml new file mode 100644 index 0000000000..68bcb278d1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c6-b128-mtp-kvoffload.yaml @@ -0,0 +1,230 @@ +name: disagg-gb300-2p3d-tep2-tep4-c6-b128-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 2 + gpus_per_prefill: 2 + decode_nodes: 3 + decode_workers: 3 + gpus_per_decode: 4 +backend: + type: trtllm + sequential_node_start: 1 + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KV_TRANSFER_NUM_THREADS: '8' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KV_TRANSFER_NUM_THREADS: '8' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: false + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 16384 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 2 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 2 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + - 16384 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + enable_padding: true + enable_attention_dp: false + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 128 + max_num_tokens: 896 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 4 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 4 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c6-b128-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c6-b128-mtp-kvoffload.yaml new file mode 100644 index 0000000000..6be4e07c9a --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c6-b128-mtp-kvoffload.yaml @@ -0,0 +1,230 @@ +name: disagg-gb300-2p3d-tep2-tep8-c6-b128-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 1 + prefill_workers: 2 + gpus_per_prefill: 2 + decode_nodes: 6 + decode_workers: 3 + gpus_per_decode: 8 +backend: + type: trtllm + sequential_node_start: 1 + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KV_TRANSFER_NUM_THREADS: '8' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_KV_TRANSFER_NUM_THREADS: '8' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: false + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 16384 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 2 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 2 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + - 16384 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + - 72 + - 80 + - 88 + - 96 + - 104 + - 112 + - 120 + - 128 + enable_padding: true + enable_attention_dp: false + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: false + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 128 + max_num_tokens: 896 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-4p1d-dep4-dep8-c896-b64-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-4p1d-dep4-dep8-c896-b64-mtp-kvoffload.yaml new file mode 100644 index 0000000000..19d999cf63 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-4p1d-dep4-dep8-c896-b64-mtp-kvoffload.yaml @@ -0,0 +1,218 @@ +name: disagg-gb300-4p1d-dep4-dep8-c896-b64-mtp-kvoffload +model: + path: qwen3.5-fp4 + container: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + precision: fp4 + +dynamo: + install: true + wheel: 1.4.0.dev20260807 + +identity: + model: + repo: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + container: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + frameworks: + tensorrt_llm: 1.3.0rc24 +health_check: + max_attempts: 720 + interval_seconds: 10 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 4 + prefill_workers: 4 + gpus_per_prefill: 4 + decode_nodes: 2 + decode_workers: 1 + gpus_per_decode: 8 +backend: + type: trtllm + prefill_environment: + CUDA_SCALE_LAUNCH_QUEUES: 4x + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + decode_environment: + MIMALLOC_ARENA_RESERVE: '0' + MIMALLOC_PURGE_DELAY: '' + NCCL_GRAPH_MIXING_SUPPORT: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + TLLM_LOG_LEVEL: INFO + TLLM_SPEC_DECODE_FORCE_NUM_ACCEPTED_TOKENS: '3.80' + TRTLLM_ENABLE_PDL: '1' + TRTLLM_PINNED_WEIGHT_STAGING: '1' + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_ENGINE_CONV_AFFINITY: "1" + trtllm_config: + prefill: + attention_dp_config: + kv_cache_routing_conversation_affinity: true + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: null + disable_overlap_scheduler: false + enable_attention_dp: true + enable_chunked_prefill: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + kv_cache_config: + block_reuse_config: + max_num_turns: 3 + policy: per_conversation + dtype: fp8 + enable_block_reuse: true + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.8 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + mamba_state_config: + additional_snapshot_offsets_from_end: + - 2 + periodic_snapshot_interval: 0 + pool_ratio: + - 0.8 + - 0.2 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 8192 + max_seq_len: 262144 + moe_config: + backend: CUTEDSL + moe_expert_parallel_size: 4 + num_serve_frontends: 8 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + tensor_parallel_size: 4 + torch_compile_config: + capture_num_tokens: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + - 8192 + enable_fullgraph: true + enable_piecewise_cuda_graph: true + trust_remote_code: true + decode: + cache_transceiver_config: + backend: NIXL + kv_transfer_sender_future_timeout_ms: 20 + kv_transfer_timeout_ms: 600000 + transceiver_runtime: PYTHON + cuda_graph_config: + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 24 + - 32 + - 40 + - 48 + - 56 + - 64 + enable_padding: true + enable_attention_dp: true + enable_iter_perf_stats: false + enable_iter_req_stats: false + enable_lm_head_tp_in_adp: true + kv_cache_config: + avg_seq_len: 102150 + dtype: fp8 + enable_block_reuse: false + event_buffer_max_size: 0 + free_gpu_memory_fraction: 0.85 + host_cache_size: 137438953472 + iteration_stats_interval: 1 + tokens_per_block: 64 + use_kv_cache_manager_v2: true + max_batch_size: 64 + max_num_tokens: 448 + max_seq_len: 262148 + moe_config: + backend: CUTEDSL + use_low_precision_moe_combine: true + moe_expert_parallel_size: 8 + num_postprocess_workers: 4 + num_serve_frontends: 4 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + pipeline_parallel_size: 1 + print_iter_log: true + return_perf_metrics: false + scheduler_config: + capacity_scheduler_policy: GUARANTEED_NO_EVICT + speculative_config: + decoding_type: MTP + max_draft_len: 6 + stream_interval: 20 + tensor_parallel_size: 8 + trust_remote_code: true +frontend: + type: dynamo + enable_multiple_frontends: false + env: + ETCD_LEASE_TTL: "120" + DYN_TOKENIZER_CACHE: "1" + DYN_TOKENIZER_CACHE_BYTES: "8000000000" + DYN_TOKENIZER: "fastokens" + args: + router-mode: kv + router-session-affinity-ttl-secs: '14400' + active-decode-blocks-threshold: None + active-prefill-tokens-threshold: None + active-prefill-tokens-threshold-frac: None +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: '8000' + IS_MULTINODE: 'true' + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: 'true' + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: '0' + AIPERF_DATASET_MMAP_CACHE_DIR: /aiperf_mmap_cache + HF_HUB_CACHE: /hf_hub_cache + WEKA_LOADER_OVERRIDE: semianalysis_cc_traces_weka_062126 + SERVED_MODEL_NAME: Qwen3.5-397B-A17B-NVFP4-V2 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 5c049d9f0c..0f0b34c696 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -6724,6 +6724,156 @@ qwen3.5-fp4-gb300-dynamo-trt-mtp: ep: 16 dp-attn: true +qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc24 + model: nvidia/Qwen3.5-397B-A17B-NVFP4-V2 + model-prefix: qwen3.5 + runner: cluster:gb300-nv + precision: fp4 + framework: dynamo-trt + router: { name: dynamo-router, version: "1.4.0.dev20260807" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - search-space: + # 1P1D: 1 prefill (TP2/EP2), 1 decode (TP4/EP4), conc=8 + - spec-decoding: "mtp" + conc-list: [8] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-1p1d-tep2-tep4-c8-b128-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 4 + dp-attn: false + # 2P3D: 2 prefill (TP2/EP2), 3 decode (TP4/EP4), conc=6 + - spec-decoding: "mtp" + conc-list: [6] + prefill: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c6-b128-mtp-kvoffload.yaml" + decode: + num-worker: 3 + tp: 4 + ep: 4 + dp-attn: false + # 2P3D: 2 prefill (TP2/EP2), 3 decode (TP8/EP8), conc=6 + - spec-decoding: "mtp" + conc-list: [6] + prefill: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep8-c6-b128-mtp-kvoffload.yaml" + decode: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: false + # 2P3D: 2 prefill (TP2/EP2), 3 decode (TP4/EP4), conc=24 + - spec-decoding: "mtp" + conc-list: [24] + prefill: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-2p3d-tep2-tep4-c24-b128-mtp-kvoffload.yaml" + decode: + num-worker: 3 + tp: 4 + ep: 4 + dp-attn: false + # 4P1D: 4 prefill (TP4/EP4/dp-attn), 1 decode (TP8/EP8/dp-attn), conc=896 + - spec-decoding: "mtp" + conc-list: [896] + prefill: + num-worker: 4 + tp: 4 + ep: 4 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-4p1d-dep4-dep8-c896-b64-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + # 15P1D: 15 prefill (TP1/EP1/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=606 + - spec-decoding: "mtp" + conc-list: [606] + prefill: + num-worker: 15 + tp: 1 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-15p1d-dep1-dep16-c606-b32-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 17P3D: 17 prefill (TP2/EP2/dp-attn), 3 decode (TP8/EP8/dp-attn), conc=1452 + - spec-decoding: "mtp" + conc-list: [1452] + prefill: + num-worker: 17 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-17p3d-dep2-dep8-c1452-b64-mtp-kvoffload.yaml" + decode: + num-worker: 3 + tp: 8 + ep: 8 + dp-attn: true + # 19P1D: 19 prefill (TP1/EP1/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=225 + - spec-decoding: "mtp" + conc-list: [225] + prefill: + num-worker: 19 + tp: 1 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c225-b16-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + # 19P1D: 19 prefill (TP1/EP1/dp-attn), 1 decode (TP16/EP16/dp-attn), conc=449 + - spec-decoding: "mtp" + conc-list: [449] + prefill: + num-worker: 19 + tp: 1 + ep: 1 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx/disagg-gb300-19p1d-dep1-dep16-c449-b16-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 16 + ep: 16 + dp-attn: true + qwen3.5-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-cu13-20260724-433429b1 model: nvidia/Qwen3.5-397B-A17B-NVFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f118b99aa7..a2d8470ec6 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5909,3 +5909,13 @@ - "Increase MTP speculative steps from 3 to 5 (num-draft-tokens 4→6) and raise SGLANG_SIMULATE_ACC_LEN from 2.99 to 3.61 to reflect higher measured acceptance rate, targeting ~15% throughput improvement" - "Replace the TP8/EP8 low-concurrency arm (conc [1,2,4]) hicache offload with kv-offloading: none to reduce per-request latency at low load; conc [1,2,4] are now tested on both tp=4+hicache and tp=8+no-offload so SemiAnalysis can select the Pareto-optimal point per concurrency" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2570 + +- config-keys: + - qwen3.5-fp4-gb300-dynamo-trt-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Add Qwen3.5-397B-A17B-NVFP4-V2 GB300 TRT-LLM disaggregated AgentX with Dynamo router 1.4.0.dev20260807 and NIXL KV transfer." + - "Cover 9 disaggregated topologies (1P1D through 19P1D/17P3D) with MTP speculative decoding across conc 6-1452 using TRT-LLM release:1.3.0rc24." + - "Attention-DP enabled (dep shapes) and disabled (tep shapes) variants, with recipe CONFIG_FILEs from recipes/trtllm/qwen3.5-fp4/agentx/submission/." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2584 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 27154c404e..a5038c572a 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -184,7 +184,14 @@ SRT_REPO_DIR="${GITHUB_WORKSPACE}/srt-slurm-${GITHUB_RUN_ID:-manual}-${GITHUB_RU SRTCTL_SETUP_SCRIPT="" rm -rf "$SRT_REPO_DIR" -if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then +if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "qwen3.5" ]]; then + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout v1.0.48 + mkdir -p recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx" \ + recipes/trtllm/qwen3.5/gb300-fp4/disagg/agentx +elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "qwen3.5" ]]; then # Qwen3.5 agentic uses NVIDIA/srt-slurm v1.0.38: the two features the # cquil11 fork was pinned for are merged upstream (present in v1.0.36) — # - `srtctl apply --no-preflight` (skip the in-process model FS check):