diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/agg-tp4-c4-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/agg-tp4-c4-mtp.yaml new file mode 100644 index 0000000000..1a8e0defe0 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/agg-tp4-c4-mtp.yaml @@ -0,0 +1,127 @@ +schema: 2 +name: "agg-b300-tp4-c4-mtp" + +# AgentX aggregate topology: one TP4 worker occupies half of one eight-GPU +# B300 node and serves both prefill and decode with DSpark K=6. + +model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" + +dynamo: + install: false + +health_check: + max_attempts: 2880 # 8h: non-EP TP MoE startup took ~3h20m alone (TP8) + interval_seconds: 10 + +resources: + gpu_type: "b300" + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: false + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 4 + + env: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + PIP_BREAK_SYSTEM_PACKAGES: "1" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" + enable-metrics: true + enable-cache-report: true + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + stream-interval: 10 + watchdog-timeout: 1000000 + mem-fraction-static: 0.90 + page-size: 256 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + moe-runner-backend: "flashinfer_mxfp4" + enable-deepseek-v4-fp4-indexer: true + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + max-running-requests: 8 + cuda-graph-max-bs-decode: 8 + scheduler-recv-interval: 30 + dp-size: 1 + tp-size: 4 + ep-size: 1 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + +sbatch_directives: + mem: "0" + exclude: "dsxe-sa-b300-prd0-gpu-00,dsxe-sa-b300-prd0-gpu-01,dsxe-sa-b300-prd0-gpu-04,dsxe-sa-b300-prd0-gpu-06,dsxe-sa-b300-prd0-gpu-11,dsxe-sa-b300-prd0-gpu-16" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "false" + TP: "4" + PP_SIZE: "1" + PCP_SIZE: "1" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/agg-tp8-c1-mtp.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/agg-tp8-c1-mtp.yaml new file mode 100644 index 0000000000..0e27b14a78 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/agg-tp8-c1-mtp.yaml @@ -0,0 +1,127 @@ +schema: 2 +name: "agg-b300-tp8-c1-mtp" + +# AgentX aggregate topology: one TP8 worker occupies one eight-GPU B300 node +# and serves both prefill and decode with DSpark K=6. + +model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" + +dynamo: + install: false + +health_check: + max_attempts: 2880 # 8h: non-EP TP MoE startup took ~3h20m alone (TP8) + interval_seconds: 10 + +resources: + gpu_type: "b300" + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: infra + - name: nats + type: nats + placement: + node: infra + options: + max_payload_mb: 32 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: false + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + DYN_NATS_REQUEST_TIMEOUT_SECS: "1800" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +engine: sglang +roles: + agg: + nodes: 1 + workers: 1 + gpus: 8 + + env: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" + SGLANG_DEFAULT_THINKING: "1" + SGLANG_DSV4_REASONING_EFFORT: high + PIP_BREAK_SYSTEM_PACKAGES: "1" + SGLANG_JIT_DEEPGEMM_PRECOMPILE: "1" + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: "1" + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: "1" + SGLANG_OPT_USE_ONLINE_COMPRESS: "0" + SGLANG_OPT_USE_JIT_INDEXER_METADATA: "1" + SGLANG_OPT_USE_JIT_NORM: "1" + SGLANG_OPT_USE_TOPK_V2: "True" + + args: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro-0813" + enable-metrics: true + enable-cache-report: true + trust-remote-code: true + weight-loader-prefetch-checkpoints: true + stream-interval: 10 + watchdog-timeout: 1000000 + mem-fraction-static: 0.90 + page-size: 256 + chunked-prefill-size: 8192 + max-prefill-tokens: 8192 + moe-runner-backend: "flashinfer_mxfp4" + enable-deepseek-v4-fp4-indexer: true + disable-flashinfer-autotune: true + swa-full-tokens-ratio: 0.1 + max-running-requests: 2 + cuda-graph-max-bs-decode: 2 + scheduler-recv-interval: 30 + dp-size: 1 + tp-size: 8 + ep-size: 1 + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + +sbatch_directives: + mem: "0" + exclude: "dsxe-sa-b300-prd0-gpu-00,dsxe-sa-b300-prd0-gpu-01,dsxe-sa-b300-prd0-gpu-04,dsxe-sa-b300-prd0-gpu-06,dsxe-sa-b300-prd0-gpu-11,dsxe-sa-b300-prd0-gpu-16" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" + RESULT_DIR: "/logs/agentic" + PORT: "8000" + IS_MULTINODE: "false" + TP: "8" + PP_SIZE: "1" + PCP_SIZE: "1" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/disagg-1p1d-dep8-dep8-c240-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/disagg-1p1d-dep8-dep8-c240-mtp-kvoffload.yaml new file mode 100644 index 0000000000..6975e0bb48 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/disagg-1p1d-dep8-dep8-c240-mtp-kvoffload.yaml @@ -0,0 +1,220 @@ +schema: 2 +name: "disagg-b300-1p1d-dep8-dep8-c240-mtp-kvoffload" + +# Agentic-coding SGLang disaggregated recipe for DeepSeek-V4-Pro-0813 on B300 +# (1P x DEP8 / 1D x DEP8, DSpark K=6 + hierarchical-cache KV offload), +# tuned for concurrency 240. Concurrency is exported by the master config. + +model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" + +dynamo: + install: false + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: b300 + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 32 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + DYN_TCP_REQUEST_TIMEOUT: "60" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_DISAGGREGATION_NIXL_BACKEND: LIBFABRIC + SGLANG_DSV4_MHC_PREWARM: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + + args: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + enable-deepseek-v4-fp4-indexer: true + enable-w4a4-mxfp4-megamoe: true + disaggregation-transfer-backend: nixl + disaggregation-mode: prefill + load-balance-method: total_tokens + mem-fraction-static: 0.85 + page-size: 256 + swa-full-tokens-ratio: 0.02 + max-running-requests: 256 + cuda-graph-max-bs-prefill: 256 + chunked-prefill-size: 65536 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 3 + hicache-io-backend: direct + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_DISAGGREGATION_NIXL_BACKEND: LIBFABRIC + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + + args: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + enable-deepseek-v4-fp4-indexer: true + enable-w4a4-mxfp4-megamoe: true + disaggregation-transfer-backend: nixl + disaggregation-mode: decode + load-balance-method: total_tokens + mem-fraction-static: 0.9 + page-size: 256 + swa-full-tokens-ratio: 0.02 + max-running-requests: 480 + cuda-graph-max-bs-decode: 256 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + + +sbatch_directives: + mem: "0" + exclude: "dsxe-sa-b300-prd0-gpu-00,dsxe-sa-b300-prd0-gpu-01,dsxe-sa-b300-prd0-gpu-04,dsxe-sa-b300-prd0-gpu-06,dsxe-sa-b300-prd0-gpu-11,dsxe-sa-b300-prd0-gpu-16" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/disagg-1p1d-dep8-dep8-c64-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/disagg-1p1d-dep8-dep8-c64-mtp-kvoffload.yaml new file mode 100644 index 0000000000..6dc77ff5e6 --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/disagg-1p1d-dep8-dep8-c64-mtp-kvoffload.yaml @@ -0,0 +1,220 @@ +schema: 2 +name: "disagg-b300-1p1d-dep8-dep8-c64-mtp-kvoffload" + +# Agentic-coding SGLang disaggregated recipe for DeepSeek-V4-Pro-0813 on B300 +# (1P x DEP8 / 1D x DEP8, DSpark K=6 + hierarchical-cache KV offload), +# tuned for concurrency 64. Concurrency is exported by the master config. + +model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" + +dynamo: + install: false + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: b300 + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 32 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + DYN_TCP_REQUEST_TIMEOUT: "60" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +engine: sglang +roles: + prefill: + nodes: 1 + workers: 1 + gpus: 8 + env: + SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_DISAGGREGATION_NIXL_BACKEND: LIBFABRIC + SGLANG_DSV4_MHC_PREWARM: '1' + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + + args: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + enable-deepseek-v4-fp4-indexer: true + enable-w4a4-mxfp4-megamoe: true + disaggregation-transfer-backend: nixl + disaggregation-mode: prefill + load-balance-method: total_tokens + mem-fraction-static: 0.85 + page-size: 256 + swa-full-tokens-ratio: 0.02 + max-running-requests: 256 + cuda-graph-max-bs-prefill: 256 + chunked-prefill-size: 65536 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 3 + hicache-io-backend: direct + + decode: + nodes: 1 + workers: 1 + gpus: 8 + + env: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_DISAGGREGATION_NIXL_BACKEND: LIBFABRIC + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + + args: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + enable-deepseek-v4-fp4-indexer: true + enable-w4a4-mxfp4-megamoe: true + disaggregation-transfer-backend: nixl + disaggregation-mode: decode + load-balance-method: total_tokens + mem-fraction-static: 0.9 + page-size: 256 + swa-full-tokens-ratio: 0.02 + max-running-requests: 128 + cuda-graph-max-bs-decode: 256 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + + +sbatch_directives: + mem: "0" + exclude: "dsxe-sa-b300-prd0-gpu-00,dsxe-sa-b300-prd0-gpu-01,dsxe-sa-b300-prd0-gpu-04,dsxe-sa-b300-prd0-gpu-06,dsxe-sa-b300-prd0-gpu-11,dsxe-sa-b300-prd0-gpu-16" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/disagg-2p1d-dep8-dep8-c480-mtp-kvoffload.yaml b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/disagg-2p1d-dep8-dep8-c480-mtp-kvoffload.yaml new file mode 100644 index 0000000000..5ef595d31b --- /dev/null +++ b/inferencex-e2e/benchmarks/multi_node/srt-slurm-recipes/dsv4/sglang/b300-fp4/agentx/disagg-2p1d-dep8-dep8-c480-mtp-kvoffload.yaml @@ -0,0 +1,222 @@ +schema: 2 +name: "disagg-b300-2p1d-dep8-dep8-c480-mtp-kvoffload" + +# Agentic-coding SGLang disaggregated recipe for DeepSeek-V4-Pro-0813 on B300 +# (2P x DEP8 / 1D x DEP8, DSpark K=6 + hierarchical-cache KV offload), +# tuned for concurrency 480. Concurrency is exported by the master config. + +model: + path: "deepseek-v4-pro-0813" + container: "dynamo-sglang" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro-0813" + container: + image: "lmsysorg/sglang:nightly-dev-20260916-c9a8fba9" + +dynamo: + install: false + +health_check: + max_attempts: 1440 + interval_seconds: 10 + +resources: + gpu_type: b300 + gpus_per_node: 8 +services: + - name: etcd + type: etcd + placement: + node: dedicated + - name: nats + type: nats + placement: + node: dedicated + options: + max_payload_mb: 32 +frontend: + type: dynamo + nginx_session_affinity: true + nginx_session_affinity_header: X-Dynamo-Session-ID + enable_multiple_frontends: true + num_additional_frontends: 4 + env: + PIP_BREAK_SYSTEM_PACKAGES: "1" + DYN_TCP_REQUEST_TIMEOUT: "60" + args: + router-mode: "kv" + router-session-affinity-ttl-secs: "3600" + active-decode-blocks-threshold: "None" + active-prefill-tokens-threshold: "None" + active-prefill-tokens-threshold-frac: "None" + +engine: sglang +roles: + prefill: + nodes: 2 + workers: 2 + gpus: 8 + env: + SGLANG_ENABLE_PREFILL_WAR_READ_DONE: "1" + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_DISAGGREGATION_NIXL_BACKEND: LIBFABRIC + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + OMP_NUM_THREADS: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_USE_CUSTOM_ALL_REDUCE_V2: '1' + SGLANG_OPT_USE_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '9216' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH: '1' + SGLANG_DSV4_MHC_PREWARM: '1' + + args: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + enable-deepseek-v4-fp4-indexer: true + enable-w4a4-mxfp4-megamoe: true + disaggregation-transfer-backend: nixl + disaggregation-mode: prefill + load-balance-method: total_tokens + mem-fraction-static: 0.85 + page-size: 256 + swa-full-tokens-ratio: 0.02 + max-running-requests: 256 + cuda-graph-max-bs-prefill: 256 + chunked-prefill-size: 65536 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5557"}' + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + enable-hierarchical-cache: true + hicache-write-policy: write_back + hicache-ratio: 3 + hicache-io-backend: direct + + decode: + nodes: 1 + workers: 1 + gpus: 8 + env: + SGLANG_RAGGED_VERIFY_MODE: "static" + SGLANG_DISAGGREGATION_WAITING_TIMEOUT: "900" + SGLANG_DISAGGREGATION_NIXL_BACKEND: LIBFABRIC + PIP_BREAK_SYSTEM_PACKAGES: '1' + PYTHONUNBUFFERED: '1' + OMP_NUM_THREADS: '1' + SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK: '0' + SGLANG_DG_CACHE_DIR: /configs/deepgemm_cache + SGLANG_JIT_DEEPGEMM_PRECOMPILE: '1' + SGLANG_JIT_DEEPGEMM_FAST_WARMUP: '1' + SGLANG_DEFAULT_THINKING: '1' + SGLANG_DSV4_REASONING_EFFORT: high + SGLANG_OPT_SWA_SPLIT_LEAF_ON_INSERT: '1' + SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS: "1" + SGLANG_OPT_DEEPGEMM_MEGA_MOE: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '4096' + SGLANG_OPT_USE_ONLINE_COMPRESS: '0' + NCCL_CUMEM_ENABLE: '1' + NCCL_TIMEOUT: '100000' + SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW: '1' + SGLANG_CLIP_MAX_NEW_TOKENS_ESTIMATION: '8' + DYN_SKIP_SGLANG_LOG_FORMATTING: '1' + DYN_LOG: info,dynamo_runtime::pipeline::network::ingress::push_handler=warn + SGLANG_LOG_FORWARD_ITERS: '1' + SGLANG_LOG_MS: '1' + SGLANG_REQUEST_STATE_WAIT_TIMEOUT: '60' + SGLANG_DSV4_MHC_PREWARM: '1' + + args: + host: 0.0.0.0 + served-model-name: deepseek-ai/DeepSeek-V4-Pro-0813 + enable-metrics: true + enable-cache-report: true + model-path: /model/ + trust-remote-code: true + watchdog-timeout: 86400 + stream-interval: 60 + tp-size: 8 + dp-size: 8 + ep-size: 8 + enable-dp-attention: true + enable-dp-lm-head: true + moe-dense-tp-size: 1 + moe-a2a-backend: megamoe + enable-deepseek-v4-fp4-indexer: true + enable-w4a4-mxfp4-megamoe: true + disaggregation-transfer-backend: nixl + disaggregation-mode: decode + load-balance-method: total_tokens + mem-fraction-static: 0.9 + page-size: 256 + swa-full-tokens-ratio: 0.02 + max-running-requests: 960 + cuda-graph-max-bs-decode: 256 + disable-flashinfer-autotune: true + model-loader-extra-config: '{"enable_multithread_load":true,"num_threads":8}' + kv-events-config: '{"publisher":"zmq","topic":"kv-events","endpoint":"tcp://*:5567"}' + speculative-algorithm: DSPARK + speculative-dspark-block-size: 6 + speculative-num-steps: 1 + speculative-eagle-topk: 1 + speculative-num-draft-tokens: 7 + + +sbatch_directives: + mem: "0" + exclude: "dsxe-sa-b300-prd0-gpu-00,dsxe-sa-b300-prd0-gpu-01,dsxe-sa-b300-prd0-gpu-04,dsxe-sa-b300-prd0-gpu-06,dsxe-sa-b300-prd0-gpu-11,dsxe-sa-b300-prd0-gpu-16" + +srun_options: + mem: "0" + container-remap-root: "" + +benchmark: + type: custom + command: bash /infmax-workspace/benchmarks/srt_agentic.sh + env: + INFMAX_CONTAINER_WORKSPACE: /infmax-workspace + RESULT_DIR: /logs/agentic + PORT: "8000" + IS_MULTINODE: "true" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" + AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" + HF_HUB_CACHE: "/hf_hub_cache" diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 07c47e6e40..a70384d0e6 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -1161,6 +1161,104 @@ dsv4-fp4-b300-sglang-agentic-hicache-eagle: - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-eagle/agentic.yaml } - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [64, 128, 256, 384, 512], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv4/sglang/b300-fp4-eagle/agentic.yaml } +dsv4-fp4-b300-dynamo-sglang-agentic-agg: + image: lmsysorg/sglang:nightly-dev-20260916-c9a8fba9 + model: deepseek-ai/DeepSeek-V4-Pro-0813 + model-prefix: dsv4 + runner: cluster:b300-dsxe + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.5.0.dev20260914" } + multinode: true + disagg: false + scenarios: + agentic-coding: + - search-space: + - spec-decoding: draft_model + conc-list: [1] + num-nodes: 1 + worker: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/dsv4/sglang/b300-fp4/agentx/agg-tp8-c1-mtp.yaml" + - spec-decoding: draft_model + conc-list: [4] + num-nodes: 1 + worker: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/dsv4/sglang/b300-fp4/agentx/agg-tp4-c4-mtp.yaml" + +dsv4-fp4-b300-dynamo-sglang-agentic-disagg: + image: lmsysorg/sglang:nightly-dev-20260916-c9a8fba9 + model: deepseek-ai/DeepSeek-V4-Pro-0813 + model-prefix: dsv4 + runner: cluster:b300-dsxe + precision: fp4 + framework: dynamo-sglang + router: { name: dynamo-router, version: "1.5.0.dev20260914" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + agentic-coding: + - dram-utilization: 0.80 + search-space: + - spec-decoding: draft_model + conc-list: [64] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/dsv4/sglang/b300-fp4/agentx/disagg-1p1d-dep8-dep8-c64-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - spec-decoding: draft_model + conc-list: [240] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/dsv4/sglang/b300-fp4/agentx/disagg-1p1d-dep8-dep8-c240-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + - spec-decoding: draft_model + conc-list: [480] + kv-offloading: dram + kv-offload-backend: { name: hicache } + prefill: + num-worker: 2 + tp: 8 + ep: 8 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/dsv4/sglang/b300-fp4/agentx/disagg-2p1d-dep8-dep8-c480-mtp-kvoffload.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 8 + dp-attn: true + qwen3.5-fp8-b200-sglang: image: lmsysorg/sglang:nightly-dev-cu13-20260918-20518d85 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/inferencex-e2e/configs/runners.yaml b/inferencex-e2e/configs/runners.yaml index cf67501a63..7f25a1221b 100644 --- a/inferencex-e2e/configs/runners.yaml +++ b/inferencex-e2e/configs/runners.yaml @@ -541,6 +541,7 @@ clusters: cpus-per-gpu: 24 salloc-args: [--mem=0] volumes: + aiperf-cache: {path: /data/home/sa-gha-runner/aiperf-cache} hf-home: {path: ~/.cache/huggingface} hf-hub-cache: {path: ~/.cache/huggingface/hub} scratch: {path: /scratch/models, visibility: node-local} diff --git a/inferencex-e2e/infx/launch/drivers/srt/lanes.py b/inferencex-e2e/infx/launch/drivers/srt/lanes.py index e8217eb607..6ec7f1d0e1 100644 --- a/inferencex-e2e/infx/launch/drivers/srt/lanes.py +++ b/inferencex-e2e/infx/launch/drivers/srt/lanes.py @@ -64,7 +64,10 @@ class SrtLane: ), mounts=_AGENTIC_CACHES, ), - ("b300-dsxe", LaunchPath.SRT_MULTI): SrtLane(frameworks=_DYNAMO), + ("b300-dsxe", LaunchPath.SRT_MULTI): SrtLane( + frameworks=_DYNAMO, + mounts=_AGENTIC_CACHES, + ), ("gb200-nv", LaunchPath.SRT_MULTI): SrtLane( frameworks=_DYNAMO, setup_scripts={"dynamo-sglang": "install-torchao.sh"}, diff --git a/inferencex-e2e/infx/tests/launch/test_srt_policy.py b/inferencex-e2e/infx/tests/launch/test_srt_policy.py index ce4b3d00d5..2404c4423f 100644 --- a/inferencex-e2e/infx/tests/launch/test_srt_policy.py +++ b/inferencex-e2e/infx/tests/launch/test_srt_policy.py @@ -73,6 +73,23 @@ def test_a_path_without_a_lane_on_the_cluster_is_refused(): srt_lane("b200-cw", LaunchPath.SRT_MULTI) +def test_b300_multinode_agentic_lane_mounts_standard_caches(): + lane = srt_lane("b300-dsxe", LaunchPath.SRT_MULTI) + agentic = request(IS_AGENTIC="1") + regular = request(IS_AGENTIC="0") + + matched = [ + (mount.volume, mount.target, mount.world_writable) + for mount in lane.mounts + if mount.when(agentic) + ] + assert matched == [ + ("aiperf-cache", "/aiperf_mmap_cache", True), + ("hf-hub-cache", "/hf_hub_cache", True), + ] + assert not any(mount.when(regular) for mount in lane.mounts) + + OVERRIDES = ( Override(Match(frameworks=any_of("sglang")), entry="M"), Override(Match(frameworks=any_of("trt")), served_name="served-m"), diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index d7fc64f0d6..7e8503c591 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9149,3 +9149,17 @@ - "Use AITER attention and allreduce fusion, FP8 KV (fp8_e4m3), and EAGLE MTP (3 steps, topk 1, 4 draft tokens); do not enable ROCm INT4 quick all-reduce. HiCache uses ratio 1.5, write_through_selective, kernel I/O and page_first layout. The matching cookbook recipe is sgl-project/sglang#41849." - "The embedded MTP head runs at its stored precision: block-FP8 expert weights and checkpoint dtype for mtp.fc and gates. No separate draft, draft dtype override or submission-side quantization is used." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3602 + +- config-keys: + - dsv4-fp4-b300-dynamo-sglang-agentic-agg + - dsv4-fp4-b300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Add DeepSeek-V4-Pro-0813 FP4 B300 Dynamo+SGLang AgentX aggregate TP8 c1 and TP4 c4 recipes, plus disaggregated 1P1D DEP8 c64/c240 and 2P1D DEP8 c480 recipes, under the refactored inferencex-e2e project layout." + - "Use NIXL with the LIBFABRIC backend for disaggregated KV transfer, relying on the networking stack provided by the pinned container and DSXE runtime without explicit host EFA/OFI mounts or runtime transport setup." + - "Use the B300 DSXE runner's 24-CPUs-per-GPU policy, HiCache ratio 3, decode max-running-requests at twice concurrency, persistent AgentX/Hugging Face caches, and exclude the known degraded DSXE nodes." + - "在重构后的 inferencex-e2e 项目布局中新增 DeepSeek-V4-Pro-0813 FP4 B300 Dynamo+SGLang AgentX 配方:聚合式 TP8 c1、TP4 c4,以及分离式 1P1D DEP8 c64/c240 和 2P1D DEP8 c480。" + - "分离式 KV 传输使用 NIXL 的 LIBFABRIC 后端,依赖固定容器与 DSXE 运行时提供的网络栈,不显式挂载主机 EFA/OFI 目录,也不在运行时安装传输组件。" + - "使用 B300 DSXE 运行器的每 GPU 24 个 CPU 策略、HiCache 比例 3,decode 的 max-running-requests 为并发数两倍,挂载持久化 AgentX/Hugging Face 缓存,并排除已知异常 DSXE 节点。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3597