diff --git a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh index 804d65b08..6091369d2 100644 --- a/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/qwen3.5_fp4_mi355x_sglang_mtp.sh @@ -11,7 +11,8 @@ source "$(dirname "$0")/../../benchmark_lib.sh" export EVAL_FRAMEWORK="lm-eval" check_env_vars \ - MODEL TP CONC EP_SIZE RESULT_DIR DURATION + MODEL TP CONC EP_SIZE KV_OFFLOADING \ + TOTAL_CPU_DRAM_GB RESULT_DIR DURATION SCHEDULER_RECV_INTERVAL=${SCHEDULER_RECV_INTERVAL:-30} @@ -53,6 +54,99 @@ trap cleanup_agentic_services EXIT trap 'exit 130' INT trap 'exit 143' TERM +# Resident arms keep the page size the TP2/EP2 and TP4 sweeps were measured +# with. HiCache arms move to 64 because Qwen3.5's hybrid attention/Mamba host +# pools transfer page-first, and page size 1 fails the EAGLE verify-graph +# compile on gfx950. +CACHE_ARGS=() +PAGE_SIZE=16 +if require_agentic_kv_offload_backend hicache; then + PAGE_SIZE=64 + + # sgl-project/sglang#30393 (merged 2026-08-06) routes an MTP draft KV cache + # to either a packed or a sidecar HiCache pool. Qwen3.5 conditional- + # generation checkpoints keep their language-model attributes in the nested + # text_config, and SGLang normalizes the draft depth only on the parent HF + # config, so ModelConfig.num_nextn_predict_layers stays None, the draft is + # misrouted to the sidecar path, and the scheduler dies during startup with + # AttributeError: 'HybridLinearKVPool' object has no attribute 'layer_num' + # Apply the one-line fix from sgl-project/sglang#34560 until it ships in an + # MI355X image. Resident runs never reach this routing, so the patch stays + # scoped to the HiCache arms and leaves the measured resident data alone. + python3 - /sgl-workspace/sglang/python/sglang/srt/configs/model_config.py <<'PYPATCH' +import sys + +path = sys.argv[1] +anchor = 'self.hf_config.architectures[0] = "Qwen3_5ForCausalLMMTP"' +assign = "self.hf_config.num_nextn_predict_layers = 1" +fix = "self.hf_text_config.num_nextn_predict_layers = 1" + +with open(path) as fh: + lines = fh.readlines() + +matches = [i for i, line in enumerate(lines) if line.strip() == anchor] +if len(matches) != 1: + sys.exit(f"sglang#34560: expected 1 Qwen3.5 MTP anchor in {path}, found {len(matches)}") + +i = matches[0] +if lines[i + 1].strip() != assign: + sys.exit(f"sglang#34560: unexpected line after anchor in {path}: {lines[i + 1]!r}") +if lines[i + 2].strip() == fix: + print("sglang#34560 already applied") + sys.exit(0) + +indent = lines[i + 1][: len(lines[i + 1]) - len(lines[i + 1].lstrip())] +lines.insert(i + 2, f"{indent}{fix}\n") +with open(path, "w") as fh: + fh.writelines(lines) +print("sglang#34560 applied") +PYPATCH + + # --hicache-size is the per-rank budget SGLang splits across Qwen3.5's two + # hybrid host pools, not a per-pool figure: on this image at TP4 with + # --hicache-size 144 the ranks allocated 93.37 GB target KV + 50.65 GB Mamba + # = 144.02 GB each. Packed NEXTN then adds its single draft layer on top of + # the 60 transferred target layers. Hold the node to 80% of the workflow + # DRAM budget so the draft layer, page alignment, and the trace-replay + # client cannot walk the host into the OOM killer mid-storm. + HICACHE_ALIGNMENT_RESERVE_GB=$TP + HICACHE_USABLE_TOTAL_GB=$((TOTAL_CPU_DRAM_GB - HICACHE_ALIGNMENT_RESERVE_GB)) + if [ "$HICACHE_USABLE_TOTAL_GB" -lt 1 ]; then + echo "Error: insufficient DRAM after HiCache alignment reserve." >&2 + exit 1 + fi + MAX_HICACHE_SIZE_GB=$((HICACHE_USABLE_TOTAL_GB * 80 / 100 * 60 / 61 / TP)) + # 144 GB/rank is the largest pool observed to allocate on this image; 180 + # is a bounded step up from it. Raise once a run confirms the larger pinned + # allocation stays inside the watchdog. + HICACHE_MAX_SIZE_GB_PER_RANK=${HICACHE_MAX_SIZE_GB_PER_RANK:-180} + if [ "$MAX_HICACHE_SIZE_GB" -gt "$HICACHE_MAX_SIZE_GB_PER_RANK" ]; then + MAX_HICACHE_SIZE_GB="$HICACHE_MAX_SIZE_GB_PER_RANK" + fi + HICACHE_SIZE_GB="${HICACHE_SIZE_GB:-$MAX_HICACHE_SIZE_GB}" + if [ "$HICACHE_SIZE_GB" -lt 1 ] || [ "$HICACHE_SIZE_GB" -gt "$MAX_HICACHE_SIZE_GB" ]; then + echo "Error: HICACHE_SIZE_GB=$HICACHE_SIZE_GB outside 1..$MAX_HICACHE_SIZE_GB." >&2 + exit 1 + fi + PROJECTED_HICACHE_TOTAL_GB=$(((HICACHE_SIZE_GB * TP * 61 + 59) / 60 + HICACHE_ALIGNMENT_RESERVE_GB)) + if [ "$PROJECTED_HICACHE_TOTAL_GB" -gt "$TOTAL_CPU_DRAM_GB" ]; then + echo "Error: projected HiCache use ${PROJECTED_HICACHE_TOTAL_GB} GB exceeds configured ${TOTAL_CPU_DRAM_GB} GB." >&2 + exit 1 + fi + echo "HiCache pools: ${HICACHE_SIZE_GB} GB per rank across TP=${TP}; projected node total ${PROJECTED_HICACHE_TOTAL_GB} GB of ${TOTAL_CPU_DRAM_GB} GB." + + # kernel + page_first is the transfer path the hybrid KV/Mamba stack already + # builds on gfx950 (both host pools and the pool-stack attach complete under + # it). write_through_selective matches the B300 Qwen3.5 MTP sibling. + CACHE_ARGS=( + --enable-hierarchical-cache + --hicache-size "$HICACHE_SIZE_GB" + --hicache-io-backend kernel + --hicache-mem-layout page_first + --hicache-write-policy write_through_selective + ) +fi + PARALLEL_ARGS=( --tp "$TP" --dp 1 @@ -93,7 +187,7 @@ SGLANG_CMD=( --mem-fraction-static 0.80 --model-loader-extra-config '{"enable_multithread_load": true}' --watchdog-timeout 1200 - --page-size 16 + --page-size "$PAGE_SIZE" --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" --max-running-requests "$MAX_RUNNING_REQUESTS" --max-prefill-tokens 32768 @@ -110,6 +204,7 @@ SGLANG_CMD=( --speculative-num-draft-tokens 4 --enable-metrics --enable-cache-report + "${CACHE_ARGS[@]}" ) printf '%q ' "${SGLANG_CMD[@]}" | tee "$RESULT_DIR/sglang_command.txt" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 07f3defd9..fbbe1ba64 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -386,8 +386,16 @@ qwen3.5-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20] } - - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40] } + # Isolating one HiCache point to prove sgl-project/sglang#34560 clears the + # HybridLinearKVPool draft-sidecar crash on gfx950. The resident arms are + # already merged and measured (#2562), so they stay commented out to keep + # this iteration off the MI355X node; restore all four rows once the + # HiCache path boots. + # - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20] } + # - { tp: 2, ep: 2, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16, 20] } + # - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: none, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40] } + # - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40] } + - { tp: 4, ep: 1, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16] } qwen3.5-fp4-mi355x-sglang-disagg: image: lmsysorg/sglang-rocm:v0.5.12.post1-rocm720-mi35x-20260523 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a1875d61b..a49d20db8 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5894,3 +5894,13 @@ - "Cover the measured resident TP2/EP2 and TP4 Pareto ranges through their HBM capacity knees, with required SGLang metrics exports." - "Use SGLang v0.5.17 and disable unstable AITER all-reduce fusion for TP2/EP2 EAGLE rank consistency." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2562 + +- config-keys: + - qwen3.5-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Add HiCache DRAM KV-offload support to the recipe and isolate a single TP4 concurrency 16 point to prove the path boots on gfx950; the resident arms and the remaining HiCache points stay commented out for this iteration." + - "Run the HiCache arm at page size 64 with kernel io-backend and page_first layout, sized to 80 percent of the workflow DRAM budget at a 180 GB per-rank ceiling." + - "Patch sgl-project/sglang#34560 into the container for the HiCache arms only: Qwen3.5 leaves num_nextn_predict_layers unset on hf_text_config, which misroutes the MTP draft cache to the sidecar path and aborts startup." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2582